diff --git a/src/cli/llm.py b/src/cli/llm.py index 846724a..39fd695 100644 --- a/src/cli/llm.py +++ b/src/cli/llm.py @@ -346,7 +346,7 @@ def tag( ) -_GOLDEN_DEFAULT = "tests/llm/golden_themes.yaml" +_GOLDEN_DEFAULT = "tests/llm/golden_themes_v3.yaml" @app.command("eval-tags") @@ -375,20 +375,22 @@ def eval_tags( help="With --live: write every comment's shortlist, verdicts and confidences to this JSON file.", ), split: str = typer.Option( - "all", "--split", help="Golden v2 half to score: all | tune | heldout." + "heldout", + "--split", + help="Golden half to score: heldout (the gate) | tune | all.", ), docket: str = typer.Option("", "--docket", help="Score one docket's entries."), - min_f1: float = typer.Option(0.70, "--min-f1", help="Gate: micro-F1 floor."), + min_f1: float = typer.Option(0.75, "--min-f1", help="Gate: micro-F1 floor."), min_precision: float = typer.Option( - 0.50, "--min-precision", help="Gate: per-theme precision floor (support >= 5)." + 0.60, "--min-precision", help="Gate: per-theme precision floor (support >= 5)." ), ) -> None: - """Score the theme tagger against the golden set (P35 gate, #577): - per-theme precision/recall, micro-F1 and the abstain rate, then the - fan-out gate (micro-F1 ≥ 0.70; no theme with support ≥ 5 under 0.5 - precision). Default reads the tags the last run stored on each item - (no model calls); --live re-tags each golden comment now. Exit 1 when - the gate fails.""" + """Score the theme tagger against the golden set: per-theme + precision/recall, micro-F1 and the abstain rate, per docket, then the + gate. Default: golden v3 (200 comments, 8 dockets), held-out half, + micro-F1 ≥ 0.75 and no theme with support ≥ 5 under 0.6 precision. + Reads the tags the last run stored on each item (no model calls); + --live re-tags each golden comment now. Exit 1 when the gate fails.""" from conf.connect import bib from llm.tageval import ( format_report, @@ -403,7 +405,10 @@ def eval_tags( if split not in ("all", "tune", "heldout"): raise typer.BadParameter("--split is all, tune or heldout") - g = select(load_golden(golden), split=split, docket=docket) + full = load_golden(golden) + if not any(e.split for e in full.entries): + split = "all" # a golden file without halves (v1 format) is scored whole + g = select(full, split=split, docket=docket) v = load_vocab() if g.vocab_version != v.version: typer.echo( diff --git a/src/llm/tagger.py b/src/llm/tagger.py index 18a6af0..6fd4a77 100644 --- a/src/llm/tagger.py +++ b/src/llm/tagger.py @@ -297,11 +297,24 @@ def run( rows = sorted(rows, key=lambda r: r[0]) texts = [t for _, t, _ in rows] digest = content_hash(texts) - same_content = not force and state_matches( - extra, version=vocab.version, model=model, digest=digest - ) stored = extra.get(STATE_KEY) or {} - if same_content and stored.get("params") == params: + # Verdicts stand when the content and model are unchanged and the + # vocabulary is this version or one it declares compatible (it only + # added themes). + same_content = ( + not force + and stored.get("model") == model + and stored.get("content_hash") == digest + and ( + stored.get("version") == vocab.version + or stored.get("version") in vocab.compatible_with + ) + ) + if ( + same_content + and stored.get("version") == vocab.version + and stored.get("params") == params + ): stats["skipped_state"] += 1 if progress: progress(key, None, "unchanged") diff --git a/src/llm/vocab/__init__.py b/src/llm/vocab/__init__.py index 0a75a91..243801f 100644 --- a/src/llm/vocab/__init__.py +++ b/src/llm/vocab/__init__.py @@ -52,6 +52,9 @@ class Vocab: themes: tuple[Theme, ...] retired: tuple[str, ...] = () source: str = "" + #: Earlier versions whose stored judge verdicts still hold — this version + #: only added themes and left every earlier theme's text unchanged. + compatible_with: tuple[int, ...] = () _by_slug: dict[str, Theme] = field(default_factory=dict, repr=False, compare=False) def __post_init__(self) -> None: @@ -129,7 +132,19 @@ def parse(text: str, *, source: str = "") -> Vocab: clash = seen & set(retired) if clash: raise VocabError(f"retired slugs still active: {sorted(clash)}") - return Vocab(version=version, themes=themes, retired=retired, source=source) + try: + compatible = tuple(int(v) for v in data.get("compatible_with") or []) + except (TypeError, ValueError) as exc: + raise VocabError(f"{source}: compatible_with must list integers") from exc + if any(v >= version for v in compatible): + raise VocabError(f"{source}: compatible_with must name earlier versions") + return Vocab( + version=version, + themes=themes, + retired=retired, + source=source, + compatible_with=compatible, + ) def load(path: Path | str | None = None) -> Vocab: diff --git a/src/llm/vocab/themes.yaml b/src/llm/vocab/themes.yaml index 11198ca..23bb69e 100644 --- a/src/llm/vocab/themes.yaml +++ b/src/llm/vocab/themes.yaml @@ -6,7 +6,12 @@ # retire a theme by moving it to `retired:` rather than deleting it, and # bump `version` whenever a slug, definition or synonym list changes, since # every tag run records the version it used. -version: 2 +# v3 (2026-11-11): added six hospital-side themes for the OPPS/ASC, IPPS +# and hospital dockets (golden v2 found their comments off-vocabulary). +# Every v2 theme's text is unchanged, so v2 verdicts still hold +# (compatible_with) and the tagger only judges the new themes. +version: 3 +compatible_with: [2] themes: - slug: conversion-factor label: Conversion factor and payment update @@ -293,4 +298,34 @@ themes: definition: Payment or coverage for radiation oncology, proton beam therapy, or radiation treatment delivery codes. synonyms: [radiation therapy, radiation oncology, proton therapy, proton beam, treatment delivery, IMRT] sections: [Radiation Therapy Services] + - slug: opps-asc-payment + label: Hospital outpatient and ASC payment + definition: Hospital outpatient (OPPS) and ambulatory surgical center payment - APC assignment and payment rates, the ASC covered procedures list and ASC rate update, the inpatient-only list and two-midnight rule, or moving a procedure between inpatient, outpatient and ASC settings. + synonyms: [OPPS, APC, ambulatory payment classification, ASC, ambulatory surgical center, covered procedures list, CPL, inpatient only list, IPO list, two-midnight rule, hospital outpatient payment, status indicator] + sections: [Updates Affecting OPPS Payments, Ambulatory Surgical Center Payment System, Changes to the Inpatient Only List] + - slug: ipps-drg-payment + label: Inpatient hospital and LTCH payment + definition: Inpatient prospective payment - MS-DRG assignment and relative weights, new technology add-on payments, outlier payments, standardized amounts, or long-term care hospital payment. + synonyms: [IPPS, MS-DRG, DRG, relative weight, new technology add-on payment, NTAP, outlier payment, LTCH, long-term care hospital, standardized amount] + sections: [Changes to MS-DRG Classifications and Relative Weights, New Technology Add-On Payments] + - slug: hospital-wage-index + label: Hospital wage index + definition: The hospital wage index, occupational mix adjustment, rural floor, geographic reclassification, or the low wage index hospital policy. + synonyms: [wage index, occupational mix, rural floor, reclassification, MGCRB, low wage index hospital, CBSA, imputed floor] + sections: [Changes to the Hospital Wage Index] + - slug: non-opioid-pain + label: Non-opioid pain management (NOPAIN) + definition: Separate Medicare payment for non-opioid drugs and devices for pain relief under the NOPAIN Act, or which products qualify for it. Not general clinical practice that limits opioid use, such as opioid-sparing anesthesia. + synonyms: [NOPAIN Act, non-opioid, non-opioid pain management, opioid alternatives, non-opioid treatments for pain relief] + sections: [Payment for Non-Opioid Treatments for Pain Relief] + - slug: hospital-quality-programs + label: Hospital quality and value programs + definition: Hospital quality reporting and value-based programs - Hospital OQR and IQR, Hospital Value-Based Purchasing, HAC Reduction, Hospital Readmissions Reduction, the Overall Star Rating, and the hospital measures in them. + synonyms: [OQR, IQR, Hospital Outpatient Quality Reporting, Inpatient Quality Reporting, Hospital Value-Based Purchasing, HAC Reduction Program, hospital-acquired condition, readmissions reduction, star rating, hospital-onset] + sections: [Hospital Outpatient Quality Reporting (OQR) Program, Hospital Inpatient Quality Reporting (IQR) Program] + - slug: hospital-conditions-of-participation + label: Hospital conditions of participation and safety + definition: The conditions hospitals must meet to participate in Medicare and their patient-safety standards, such as infection-control or masking requirements, emergency services readiness, or obstetric services standards. Not quality measures or quality-program reporting, such as hospital-onset infection measures in the HAC or IQR programs (that is hospital-quality-programs). + synonyms: [conditions of participation, CoP, infection control requirement, masking requirement, patient safety standard, emergency services readiness, obstetric services] + sections: [Conditions of Participation] retired: [] diff --git a/tests/llm/golden_themes_v3.yaml b/tests/llm/golden_themes_v3.yaml new file mode 100644 index 0000000..1936af6 --- /dev/null +++ b/tests/llm/golden_themes_v3.yaml @@ -0,0 +1,207 @@ +# Golden set v3: golden v2 (same 200 comments, same split) relabelled for +# vocab v3 on 2026-11-11 — off-vocabulary OPPS/ASC, IPPS, wage index and +# masking comments mapped to the new hospital themes; hospital-program +# quality comments moved to hospital-quality-programs and NOPAIN letters +# to non-opioid-pain (their v2 labels kept as acceptable). +vocab_version: 3 +entries: + - {comment_id: CMS-2026-2377-1425, docket: CMS-2026-2377, split: tune, expected: [work-rvu, behavioral-health], acceptable: [misvalued-codes, rural-access]} + - {comment_id: CMS-2026-2377-1437, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [evaluation-management, beneficiary-cost-sharing]} + - {comment_id: CMS-2026-2377-4505, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [evaluation-management, beneficiary-cost-sharing, rural-access]} + - {comment_id: CMS-2026-2377-4792, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [evaluation-management, practice-expense, global-surgery]} + - {comment_id: CMS-2026-2377-5030, docket: CMS-2026-2377, split: tune, expected: [conversion-factor], acceptable: [rural-access]} + - {comment_id: CMS-2026-2377-11968, docket: CMS-2026-2377, split: heldout, expected: [behavioral-health], acceptable: [work-rvu, conversion-factor, practice-expense]} + - {comment_id: CMS-2026-2377-12535, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [evaluation-management, practice-expense, global-surgery]} + - {comment_id: CMS-2026-2377-14749, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [work-rvu, global-surgery, efficiency-adjustment, conversion-factor, practice-expense]} + - {comment_id: CMS-2026-2377-17164, docket: CMS-2026-2377, split: heldout, expected: [], acceptable: [], off_vocab: "off-topic: Medicaid/welfare eligibility complaint"} + - {comment_id: CMS-2026-2377-17974, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [beneficiary-cost-sharing, rural-access, evaluation-management]} + - {comment_id: CMS-2026-2377-20108, docket: CMS-2026-2377, split: heldout, expected: [misvalued-codes], acceptable: [work-rvu, laboratory, coding-descriptors]} + - {comment_id: CMS-2026-2377-21257, docket: CMS-2026-2377, split: heldout, expected: [coding-descriptors], acceptable: [supervision, non-physician-practitioners, work-rvu]} + - {comment_id: CMS-2026-2377-23075, docket: CMS-2026-2377, split: heldout, expected: [], acceptable: [conversion-factor, rural-access], off_vocab: "general opposition to the rule (no specific provision)"} + - {comment_id: CMS-2026-2377-24046, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [conversion-factor, rural-access, beneficiary-cost-sharing, evaluation-management]} + - {comment_id: CMS-2026-2377-24284, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [beneficiary-cost-sharing, rural-access, evaluation-management]} + - {comment_id: CMS-2026-2377-27938, docket: CMS-2026-2377, split: tune, expected: [therapy-services, practice-expense], acceptable: [conversion-factor, coding-descriptors, regulatory-impact, work-rvu, rural-access]} + - {comment_id: CMS-2026-2377-29215, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [beneficiary-cost-sharing, conversion-factor]} + - {comment_id: CMS-2026-2377-29510, docket: CMS-2026-2377, split: heldout, expected: [conversion-factor, modifier-25-same-day, practice-expense], acceptable: [administrative-burden, rural-access, evaluation-management]} + - {comment_id: CMS-2026-2377-30572, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [work-rvu, global-surgery, efficiency-adjustment, conversion-factor]} + - {comment_id: CMS-2026-2377-32438, docket: CMS-2026-2377, split: tune, expected: [lactation-services], acceptable: [beneficiary-cost-sharing, work-rvu]} + - {comment_id: CMS-2026-2377-32542, docket: CMS-2026-2377, split: tune, expected: [therapy-services], acceptable: [coding-descriptors, work-rvu, practice-expense]} + - {comment_id: CMS-2026-2377-34620, docket: CMS-2026-2377, split: heldout, expected: [misvalued-codes], acceptable: [work-rvu, laboratory, coding-descriptors]} + - {comment_id: CMS-2026-2377-35870, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [conversion-factor, beneficiary-cost-sharing]} + - {comment_id: CMS-2026-2377-41579, docket: CMS-2026-2377, split: tune, expected: [conversion-factor, modifier-25-same-day, innovation-models], acceptable: [work-rvu, global-surgery, efficiency-adjustment, evaluation-management]} + - {comment_id: CMS-2026-2377-42468, docket: CMS-2026-2377, split: tune, expected: [therapy-services], acceptable: [practice-expense, coding-descriptors, rural-access, regulatory-impact, administrative-burden, conversion-factor]} + - {comment_id: CMS-2018-0076-0848, docket: CMS-2018-0076, split: tune, expected: [supervision, imaging-radiology], acceptable: [non-physician-practitioners]} + - {comment_id: CMS-2018-0076-1405, docket: CMS-2018-0076, split: tune, expected: [supervision, imaging-radiology], acceptable: [non-physician-practitioners]} + - {comment_id: CMS-2018-0076-2440, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [coding-descriptors, administrative-burden]} + - {comment_id: CMS-2018-0076-2680, docket: CMS-2018-0076, split: tune, expected: [evaluation-management, modifier-25-same-day], acceptable: [coding-descriptors]} + - {comment_id: CMS-2018-0076-2964, docket: CMS-2018-0076, split: tune, expected: [evaluation-management, modifier-25-same-day], acceptable: [coding-descriptors, administrative-burden]} + - {comment_id: CMS-2018-0076-3018, docket: CMS-2018-0076, split: tune, expected: [modifier-25-same-day], acceptable: [beneficiary-cost-sharing, evaluation-management]} + - {comment_id: CMS-2018-0076-4786, docket: CMS-2018-0076, split: tune, expected: [laboratory], acceptable: [data-requests-rfi]} + - {comment_id: CMS-2018-0076-5907, docket: CMS-2018-0076, split: heldout, expected: [modifier-25-same-day, evaluation-management], acceptable: [administrative-burden, coding-descriptors]} + - {comment_id: CMS-2018-0076-6395, docket: CMS-2018-0076, split: heldout, expected: [evaluation-management], acceptable: []} + - {comment_id: CMS-2018-0076-7622, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [modifier-25-same-day, administrative-burden, quality-payment-program]} + - {comment_id: CMS-2018-0076-8431, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [modifier-25-same-day, administrative-burden, quality-payment-program]} + - {comment_id: CMS-2018-0076-9484, docket: CMS-2018-0076, split: heldout, expected: [practice-expense], acceptable: [site-of-service, work-rvu]} + - {comment_id: CMS-2018-0076-9808, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [beneficiary-cost-sharing, administrative-burden]} + - {comment_id: CMS-2018-0076-9852, docket: CMS-2018-0076, split: heldout, expected: [evaluation-management], acceptable: [beneficiary-cost-sharing, administrative-burden]} + - {comment_id: CMS-2018-0076-10355, docket: CMS-2018-0076, split: tune, expected: [modifier-25-same-day, evaluation-management], acceptable: [administrative-burden, coding-descriptors]} + - {comment_id: CMS-2018-0076-10926, docket: CMS-2018-0076, split: heldout, expected: [communication-technology-services], acceptable: [telehealth]} + - {comment_id: CMS-2018-0076-11026, docket: CMS-2018-0076, split: tune, expected: [supervision], acceptable: [administrative-burden, evaluation-management, non-physician-practitioners]} + - {comment_id: CMS-2018-0076-11395, docket: CMS-2018-0076, split: heldout, expected: [evaluation-management], acceptable: [conversion-factor]} + - {comment_id: CMS-2018-0076-11627, docket: CMS-2018-0076, split: tune, expected: [evaluation-management, modifier-25-same-day], acceptable: [opioid-treatment]} + - {comment_id: CMS-2018-0076-12012, docket: CMS-2018-0076, split: tune, expected: [supervision], acceptable: [administrative-burden, non-physician-practitioners, evaluation-management]} + - {comment_id: CMS-2018-0076-13074, docket: CMS-2018-0076, split: tune, expected: [evaluation-management, quality-payment-program], acceptable: [administrative-burden, quality-measures]} + - {comment_id: CMS-2018-0076-13338, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [coding-descriptors]} + - {comment_id: CMS-2018-0076-13645, docket: CMS-2018-0076, split: heldout, expected: [evaluation-management, modifier-25-same-day], acceptable: [administrative-burden]} + - {comment_id: CMS-2018-0076-14524, docket: CMS-2018-0076, split: heldout, expected: [practice-expense, imaging-radiology], acceptable: [coverage-policy, administrative-burden]} + - {comment_id: CMS-2018-0076-14790, docket: CMS-2018-0076, split: heldout, expected: [gpci-localities], acceptable: [rural-access]} + - {comment_id: CMS-2019-0109-0163, docket: CMS-2019-0109, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service, coverage-policy]} + - {comment_id: CMS-2019-0109-0299, docket: CMS-2019-0109, split: tune, expected: [], acceptable: [remote-monitoring], off_vocab: "medical device testimonial (wearable post-surgical monitor)"} + - {comment_id: CMS-2019-0109-0430, docket: CMS-2019-0109, split: tune, expected: [price-transparency, site-of-service], acceptable: [beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-0444, docket: CMS-2019-0109, split: heldout, expected: [price-transparency, site-of-service], acceptable: [beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-0502, docket: CMS-2019-0109, split: tune, expected: [opps-asc-payment], acceptable: [work-rvu]} + - {comment_id: CMS-2019-0109-0551, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [drugs-biologicals]} + - {comment_id: CMS-2019-0109-0651, docket: CMS-2019-0109, split: heldout, expected: [price-transparency, site-of-service], acceptable: [beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-0694, docket: CMS-2019-0109, split: heldout, expected: [drugs-biologicals], acceptable: []} + - {comment_id: CMS-2019-0109-0867, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: []} + - {comment_id: CMS-2019-0109-0944, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [interoperability-health-it, beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-0964, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: []} + - {comment_id: CMS-2019-0109-1021, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: []} + - {comment_id: CMS-2019-0109-1120, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [drugs-biologicals]} + - {comment_id: CMS-2019-0109-1283, docket: CMS-2019-0109, split: heldout, expected: [drugs-biologicals], acceptable: [coding-descriptors]} + - {comment_id: CMS-2019-0109-1359, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [site-of-service, beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-1730, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-1796, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [drugs-biologicals, beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-1910, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-1924, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-2340, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [drugs-biologicals]} + - {comment_id: CMS-2019-0109-2494, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-2715, docket: CMS-2019-0109, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service, coverage-policy]} + - {comment_id: CMS-2019-0109-3046, docket: CMS-2019-0109, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service, coverage-policy, innovation-models]} + - {comment_id: CMS-2019-0109-3237, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]} + - {comment_id: CMS-2019-0109-3701, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]} + - {comment_id: CMS-2020-0088-1134, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe]} + - {comment_id: CMS-2020-0088-1246, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe]} + - {comment_id: CMS-2020-0088-1340, docket: CMS-2020-0088, split: tune, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe, administrative-burden]} + - {comment_id: CMS-2020-0088-1386, docket: CMS-2020-0088, split: tune, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe]} + - {comment_id: CMS-2020-0088-1961, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-3797, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor, telehealth], acceptable: [communication-technology-services, covid-phe, supervision, rural-access, regulatory-impact]} + - {comment_id: CMS-2020-0088-5375, docket: CMS-2020-0088, split: tune, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe]} + - {comment_id: CMS-2020-0088-7401, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor, telehealth], acceptable: [communication-technology-services, covid-phe, supervision, rural-access, regulatory-impact]} + - {comment_id: CMS-2020-0088-8032, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [communication-technology-services, telehealth, covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-8116, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [communication-technology-services, telehealth, covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-10383, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [communication-technology-services, telehealth, covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-12653, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor, telehealth], acceptable: [communication-technology-services, remote-monitoring, covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-14620, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-16755, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-17272, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-17866, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-18655, docket: CMS-2020-0088, split: heldout, expected: [telehealth], acceptable: [rural-access, covid-phe]} + - {comment_id: CMS-2020-0088-20121, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-22592, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-23016, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor, telehealth], acceptable: [therapy-services, covid-phe, work-rvu, evaluation-management]} + - {comment_id: CMS-2020-0088-23505, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-23605, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor, telehealth], acceptable: [covid-phe, regulatory-impact]} + - {comment_id: CMS-2020-0088-26793, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor], acceptable: [covid-phe]} + - {comment_id: CMS-2020-0088-28880, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor], acceptable: [work-rvu, non-physician-practitioners]} + - {comment_id: CMS-2020-0088-29770, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [evaluation-management, modifier-25-same-day, covid-phe, regulatory-impact]} + - {comment_id: CMS-2022-0113-1724, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management]} + - {comment_id: CMS-2022-0113-4292, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management]} + - {comment_id: CMS-2022-0113-5197, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-6188, docket: CMS-2022-0113, split: tune, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]} + - {comment_id: CMS-2022-0113-7459, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-8412, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management]} + - {comment_id: CMS-2022-0113-10435, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-11359, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-11411, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management]} + - {comment_id: CMS-2022-0113-12454, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-12647, docket: CMS-2022-0113, split: heldout, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]} + - {comment_id: CMS-2022-0113-13378, docket: CMS-2022-0113, split: tune, expected: [work-rvu], acceptable: [practice-expense, coding-descriptors]} + - {comment_id: CMS-2022-0113-13826, docket: CMS-2022-0113, split: heldout, expected: [non-physician-practitioners, coding-descriptors], acceptable: [coverage-policy, beneficiary-cost-sharing, administrative-burden]} + - {comment_id: CMS-2022-0113-14587, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-15010, docket: CMS-2022-0113, split: tune, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]} + - {comment_id: CMS-2022-0113-16694, docket: CMS-2022-0113, split: heldout, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]} + - {comment_id: CMS-2022-0113-17371, docket: CMS-2022-0113, split: tune, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]} + - {comment_id: CMS-2022-0113-17843, docket: CMS-2022-0113, split: heldout, expected: [work-rvu], acceptable: [misvalued-codes]} + - {comment_id: CMS-2022-0113-18650, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-19609, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-20145, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-22707, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-23123, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2022-0113-23609, docket: CMS-2022-0113, split: tune, expected: [dental-services], acceptable: [coverage-policy, non-physician-practitioners, rhc-fqhc, telehealth, opioid-treatment, data-requests-rfi, rural-access]} + - {comment_id: CMS-2022-0113-23709, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]} + - {comment_id: CMS-2023-0057-0182, docket: CMS-2023-0057, split: tune, expected: [hospital-conditions-of-participation], acceptable: [covid-phe]} + - {comment_id: CMS-2023-0057-0261, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]} + - {comment_id: CMS-2023-0057-0356, docket: CMS-2023-0057, split: heldout, expected: [hospital-conditions-of-participation], acceptable: [covid-phe, quality-measures, rural-access]} + - {comment_id: CMS-2023-0057-0371, docket: CMS-2023-0057, split: heldout, expected: [interoperability-health-it], acceptable: [administrative-burden, quality-measures]} + - {comment_id: CMS-2023-0057-0456, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, rural-access, quality-measures]} + - {comment_id: CMS-2023-0057-0492, docket: CMS-2023-0057, split: tune, expected: [hospital-conditions-of-participation], acceptable: [covid-phe]} + - {comment_id: CMS-2023-0057-0587, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, rural-access, quality-measures]} + - {comment_id: CMS-2023-0057-0814, docket: CMS-2023-0057, split: heldout, expected: [ipps-drg-payment], acceptable: []} + - {comment_id: CMS-2023-0057-0852, docket: CMS-2023-0057, split: heldout, expected: [ipps-drg-payment], acceptable: []} + - {comment_id: CMS-2023-0057-0895, docket: CMS-2023-0057, split: tune, expected: [hospital-conditions-of-participation], acceptable: [covid-phe, quality-measures, rural-access]} + - {comment_id: CMS-2023-0057-0952, docket: CMS-2023-0057, split: tune, expected: [hospital-conditions-of-participation], acceptable: [covid-phe]} + - {comment_id: CMS-2023-0057-1039, docket: CMS-2023-0057, split: tune, expected: [ipps-drg-payment], acceptable: []} + - {comment_id: CMS-2023-0057-1080, docket: CMS-2023-0057, split: heldout, expected: [ipps-drg-payment], acceptable: []} + - {comment_id: CMS-2023-0057-1306, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]} + - {comment_id: CMS-2023-0057-1350, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]} + - {comment_id: CMS-2023-0057-1408, docket: CMS-2023-0057, split: tune, expected: [ipps-drg-payment], acceptable: [behavioral-health, coverage-policy]} + - {comment_id: CMS-2023-0057-1420, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]} + - {comment_id: CMS-2023-0057-1860, docket: CMS-2023-0057, split: heldout, expected: [hospital-wage-index], acceptable: [gpci-localities]} + - {comment_id: CMS-2023-0057-2121, docket: CMS-2023-0057, split: tune, expected: [hospital-wage-index], acceptable: [gpci-localities]} + - {comment_id: CMS-2023-0057-2460, docket: CMS-2023-0057, split: tune, expected: [hospital-wage-index], acceptable: [gpci-localities]} + - {comment_id: CMS-2023-0057-2619, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]} + - {comment_id: CMS-2023-0057-2874, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]} + - {comment_id: CMS-2023-0057-2959, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]} + - {comment_id: CMS-2023-0057-2968, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]} + - {comment_id: CMS-2023-0057-3256, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, rural-access, quality-measures]} + - {comment_id: CMS-2024-0256-0359, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]} + - {comment_id: CMS-2024-0256-0614, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]} + - {comment_id: CMS-2024-0256-0918, docket: CMS-2024-0256, split: heldout, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]} + - {comment_id: CMS-2024-0256-0972, docket: CMS-2024-0256, split: heldout, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]} + - {comment_id: CMS-2024-0256-1173, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]} + - {comment_id: CMS-2024-0256-1296, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]} + - {comment_id: CMS-2024-0256-1504, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [supervision, coverage-policy, practice-expense, dme-supplies]} + - {comment_id: CMS-2024-0256-2071, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-2214, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-2250, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-2377, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-2594, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-2852, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-2992, docket: CMS-2024-0256, split: tune, expected: [dme-supplies], acceptable: [coverage-policy, dental-services, administrative-burden]} + - {comment_id: CMS-2024-0256-3674, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-3788, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-3919, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-3947, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-4590, docket: CMS-2024-0256, split: heldout, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]} + - {comment_id: CMS-2024-0256-4823, docket: CMS-2024-0256, split: heldout, expected: [evaluation-management], acceptable: [coding-descriptors, work-rvu, rural-access]} + - {comment_id: CMS-2024-0256-5075, docket: CMS-2024-0256, split: tune, expected: [evaluation-management], acceptable: [coding-descriptors, work-rvu, rural-access]} + - {comment_id: CMS-2024-0256-5396, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-5638, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-6253, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]} + - {comment_id: CMS-2024-0256-6651, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [coding-descriptors, beneficiary-cost-sharing, supervision]} + - {comment_id: CMS-2025-0306-0187, docket: CMS-2025-0306, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service]} + - {comment_id: CMS-2025-0306-0237, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]} + - {comment_id: CMS-2025-0306-0344, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]} + - {comment_id: CMS-2025-0306-0351, docket: CMS-2025-0306, split: tune, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]} + - {comment_id: CMS-2025-0306-0375, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]} + - {comment_id: CMS-2025-0306-0541, docket: CMS-2025-0306, split: heldout, expected: [opps-asc-payment], acceptable: [work-rvu, coding-descriptors]} + - {comment_id: CMS-2025-0306-0570, docket: CMS-2025-0306, split: tune, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]} + - {comment_id: CMS-2025-0306-0694, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]} + - {comment_id: CMS-2025-0306-0797, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]} + - {comment_id: CMS-2025-0306-0922, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]} + - {comment_id: CMS-2025-0306-0980, docket: CMS-2025-0306, split: heldout, expected: [efficiency-adjustment, practice-expense], acceptable: [site-of-service, work-rvu]} + - {comment_id: CMS-2025-0306-0994, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]} + - {comment_id: CMS-2025-0306-1219, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]} + - {comment_id: CMS-2025-0306-1306, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]} + - {comment_id: CMS-2025-0306-1323, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]} + - {comment_id: CMS-2025-0306-1413, docket: CMS-2025-0306, split: tune, expected: [opps-asc-payment], acceptable: [coverage-policy, site-of-service]} + - {comment_id: CMS-2025-0306-1548, docket: CMS-2025-0306, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service, conversion-factor]} + - {comment_id: CMS-2025-0306-1559, docket: CMS-2025-0306, split: tune, expected: [opps-asc-payment], acceptable: [behavioral-health, work-rvu]} + - {comment_id: CMS-2025-0306-1737, docket: CMS-2025-0306, split: tune, expected: [site-of-service], acceptable: [drugs-biologicals, beneficiary-cost-sharing]} + - {comment_id: CMS-2025-0306-1817, docket: CMS-2025-0306, split: tune, expected: [site-of-service], acceptable: [drugs-biologicals, beneficiary-cost-sharing]} + - {comment_id: CMS-2025-0306-2427, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]} + - {comment_id: CMS-2025-0306-2581, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [community-health-social-needs, rural-access, administrative-burden, quality-measures]} + - {comment_id: CMS-2025-0306-2712, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]} + - {comment_id: CMS-2025-0306-2728, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]} + - {comment_id: CMS-2025-0306-2742, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]} diff --git a/tests/llm/test_tageval.py b/tests/llm/test_tageval.py index 05275a3..a4319a8 100644 --- a/tests/llm/test_tageval.py +++ b/tests/llm/test_tageval.py @@ -22,13 +22,28 @@ from llm.vocab import load as load_vocab GOLDEN = Path(__file__).with_name("golden_themes.yaml") +class TestGoldenV3File: + def test_current_vocab_two_halves_and_known_slugs(self): + from pathlib import Path + + g = load_golden(Path(__file__).parent / "golden_themes_v3.yaml") + v = load_vocab() + assert g.vocab_version == v.version + assert len(g.entries) == 200 + assert {e.split for e in g.entries} == {"tune", "heldout"} + for e in g.entries: + assert e.expected or e.off_vocab, e.comment_id + for s in (*e.expected, *e.acceptable): + assert s in v, f"{e.comment_id}: unknown slug {s}" + + class TestGoldenFile: def test_loads_and_only_uses_known_slugs(self): g = load_golden(GOLDEN) v = load_vocab() assert ( isinstance(g, Golden) - and g.vocab_version == v.version + and (g.vocab_version == v.version or g.vocab_version in v.compatible_with) and g.docket == "CMS-2026-2377" ) assert len(g.entries) >= 30 diff --git a/tests/llm/test_tagger.py b/tests/llm/test_tagger.py index 2cd932b..79e4a4e 100644 --- a/tests/llm/test_tagger.py +++ b/tests/llm/test_tagger.py @@ -504,3 +504,43 @@ class TestRetune: stats = self._run(s, keys, judge, top=3, max_tags=3, min_confidence=0.0) assert stats["retuned"] == 1 assert not set(calls) & set(first) and len(calls) == 2 + + +def test_compatible_vocab_reuses_old_verdicts_and_judges_only_new_themes(tmp_path): + """v3 only adds themes (compatible_with: [2]): stored v2 verdicts stand; + a theme that is new to the vocab is judged the first time it is shortlisted.""" + from dataclasses import replace + + s, keys = _store(tmp_path) + calls = [] + + def judge(theme, evidence): + calls.append(theme.slug) + return True + + kw = dict( + docket="CMS-2026-2377", + load_chunks=TestRun()._chunks(keys), + card_vecs=CARDS, + judge=judge, + model="m", + top=2, + max_tags=3, + min_confidence=0.0, + ) + v2 = replace(VOCAB, version=2) + run(s, vocab=v2, **kw) + first = set(calls) + calls.clear() + v3 = replace(VOCAB, version=3, compatible_with=(2,)) + stats = run(s, vocab=v3, **kw) + assert stats["retuned"] == 1 and stats["tagged"] == 0 + assert not set(calls) & first + k = keys["CMS-2026-2377-1"] + extra = json.loads( + s._con().execute("SELECT extra_json FROM items WHERE key=?", (k,)).fetchone()[0] # noqa: SLF001 + ) + assert extra["llm_tags"]["version"] == 3 + calls.clear() + v4 = replace(VOCAB, version=4) # not compatible: full re-judge + assert run(s, vocab=v4, **kw)["tagged"] == 1 and calls diff --git a/tests/llm/test_vocab.py b/tests/llm/test_vocab.py index a043524..bc6038a 100644 --- a/tests/llm/test_vocab.py +++ b/tests/llm/test_vocab.py @@ -112,3 +112,17 @@ class TestTagHelpers: "docket:x", ] assert is_llm_tag("llm:x") and not is_llm_tag("llmx:y") + + +def test_compatible_with_lists_versions_whose_verdicts_still_hold(): + from llm.vocab import parse + + v = parse( + "version: 3\ncompatible_with: [2]\nthemes:\n" + " - {slug: telehealth, label: T, definition: d}\n" + ) + assert v.compatible_with == (2,) + assert ( + parse("version: 2\nthemes:\n - {slug: a, definition: d}\n").compatible_with + == () + )