feat(llm): theme vocabulary v3 for hospital dockets; eval-tags gates held-out v3
Some checks failed
CI / lint (push) Successful in 1m32s
CI / notebooks-smoke (push) Successful in 2m46s
Deploy / changes (push) Successful in 35s
CI / test (push) Failing after 8m25s
Infra CI / notebooks (push) Successful in 3m22s
Infra CI / zotero (push) Successful in 51s
Infra CI / docs (push) Successful in 49s
Infra CI / api (push) Successful in 5m16s
Infra CI / bcda (push) Successful in 5m44s
Infra CI / mc (push) Successful in 57s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 2m49s
Deploy / api (push) Successful in 2m12s
Deploy / bcda (push) Successful in 2m30s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m51s
Deploy / report (push) Successful in 41s
Some checks failed
CI / lint (push) Successful in 1m32s
CI / notebooks-smoke (push) Successful in 2m46s
Deploy / changes (push) Successful in 35s
CI / test (push) Failing after 8m25s
Infra CI / notebooks (push) Successful in 3m22s
Infra CI / zotero (push) Successful in 51s
Infra CI / docs (push) Successful in 49s
Infra CI / api (push) Successful in 5m16s
Infra CI / bcda (push) Successful in 5m44s
Infra CI / mc (push) Successful in 57s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 2m49s
Deploy / api (push) Successful in 2m12s
Deploy / bcda (push) Successful in 2m30s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m51s
Deploy / report (push) Successful in 41s
Vocab v3 adds six hospital-side themes the golden set showed missing: opps-asc-payment, ipps-drg-payment, hospital-wage-index, non-opioid-pain (NOPAIN), hospital-quality-programs and hospital-conditions-of-participation. Every v2 theme's text is unchanged and the file declares compatible_with: [2]; the tagger reuses all stored v2 verdicts and judges a new theme only when it enters a shortlist, so the corpus need not be re-judged. Golden v3 is golden v2 relabelled for v3 (50 entries; 3 genuinely off-topic comments remain). Live, 14b, top 12, floor 0.45: v2 held-out F1 0.74 (P 0.95 R 0.61, abstain 23%) v3 held-out F1 0.76 (P 0.89 R 0.67, abstain 14%) - gate PASS IPPS 0.29 -> 0.70, OPPS 0.27 -> 0.70; tune F1 0.81. Two new definitions were narrowed after both halves showed the same false positives (CoP on hospital-quality comments, NOPAIN on opioid-sparing letters). stack llm eval-tags with no flags now runs that gate: golden v3, held-out half, micro-F1 >= 0.75, per-theme precision >= 0.6; a golden file without halves is scored whole.
This commit is contained in:
kert
@@ -346,7 +346,7 @@ def tag(
|
||||
)
|
||||
|
||||
|
||||
_GOLDEN_DEFAULT = "tests/llm/golden_themes.yaml"
|
||||
_GOLDEN_DEFAULT = "tests/llm/golden_themes_v3.yaml"
|
||||
|
||||
|
||||
@app.command("eval-tags")
|
||||
@@ -375,20 +375,22 @@ def eval_tags(
|
||||
help="With --live: write every comment's shortlist, verdicts and confidences to this JSON file.",
|
||||
),
|
||||
split: str = typer.Option(
|
||||
"all", "--split", help="Golden v2 half to score: all | tune | heldout."
|
||||
"heldout",
|
||||
"--split",
|
||||
help="Golden half to score: heldout (the gate) | tune | all.",
|
||||
),
|
||||
docket: str = typer.Option("", "--docket", help="Score one docket's entries."),
|
||||
min_f1: float = typer.Option(0.70, "--min-f1", help="Gate: micro-F1 floor."),
|
||||
min_f1: float = typer.Option(0.75, "--min-f1", help="Gate: micro-F1 floor."),
|
||||
min_precision: float = typer.Option(
|
||||
0.50, "--min-precision", help="Gate: per-theme precision floor (support >= 5)."
|
||||
0.60, "--min-precision", help="Gate: per-theme precision floor (support >= 5)."
|
||||
),
|
||||
) -> None:
|
||||
"""Score the theme tagger against the golden set (P35 gate, #577):
|
||||
per-theme precision/recall, micro-F1 and the abstain rate, then the
|
||||
fan-out gate (micro-F1 ≥ 0.70; no theme with support ≥ 5 under 0.5
|
||||
precision). Default reads the tags the last run stored on each item
|
||||
(no model calls); --live re-tags each golden comment now. Exit 1 when
|
||||
the gate fails."""
|
||||
"""Score the theme tagger against the golden set: per-theme
|
||||
precision/recall, micro-F1 and the abstain rate, per docket, then the
|
||||
gate. Default: golden v3 (200 comments, 8 dockets), held-out half,
|
||||
micro-F1 ≥ 0.75 and no theme with support ≥ 5 under 0.6 precision.
|
||||
Reads the tags the last run stored on each item (no model calls);
|
||||
--live re-tags each golden comment now. Exit 1 when the gate fails."""
|
||||
from conf.connect import bib
|
||||
from llm.tageval import (
|
||||
format_report,
|
||||
@@ -403,7 +405,10 @@ def eval_tags(
|
||||
|
||||
if split not in ("all", "tune", "heldout"):
|
||||
raise typer.BadParameter("--split is all, tune or heldout")
|
||||
g = select(load_golden(golden), split=split, docket=docket)
|
||||
full = load_golden(golden)
|
||||
if not any(e.split for e in full.entries):
|
||||
split = "all" # a golden file without halves (v1 format) is scored whole
|
||||
g = select(full, split=split, docket=docket)
|
||||
v = load_vocab()
|
||||
if g.vocab_version != v.version:
|
||||
typer.echo(
|
||||
|
||||
@@ -297,11 +297,24 @@ def run(
|
||||
rows = sorted(rows, key=lambda r: r[0])
|
||||
texts = [t for _, t, _ in rows]
|
||||
digest = content_hash(texts)
|
||||
same_content = not force and state_matches(
|
||||
extra, version=vocab.version, model=model, digest=digest
|
||||
)
|
||||
stored = extra.get(STATE_KEY) or {}
|
||||
if same_content and stored.get("params") == params:
|
||||
# Verdicts stand when the content and model are unchanged and the
|
||||
# vocabulary is this version or one it declares compatible (it only
|
||||
# added themes).
|
||||
same_content = (
|
||||
not force
|
||||
and stored.get("model") == model
|
||||
and stored.get("content_hash") == digest
|
||||
and (
|
||||
stored.get("version") == vocab.version
|
||||
or stored.get("version") in vocab.compatible_with
|
||||
)
|
||||
)
|
||||
if (
|
||||
same_content
|
||||
and stored.get("version") == vocab.version
|
||||
and stored.get("params") == params
|
||||
):
|
||||
stats["skipped_state"] += 1
|
||||
if progress:
|
||||
progress(key, None, "unchanged")
|
||||
|
||||
@@ -52,6 +52,9 @@ class Vocab:
|
||||
themes: tuple[Theme, ...]
|
||||
retired: tuple[str, ...] = ()
|
||||
source: str = ""
|
||||
#: Earlier versions whose stored judge verdicts still hold — this version
|
||||
#: only added themes and left every earlier theme's text unchanged.
|
||||
compatible_with: tuple[int, ...] = ()
|
||||
_by_slug: dict[str, Theme] = field(default_factory=dict, repr=False, compare=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
@@ -129,7 +132,19 @@ def parse(text: str, *, source: str = "") -> Vocab:
|
||||
clash = seen & set(retired)
|
||||
if clash:
|
||||
raise VocabError(f"retired slugs still active: {sorted(clash)}")
|
||||
return Vocab(version=version, themes=themes, retired=retired, source=source)
|
||||
try:
|
||||
compatible = tuple(int(v) for v in data.get("compatible_with") or [])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise VocabError(f"{source}: compatible_with must list integers") from exc
|
||||
if any(v >= version for v in compatible):
|
||||
raise VocabError(f"{source}: compatible_with must name earlier versions")
|
||||
return Vocab(
|
||||
version=version,
|
||||
themes=themes,
|
||||
retired=retired,
|
||||
source=source,
|
||||
compatible_with=compatible,
|
||||
)
|
||||
|
||||
|
||||
def load(path: Path | str | None = None) -> Vocab:
|
||||
|
||||
@@ -6,7 +6,12 @@
|
||||
# retire a theme by moving it to `retired:` rather than deleting it, and
|
||||
# bump `version` whenever a slug, definition or synonym list changes, since
|
||||
# every tag run records the version it used.
|
||||
version: 2
|
||||
# v3 (2026-11-11): added six hospital-side themes for the OPPS/ASC, IPPS
|
||||
# and hospital dockets (golden v2 found their comments off-vocabulary).
|
||||
# Every v2 theme's text is unchanged, so v2 verdicts still hold
|
||||
# (compatible_with) and the tagger only judges the new themes.
|
||||
version: 3
|
||||
compatible_with: [2]
|
||||
themes:
|
||||
- slug: conversion-factor
|
||||
label: Conversion factor and payment update
|
||||
@@ -293,4 +298,34 @@ themes:
|
||||
definition: Payment or coverage for radiation oncology, proton beam therapy, or radiation treatment delivery codes.
|
||||
synonyms: [radiation therapy, radiation oncology, proton therapy, proton beam, treatment delivery, IMRT]
|
||||
sections: [Radiation Therapy Services]
|
||||
- slug: opps-asc-payment
|
||||
label: Hospital outpatient and ASC payment
|
||||
definition: Hospital outpatient (OPPS) and ambulatory surgical center payment - APC assignment and payment rates, the ASC covered procedures list and ASC rate update, the inpatient-only list and two-midnight rule, or moving a procedure between inpatient, outpatient and ASC settings.
|
||||
synonyms: [OPPS, APC, ambulatory payment classification, ASC, ambulatory surgical center, covered procedures list, CPL, inpatient only list, IPO list, two-midnight rule, hospital outpatient payment, status indicator]
|
||||
sections: [Updates Affecting OPPS Payments, Ambulatory Surgical Center Payment System, Changes to the Inpatient Only List]
|
||||
- slug: ipps-drg-payment
|
||||
label: Inpatient hospital and LTCH payment
|
||||
definition: Inpatient prospective payment - MS-DRG assignment and relative weights, new technology add-on payments, outlier payments, standardized amounts, or long-term care hospital payment.
|
||||
synonyms: [IPPS, MS-DRG, DRG, relative weight, new technology add-on payment, NTAP, outlier payment, LTCH, long-term care hospital, standardized amount]
|
||||
sections: [Changes to MS-DRG Classifications and Relative Weights, New Technology Add-On Payments]
|
||||
- slug: hospital-wage-index
|
||||
label: Hospital wage index
|
||||
definition: The hospital wage index, occupational mix adjustment, rural floor, geographic reclassification, or the low wage index hospital policy.
|
||||
synonyms: [wage index, occupational mix, rural floor, reclassification, MGCRB, low wage index hospital, CBSA, imputed floor]
|
||||
sections: [Changes to the Hospital Wage Index]
|
||||
- slug: non-opioid-pain
|
||||
label: Non-opioid pain management (NOPAIN)
|
||||
definition: Separate Medicare payment for non-opioid drugs and devices for pain relief under the NOPAIN Act, or which products qualify for it. Not general clinical practice that limits opioid use, such as opioid-sparing anesthesia.
|
||||
synonyms: [NOPAIN Act, non-opioid, non-opioid pain management, opioid alternatives, non-opioid treatments for pain relief]
|
||||
sections: [Payment for Non-Opioid Treatments for Pain Relief]
|
||||
- slug: hospital-quality-programs
|
||||
label: Hospital quality and value programs
|
||||
definition: Hospital quality reporting and value-based programs - Hospital OQR and IQR, Hospital Value-Based Purchasing, HAC Reduction, Hospital Readmissions Reduction, the Overall Star Rating, and the hospital measures in them.
|
||||
synonyms: [OQR, IQR, Hospital Outpatient Quality Reporting, Inpatient Quality Reporting, Hospital Value-Based Purchasing, HAC Reduction Program, hospital-acquired condition, readmissions reduction, star rating, hospital-onset]
|
||||
sections: [Hospital Outpatient Quality Reporting (OQR) Program, Hospital Inpatient Quality Reporting (IQR) Program]
|
||||
- slug: hospital-conditions-of-participation
|
||||
label: Hospital conditions of participation and safety
|
||||
definition: The conditions hospitals must meet to participate in Medicare and their patient-safety standards, such as infection-control or masking requirements, emergency services readiness, or obstetric services standards. Not quality measures or quality-program reporting, such as hospital-onset infection measures in the HAC or IQR programs (that is hospital-quality-programs).
|
||||
synonyms: [conditions of participation, CoP, infection control requirement, masking requirement, patient safety standard, emergency services readiness, obstetric services]
|
||||
sections: [Conditions of Participation]
|
||||
retired: []
|
||||
|
||||
207
tests/llm/golden_themes_v3.yaml
Normal file
207
tests/llm/golden_themes_v3.yaml
Normal file
@@ -0,0 +1,207 @@
|
||||
# Golden set v3: golden v2 (same 200 comments, same split) relabelled for
|
||||
# vocab v3 on 2026-11-11 — off-vocabulary OPPS/ASC, IPPS, wage index and
|
||||
# masking comments mapped to the new hospital themes; hospital-program
|
||||
# quality comments moved to hospital-quality-programs and NOPAIN letters
|
||||
# to non-opioid-pain (their v2 labels kept as acceptable).
|
||||
vocab_version: 3
|
||||
entries:
|
||||
- {comment_id: CMS-2026-2377-1425, docket: CMS-2026-2377, split: tune, expected: [work-rvu, behavioral-health], acceptable: [misvalued-codes, rural-access]}
|
||||
- {comment_id: CMS-2026-2377-1437, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [evaluation-management, beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2026-2377-4505, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [evaluation-management, beneficiary-cost-sharing, rural-access]}
|
||||
- {comment_id: CMS-2026-2377-4792, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [evaluation-management, practice-expense, global-surgery]}
|
||||
- {comment_id: CMS-2026-2377-5030, docket: CMS-2026-2377, split: tune, expected: [conversion-factor], acceptable: [rural-access]}
|
||||
- {comment_id: CMS-2026-2377-11968, docket: CMS-2026-2377, split: heldout, expected: [behavioral-health], acceptable: [work-rvu, conversion-factor, practice-expense]}
|
||||
- {comment_id: CMS-2026-2377-12535, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [evaluation-management, practice-expense, global-surgery]}
|
||||
- {comment_id: CMS-2026-2377-14749, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [work-rvu, global-surgery, efficiency-adjustment, conversion-factor, practice-expense]}
|
||||
- {comment_id: CMS-2026-2377-17164, docket: CMS-2026-2377, split: heldout, expected: [], acceptable: [], off_vocab: "off-topic: Medicaid/welfare eligibility complaint"}
|
||||
- {comment_id: CMS-2026-2377-17974, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [beneficiary-cost-sharing, rural-access, evaluation-management]}
|
||||
- {comment_id: CMS-2026-2377-20108, docket: CMS-2026-2377, split: heldout, expected: [misvalued-codes], acceptable: [work-rvu, laboratory, coding-descriptors]}
|
||||
- {comment_id: CMS-2026-2377-21257, docket: CMS-2026-2377, split: heldout, expected: [coding-descriptors], acceptable: [supervision, non-physician-practitioners, work-rvu]}
|
||||
- {comment_id: CMS-2026-2377-23075, docket: CMS-2026-2377, split: heldout, expected: [], acceptable: [conversion-factor, rural-access], off_vocab: "general opposition to the rule (no specific provision)"}
|
||||
- {comment_id: CMS-2026-2377-24046, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [conversion-factor, rural-access, beneficiary-cost-sharing, evaluation-management]}
|
||||
- {comment_id: CMS-2026-2377-24284, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [beneficiary-cost-sharing, rural-access, evaluation-management]}
|
||||
- {comment_id: CMS-2026-2377-27938, docket: CMS-2026-2377, split: tune, expected: [therapy-services, practice-expense], acceptable: [conversion-factor, coding-descriptors, regulatory-impact, work-rvu, rural-access]}
|
||||
- {comment_id: CMS-2026-2377-29215, docket: CMS-2026-2377, split: tune, expected: [modifier-25-same-day], acceptable: [beneficiary-cost-sharing, conversion-factor]}
|
||||
- {comment_id: CMS-2026-2377-29510, docket: CMS-2026-2377, split: heldout, expected: [conversion-factor, modifier-25-same-day, practice-expense], acceptable: [administrative-burden, rural-access, evaluation-management]}
|
||||
- {comment_id: CMS-2026-2377-30572, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [work-rvu, global-surgery, efficiency-adjustment, conversion-factor]}
|
||||
- {comment_id: CMS-2026-2377-32438, docket: CMS-2026-2377, split: tune, expected: [lactation-services], acceptable: [beneficiary-cost-sharing, work-rvu]}
|
||||
- {comment_id: CMS-2026-2377-32542, docket: CMS-2026-2377, split: tune, expected: [therapy-services], acceptable: [coding-descriptors, work-rvu, practice-expense]}
|
||||
- {comment_id: CMS-2026-2377-34620, docket: CMS-2026-2377, split: heldout, expected: [misvalued-codes], acceptable: [work-rvu, laboratory, coding-descriptors]}
|
||||
- {comment_id: CMS-2026-2377-35870, docket: CMS-2026-2377, split: heldout, expected: [modifier-25-same-day], acceptable: [conversion-factor, beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2026-2377-41579, docket: CMS-2026-2377, split: tune, expected: [conversion-factor, modifier-25-same-day, innovation-models], acceptable: [work-rvu, global-surgery, efficiency-adjustment, evaluation-management]}
|
||||
- {comment_id: CMS-2026-2377-42468, docket: CMS-2026-2377, split: tune, expected: [therapy-services], acceptable: [practice-expense, coding-descriptors, rural-access, regulatory-impact, administrative-burden, conversion-factor]}
|
||||
- {comment_id: CMS-2018-0076-0848, docket: CMS-2018-0076, split: tune, expected: [supervision, imaging-radiology], acceptable: [non-physician-practitioners]}
|
||||
- {comment_id: CMS-2018-0076-1405, docket: CMS-2018-0076, split: tune, expected: [supervision, imaging-radiology], acceptable: [non-physician-practitioners]}
|
||||
- {comment_id: CMS-2018-0076-2440, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [coding-descriptors, administrative-burden]}
|
||||
- {comment_id: CMS-2018-0076-2680, docket: CMS-2018-0076, split: tune, expected: [evaluation-management, modifier-25-same-day], acceptable: [coding-descriptors]}
|
||||
- {comment_id: CMS-2018-0076-2964, docket: CMS-2018-0076, split: tune, expected: [evaluation-management, modifier-25-same-day], acceptable: [coding-descriptors, administrative-burden]}
|
||||
- {comment_id: CMS-2018-0076-3018, docket: CMS-2018-0076, split: tune, expected: [modifier-25-same-day], acceptable: [beneficiary-cost-sharing, evaluation-management]}
|
||||
- {comment_id: CMS-2018-0076-4786, docket: CMS-2018-0076, split: tune, expected: [laboratory], acceptable: [data-requests-rfi]}
|
||||
- {comment_id: CMS-2018-0076-5907, docket: CMS-2018-0076, split: heldout, expected: [modifier-25-same-day, evaluation-management], acceptable: [administrative-burden, coding-descriptors]}
|
||||
- {comment_id: CMS-2018-0076-6395, docket: CMS-2018-0076, split: heldout, expected: [evaluation-management], acceptable: []}
|
||||
- {comment_id: CMS-2018-0076-7622, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [modifier-25-same-day, administrative-burden, quality-payment-program]}
|
||||
- {comment_id: CMS-2018-0076-8431, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [modifier-25-same-day, administrative-burden, quality-payment-program]}
|
||||
- {comment_id: CMS-2018-0076-9484, docket: CMS-2018-0076, split: heldout, expected: [practice-expense], acceptable: [site-of-service, work-rvu]}
|
||||
- {comment_id: CMS-2018-0076-9808, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [beneficiary-cost-sharing, administrative-burden]}
|
||||
- {comment_id: CMS-2018-0076-9852, docket: CMS-2018-0076, split: heldout, expected: [evaluation-management], acceptable: [beneficiary-cost-sharing, administrative-burden]}
|
||||
- {comment_id: CMS-2018-0076-10355, docket: CMS-2018-0076, split: tune, expected: [modifier-25-same-day, evaluation-management], acceptable: [administrative-burden, coding-descriptors]}
|
||||
- {comment_id: CMS-2018-0076-10926, docket: CMS-2018-0076, split: heldout, expected: [communication-technology-services], acceptable: [telehealth]}
|
||||
- {comment_id: CMS-2018-0076-11026, docket: CMS-2018-0076, split: tune, expected: [supervision], acceptable: [administrative-burden, evaluation-management, non-physician-practitioners]}
|
||||
- {comment_id: CMS-2018-0076-11395, docket: CMS-2018-0076, split: heldout, expected: [evaluation-management], acceptable: [conversion-factor]}
|
||||
- {comment_id: CMS-2018-0076-11627, docket: CMS-2018-0076, split: tune, expected: [evaluation-management, modifier-25-same-day], acceptable: [opioid-treatment]}
|
||||
- {comment_id: CMS-2018-0076-12012, docket: CMS-2018-0076, split: tune, expected: [supervision], acceptable: [administrative-burden, non-physician-practitioners, evaluation-management]}
|
||||
- {comment_id: CMS-2018-0076-13074, docket: CMS-2018-0076, split: tune, expected: [evaluation-management, quality-payment-program], acceptable: [administrative-burden, quality-measures]}
|
||||
- {comment_id: CMS-2018-0076-13338, docket: CMS-2018-0076, split: tune, expected: [evaluation-management], acceptable: [coding-descriptors]}
|
||||
- {comment_id: CMS-2018-0076-13645, docket: CMS-2018-0076, split: heldout, expected: [evaluation-management, modifier-25-same-day], acceptable: [administrative-burden]}
|
||||
- {comment_id: CMS-2018-0076-14524, docket: CMS-2018-0076, split: heldout, expected: [practice-expense, imaging-radiology], acceptable: [coverage-policy, administrative-burden]}
|
||||
- {comment_id: CMS-2018-0076-14790, docket: CMS-2018-0076, split: heldout, expected: [gpci-localities], acceptable: [rural-access]}
|
||||
- {comment_id: CMS-2019-0109-0163, docket: CMS-2019-0109, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service, coverage-policy]}
|
||||
- {comment_id: CMS-2019-0109-0299, docket: CMS-2019-0109, split: tune, expected: [], acceptable: [remote-monitoring], off_vocab: "medical device testimonial (wearable post-surgical monitor)"}
|
||||
- {comment_id: CMS-2019-0109-0430, docket: CMS-2019-0109, split: tune, expected: [price-transparency, site-of-service], acceptable: [beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-0444, docket: CMS-2019-0109, split: heldout, expected: [price-transparency, site-of-service], acceptable: [beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-0502, docket: CMS-2019-0109, split: tune, expected: [opps-asc-payment], acceptable: [work-rvu]}
|
||||
- {comment_id: CMS-2019-0109-0551, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [drugs-biologicals]}
|
||||
- {comment_id: CMS-2019-0109-0651, docket: CMS-2019-0109, split: heldout, expected: [price-transparency, site-of-service], acceptable: [beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-0694, docket: CMS-2019-0109, split: heldout, expected: [drugs-biologicals], acceptable: []}
|
||||
- {comment_id: CMS-2019-0109-0867, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: []}
|
||||
- {comment_id: CMS-2019-0109-0944, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [interoperability-health-it, beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-0964, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: []}
|
||||
- {comment_id: CMS-2019-0109-1021, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: []}
|
||||
- {comment_id: CMS-2019-0109-1120, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [drugs-biologicals]}
|
||||
- {comment_id: CMS-2019-0109-1283, docket: CMS-2019-0109, split: heldout, expected: [drugs-biologicals], acceptable: [coding-descriptors]}
|
||||
- {comment_id: CMS-2019-0109-1359, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [site-of-service, beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-1730, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-1796, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [drugs-biologicals, beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-1910, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-1924, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-2340, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [drugs-biologicals]}
|
||||
- {comment_id: CMS-2019-0109-2494, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-2715, docket: CMS-2019-0109, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service, coverage-policy]}
|
||||
- {comment_id: CMS-2019-0109-3046, docket: CMS-2019-0109, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service, coverage-policy, innovation-models]}
|
||||
- {comment_id: CMS-2019-0109-3237, docket: CMS-2019-0109, split: heldout, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2019-0109-3701, docket: CMS-2019-0109, split: tune, expected: [price-transparency], acceptable: [beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2020-0088-1134, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe]}
|
||||
- {comment_id: CMS-2020-0088-1246, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe]}
|
||||
- {comment_id: CMS-2020-0088-1340, docket: CMS-2020-0088, split: tune, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe, administrative-burden]}
|
||||
- {comment_id: CMS-2020-0088-1386, docket: CMS-2020-0088, split: tune, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe]}
|
||||
- {comment_id: CMS-2020-0088-1961, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-3797, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor, telehealth], acceptable: [communication-technology-services, covid-phe, supervision, rural-access, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-5375, docket: CMS-2020-0088, split: tune, expected: [conversion-factor, supervision], acceptable: [non-physician-practitioners, anesthesia, rural-access, regulatory-impact, covid-phe]}
|
||||
- {comment_id: CMS-2020-0088-7401, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor, telehealth], acceptable: [communication-technology-services, covid-phe, supervision, rural-access, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-8032, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [communication-technology-services, telehealth, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-8116, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [communication-technology-services, telehealth, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-10383, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [communication-technology-services, telehealth, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-12653, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor, telehealth], acceptable: [communication-technology-services, remote-monitoring, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-14620, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-16755, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-17272, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-17866, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-18655, docket: CMS-2020-0088, split: heldout, expected: [telehealth], acceptable: [rural-access, covid-phe]}
|
||||
- {comment_id: CMS-2020-0088-20121, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-22592, docket: CMS-2020-0088, split: tune, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-23016, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor, telehealth], acceptable: [therapy-services, covid-phe, work-rvu, evaluation-management]}
|
||||
- {comment_id: CMS-2020-0088-23505, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-23605, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor, telehealth], acceptable: [covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2020-0088-26793, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor], acceptable: [covid-phe]}
|
||||
- {comment_id: CMS-2020-0088-28880, docket: CMS-2020-0088, split: heldout, expected: [conversion-factor], acceptable: [work-rvu, non-physician-practitioners]}
|
||||
- {comment_id: CMS-2020-0088-29770, docket: CMS-2020-0088, split: heldout, expected: [therapy-services, conversion-factor], acceptable: [evaluation-management, modifier-25-same-day, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2022-0113-1724, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management]}
|
||||
- {comment_id: CMS-2022-0113-4292, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management]}
|
||||
- {comment_id: CMS-2022-0113-5197, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-6188, docket: CMS-2022-0113, split: tune, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]}
|
||||
- {comment_id: CMS-2022-0113-7459, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-8412, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management]}
|
||||
- {comment_id: CMS-2022-0113-10435, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-11359, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-11411, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management]}
|
||||
- {comment_id: CMS-2022-0113-12454, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-12647, docket: CMS-2022-0113, split: heldout, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]}
|
||||
- {comment_id: CMS-2022-0113-13378, docket: CMS-2022-0113, split: tune, expected: [work-rvu], acceptable: [practice-expense, coding-descriptors]}
|
||||
- {comment_id: CMS-2022-0113-13826, docket: CMS-2022-0113, split: heldout, expected: [non-physician-practitioners, coding-descriptors], acceptable: [coverage-policy, beneficiary-cost-sharing, administrative-burden]}
|
||||
- {comment_id: CMS-2022-0113-14587, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-15010, docket: CMS-2022-0113, split: tune, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]}
|
||||
- {comment_id: CMS-2022-0113-16694, docket: CMS-2022-0113, split: heldout, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]}
|
||||
- {comment_id: CMS-2022-0113-17371, docket: CMS-2022-0113, split: tune, expected: [therapy-services, conversion-factor, remote-monitoring], acceptable: [supervision, administrative-burden, regulatory-impact]}
|
||||
- {comment_id: CMS-2022-0113-17843, docket: CMS-2022-0113, split: heldout, expected: [work-rvu], acceptable: [misvalued-codes]}
|
||||
- {comment_id: CMS-2022-0113-18650, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-19609, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-20145, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-22707, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-23123, docket: CMS-2022-0113, split: heldout, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2022-0113-23609, docket: CMS-2022-0113, split: tune, expected: [dental-services], acceptable: [coverage-policy, non-physician-practitioners, rhc-fqhc, telehealth, opioid-treatment, data-requests-rfi, rural-access]}
|
||||
- {comment_id: CMS-2022-0113-23709, docket: CMS-2022-0113, split: tune, expected: [behavioral-health], acceptable: [telehealth, supervision, coding-descriptors, caregiver-training, non-physician-practitioners, rural-access, data-requests-rfi, care-management, practice-expense]}
|
||||
- {comment_id: CMS-2023-0057-0182, docket: CMS-2023-0057, split: tune, expected: [hospital-conditions-of-participation], acceptable: [covid-phe]}
|
||||
- {comment_id: CMS-2023-0057-0261, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-0356, docket: CMS-2023-0057, split: heldout, expected: [hospital-conditions-of-participation], acceptable: [covid-phe, quality-measures, rural-access]}
|
||||
- {comment_id: CMS-2023-0057-0371, docket: CMS-2023-0057, split: heldout, expected: [interoperability-health-it], acceptable: [administrative-burden, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-0456, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, rural-access, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-0492, docket: CMS-2023-0057, split: tune, expected: [hospital-conditions-of-participation], acceptable: [covid-phe]}
|
||||
- {comment_id: CMS-2023-0057-0587, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, rural-access, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-0814, docket: CMS-2023-0057, split: heldout, expected: [ipps-drg-payment], acceptable: []}
|
||||
- {comment_id: CMS-2023-0057-0852, docket: CMS-2023-0057, split: heldout, expected: [ipps-drg-payment], acceptable: []}
|
||||
- {comment_id: CMS-2023-0057-0895, docket: CMS-2023-0057, split: tune, expected: [hospital-conditions-of-participation], acceptable: [covid-phe, quality-measures, rural-access]}
|
||||
- {comment_id: CMS-2023-0057-0952, docket: CMS-2023-0057, split: tune, expected: [hospital-conditions-of-participation], acceptable: [covid-phe]}
|
||||
- {comment_id: CMS-2023-0057-1039, docket: CMS-2023-0057, split: tune, expected: [ipps-drg-payment], acceptable: []}
|
||||
- {comment_id: CMS-2023-0057-1080, docket: CMS-2023-0057, split: heldout, expected: [ipps-drg-payment], acceptable: []}
|
||||
- {comment_id: CMS-2023-0057-1306, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-1350, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-1408, docket: CMS-2023-0057, split: tune, expected: [ipps-drg-payment], acceptable: [behavioral-health, coverage-policy]}
|
||||
- {comment_id: CMS-2023-0057-1420, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-1860, docket: CMS-2023-0057, split: heldout, expected: [hospital-wage-index], acceptable: [gpci-localities]}
|
||||
- {comment_id: CMS-2023-0057-2121, docket: CMS-2023-0057, split: tune, expected: [hospital-wage-index], acceptable: [gpci-localities]}
|
||||
- {comment_id: CMS-2023-0057-2460, docket: CMS-2023-0057, split: tune, expected: [hospital-wage-index], acceptable: [gpci-localities]}
|
||||
- {comment_id: CMS-2023-0057-2619, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-2874, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-2959, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-2968, docket: CMS-2023-0057, split: tune, expected: [hospital-quality-programs], acceptable: [covid-phe, quality-measures]}
|
||||
- {comment_id: CMS-2023-0057-3256, docket: CMS-2023-0057, split: heldout, expected: [hospital-quality-programs], acceptable: [covid-phe, rural-access, quality-measures]}
|
||||
- {comment_id: CMS-2024-0256-0359, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]}
|
||||
- {comment_id: CMS-2024-0256-0614, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]}
|
||||
- {comment_id: CMS-2024-0256-0918, docket: CMS-2024-0256, split: heldout, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]}
|
||||
- {comment_id: CMS-2024-0256-0972, docket: CMS-2024-0256, split: heldout, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]}
|
||||
- {comment_id: CMS-2024-0256-1173, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]}
|
||||
- {comment_id: CMS-2024-0256-1296, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]}
|
||||
- {comment_id: CMS-2024-0256-1504, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [supervision, coverage-policy, practice-expense, dme-supplies]}
|
||||
- {comment_id: CMS-2024-0256-2071, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-2214, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-2250, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-2377, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-2594, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-2852, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-2992, docket: CMS-2024-0256, split: tune, expected: [dme-supplies], acceptable: [coverage-policy, dental-services, administrative-burden]}
|
||||
- {comment_id: CMS-2024-0256-3674, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-3788, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-3919, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-3947, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-4590, docket: CMS-2024-0256, split: heldout, expected: [behavioral-health], acceptable: [telehealth, opioid-treatment, caregiver-training, supervision, coding-descriptors, rural-access, covid-phe, work-rvu]}
|
||||
- {comment_id: CMS-2024-0256-4823, docket: CMS-2024-0256, split: heldout, expected: [evaluation-management], acceptable: [coding-descriptors, work-rvu, rural-access]}
|
||||
- {comment_id: CMS-2024-0256-5075, docket: CMS-2024-0256, split: tune, expected: [evaluation-management], acceptable: [coding-descriptors, work-rvu, rural-access]}
|
||||
- {comment_id: CMS-2024-0256-5396, docket: CMS-2024-0256, split: tune, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-5638, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-6253, docket: CMS-2024-0256, split: heldout, expected: [conversion-factor, therapy-services], acceptable: [supervision, non-physician-practitioners, rural-access, administrative-burden, covid-phe, regulatory-impact]}
|
||||
- {comment_id: CMS-2024-0256-6651, docket: CMS-2024-0256, split: tune, expected: [behavioral-health], acceptable: [coding-descriptors, beneficiary-cost-sharing, supervision]}
|
||||
- {comment_id: CMS-2025-0306-0187, docket: CMS-2025-0306, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service]}
|
||||
- {comment_id: CMS-2025-0306-0237, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]}
|
||||
- {comment_id: CMS-2025-0306-0344, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]}
|
||||
- {comment_id: CMS-2025-0306-0351, docket: CMS-2025-0306, split: tune, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]}
|
||||
- {comment_id: CMS-2025-0306-0375, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]}
|
||||
- {comment_id: CMS-2025-0306-0541, docket: CMS-2025-0306, split: heldout, expected: [opps-asc-payment], acceptable: [work-rvu, coding-descriptors]}
|
||||
- {comment_id: CMS-2025-0306-0570, docket: CMS-2025-0306, split: tune, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]}
|
||||
- {comment_id: CMS-2025-0306-0694, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]}
|
||||
- {comment_id: CMS-2025-0306-0797, docket: CMS-2025-0306, split: heldout, expected: [non-opioid-pain], acceptable: [opioid-treatment, coverage-policy, beneficiary-cost-sharing, drugs-biologicals]}
|
||||
- {comment_id: CMS-2025-0306-0922, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]}
|
||||
- {comment_id: CMS-2025-0306-0980, docket: CMS-2025-0306, split: heldout, expected: [efficiency-adjustment, practice-expense], acceptable: [site-of-service, work-rvu]}
|
||||
- {comment_id: CMS-2025-0306-0994, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]}
|
||||
- {comment_id: CMS-2025-0306-1219, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]}
|
||||
- {comment_id: CMS-2025-0306-1306, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]}
|
||||
- {comment_id: CMS-2025-0306-1323, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]}
|
||||
- {comment_id: CMS-2025-0306-1413, docket: CMS-2025-0306, split: tune, expected: [opps-asc-payment], acceptable: [coverage-policy, site-of-service]}
|
||||
- {comment_id: CMS-2025-0306-1548, docket: CMS-2025-0306, split: tune, expected: [opps-asc-payment], acceptable: [site-of-service, conversion-factor]}
|
||||
- {comment_id: CMS-2025-0306-1559, docket: CMS-2025-0306, split: tune, expected: [opps-asc-payment], acceptable: [behavioral-health, work-rvu]}
|
||||
- {comment_id: CMS-2025-0306-1737, docket: CMS-2025-0306, split: tune, expected: [site-of-service], acceptable: [drugs-biologicals, beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2025-0306-1817, docket: CMS-2025-0306, split: tune, expected: [site-of-service], acceptable: [drugs-biologicals, beneficiary-cost-sharing]}
|
||||
- {comment_id: CMS-2025-0306-2427, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]}
|
||||
- {comment_id: CMS-2025-0306-2581, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [community-health-social-needs, rural-access, administrative-burden, quality-measures]}
|
||||
- {comment_id: CMS-2025-0306-2712, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]}
|
||||
- {comment_id: CMS-2025-0306-2728, docket: CMS-2025-0306, split: tune, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]}
|
||||
- {comment_id: CMS-2025-0306-2742, docket: CMS-2025-0306, split: heldout, expected: [hospital-quality-programs], acceptable: [imaging-radiology, quality-measures]}
|
||||
@@ -22,13 +22,28 @@ from llm.vocab import load as load_vocab
|
||||
GOLDEN = Path(__file__).with_name("golden_themes.yaml")
|
||||
|
||||
|
||||
class TestGoldenV3File:
|
||||
def test_current_vocab_two_halves_and_known_slugs(self):
|
||||
from pathlib import Path
|
||||
|
||||
g = load_golden(Path(__file__).parent / "golden_themes_v3.yaml")
|
||||
v = load_vocab()
|
||||
assert g.vocab_version == v.version
|
||||
assert len(g.entries) == 200
|
||||
assert {e.split for e in g.entries} == {"tune", "heldout"}
|
||||
for e in g.entries:
|
||||
assert e.expected or e.off_vocab, e.comment_id
|
||||
for s in (*e.expected, *e.acceptable):
|
||||
assert s in v, f"{e.comment_id}: unknown slug {s}"
|
||||
|
||||
|
||||
class TestGoldenFile:
|
||||
def test_loads_and_only_uses_known_slugs(self):
|
||||
g = load_golden(GOLDEN)
|
||||
v = load_vocab()
|
||||
assert (
|
||||
isinstance(g, Golden)
|
||||
and g.vocab_version == v.version
|
||||
and (g.vocab_version == v.version or g.vocab_version in v.compatible_with)
|
||||
and g.docket == "CMS-2026-2377"
|
||||
)
|
||||
assert len(g.entries) >= 30
|
||||
|
||||
@@ -504,3 +504,43 @@ class TestRetune:
|
||||
stats = self._run(s, keys, judge, top=3, max_tags=3, min_confidence=0.0)
|
||||
assert stats["retuned"] == 1
|
||||
assert not set(calls) & set(first) and len(calls) == 2
|
||||
|
||||
|
||||
def test_compatible_vocab_reuses_old_verdicts_and_judges_only_new_themes(tmp_path):
|
||||
"""v3 only adds themes (compatible_with: [2]): stored v2 verdicts stand;
|
||||
a theme that is new to the vocab is judged the first time it is shortlisted."""
|
||||
from dataclasses import replace
|
||||
|
||||
s, keys = _store(tmp_path)
|
||||
calls = []
|
||||
|
||||
def judge(theme, evidence):
|
||||
calls.append(theme.slug)
|
||||
return True
|
||||
|
||||
kw = dict(
|
||||
docket="CMS-2026-2377",
|
||||
load_chunks=TestRun()._chunks(keys),
|
||||
card_vecs=CARDS,
|
||||
judge=judge,
|
||||
model="m",
|
||||
top=2,
|
||||
max_tags=3,
|
||||
min_confidence=0.0,
|
||||
)
|
||||
v2 = replace(VOCAB, version=2)
|
||||
run(s, vocab=v2, **kw)
|
||||
first = set(calls)
|
||||
calls.clear()
|
||||
v3 = replace(VOCAB, version=3, compatible_with=(2,))
|
||||
stats = run(s, vocab=v3, **kw)
|
||||
assert stats["retuned"] == 1 and stats["tagged"] == 0
|
||||
assert not set(calls) & first
|
||||
k = keys["CMS-2026-2377-1"]
|
||||
extra = json.loads(
|
||||
s._con().execute("SELECT extra_json FROM items WHERE key=?", (k,)).fetchone()[0] # noqa: SLF001
|
||||
)
|
||||
assert extra["llm_tags"]["version"] == 3
|
||||
calls.clear()
|
||||
v4 = replace(VOCAB, version=4) # not compatible: full re-judge
|
||||
assert run(s, vocab=v4, **kw)["tagged"] == 1 and calls
|
||||
|
||||
@@ -112,3 +112,17 @@ class TestTagHelpers:
|
||||
"docket:x",
|
||||
]
|
||||
assert is_llm_tag("llm:x") and not is_llm_tag("llmx:y")
|
||||
|
||||
|
||||
def test_compatible_with_lists_versions_whose_verdicts_still_hold():
|
||||
from llm.vocab import parse
|
||||
|
||||
v = parse(
|
||||
"version: 3\ncompatible_with: [2]\nthemes:\n"
|
||||
" - {slug: telehealth, label: T, definition: d}\n"
|
||||
)
|
||||
assert v.compatible_with == (2,)
|
||||
assert (
|
||||
parse("version: 2\nthemes:\n - {slug: a, definition: d}\n").compatible_with
|
||||
== ()
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user