100 lines
2.9 KiB
YAML
100 lines
2.9 KiB
YAML
# Golden longitudinal evaluation set (P49 Task 7, refs #692).
|
|
#
|
|
# Anchors verified against the corpus in #692. Run with:
|
|
# uv run python dev/scripts/llm_golden.py run --url <chat-url> \
|
|
# --set tests/llm/golden_lineage.yaml --report report.json
|
|
#
|
|
# Schema per entry:
|
|
# id unique string
|
|
# question the question sent to POST /chat
|
|
# mode "auto" (default), "timeline" or "recent"
|
|
# expect_anchors [{item_key, p_id}] p_id may be [lo, hi] for a range;
|
|
# each anchor must appear among the streamed `sources`
|
|
# expect_labels [regex] each must be matched somewhere in the
|
|
# concatenated `token` text (the cited answer prose)
|
|
# forbid [{pattern, unless_label}] `pattern` may appear only
|
|
# in a sentence that also cites a bracketed label
|
|
# matching `unless_label`
|
|
# expect_events [{code, kind, year}] kind may be "a|b" alternatives;
|
|
# must appear in the `lineage` event payload
|
|
# expect_dockets [docket_id] must appear among comment-kind sources
|
|
# min_eras distinct rule years among sources (rule/corpus: the
|
|
# source's date year; comment: its docket id's year)
|
|
|
|
- id: ccm-history
|
|
question: What is the history of CCM coding and payment?
|
|
mode: timeline
|
|
expect_anchors:
|
|
- item_key: DE2VH9PD
|
|
p_id: [1249, 1251]
|
|
- item_key: YBM4IZUS
|
|
p_id: 1578
|
|
expect_events:
|
|
- code: "99490"
|
|
kind: created
|
|
year: 2015
|
|
- code: G2058
|
|
kind: replaced_by
|
|
year: 2021
|
|
min_eras: 4
|
|
|
|
- id: g2058-replacement
|
|
question: What replaced G2058, and why?
|
|
mode: timeline
|
|
expect_anchors:
|
|
- item_key: YBM4IZUS
|
|
p_id: 1578
|
|
- item_key: YBM4IZUS
|
|
p_id: 2369
|
|
expect_events:
|
|
- code: G2058
|
|
kind: replaced_by
|
|
year: 2021
|
|
|
|
- id: apcm-vs-ccm-elements
|
|
question: How do APCM's elements differ from CCM's?
|
|
expect_anchors:
|
|
- item_key: JJ6AM5HJ
|
|
p_id: [1164, 1185]
|
|
- item_key: DE2VH9PD
|
|
p_id: [1245, 1247]
|
|
|
|
- id: audio-only-em-99441
|
|
question: When were audio-only E/M codes payable, and why did 99441-99443 end?
|
|
mode: timeline
|
|
expect_anchors:
|
|
- item_key: XFGGRBDH
|
|
p_id: 489
|
|
expect_events:
|
|
- code: "99441"
|
|
kind: deleted|disappeared|cpt_deleted
|
|
year: 2025
|
|
|
|
- id: g2211-commenters-2023-vs-2025
|
|
question: What did commenters say about G2211 in 2023 versus 2025?
|
|
mode: timeline
|
|
expect_dockets:
|
|
- CMS-2023-0121
|
|
- CMS-2025-0304
|
|
|
|
- id: 99490-telehealth-steps
|
|
question: Does 99490 pass telehealth Steps 1 through 3?
|
|
expect_anchors:
|
|
- item_key: 2KVJ2HKX
|
|
p_id: 394
|
|
- item_key: 2KVJ2HKX
|
|
p_id: 396
|
|
- item_key: 2KVJ2HKX
|
|
p_id: 398
|
|
|
|
- id: g2064-g2065-to-99424-99426-rate
|
|
question: How did G2064/G2065 becoming 99424/99426 change the RHC/FQHC G0511 rate?
|
|
mode: timeline
|
|
expect_anchors:
|
|
- item_key: JE7KYBW3
|
|
p_id: 1100
|
|
- item_key: JE7KYBW3
|
|
p_id: 1111
|
|
- item_key: MZ24MX5S
|
|
p_id: 1305
|