diff --git a/src/cli/market.py b/src/cli/market.py index a5de17c..bf394a6 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -7,6 +7,7 @@ uv run stack market hud --year 2025 --quarter 2 --congress 119 --write uv run stack market footprint --entities --write uv run stack market footprint --family HOME-VISITS --write uv run stack market district-view --family HOME-VISITS --top 10 +uv run stack market events --write # releases, meetings, tenures uv run stack market gaps # sources that could not be read Every command fetches first, then opens one short DuckDB batch that only @@ -28,6 +29,7 @@ import typer from market import congress as congress_api from market import districts as district_api from market import edgar, http, tiingo +from market import events as events_api from market import fec as fec_api from market import footprint as footprint_api from market import lda as lda_api @@ -69,6 +71,7 @@ from market.tables import ( write_county_districts, write_entities, write_entity_sites, + write_events, write_fec_committees, write_fec_contributions, write_fec_totals, @@ -1977,6 +1980,118 @@ def district_view( ) +def _echo_events(events: list, skipped: int, tenures: list) -> None: + """The live report: rules, copies, meetings, and every tenure.""" + from collections import Counter + + typer.echo(f"skipped rule: {skipped}") + typer.echo(f"{len(events)} events") + program = [ + event + for event in events + if event.scope == "program" and event.kind.startswith("rule-") + ] + for kind in ("rule-proposed", "rule-final"): + rows = [event for event in program if event.kind == kind] + filed = sum(1 for event in rows if event.filed_at) + typer.echo( + f"{kind}: {len(rows)} " + f"(public inspection {filed}, publication {len(rows) - filed})" + ) + for scope in ("family", "specialty"): + counts = Counter(event.target for event in events if event.scope == scope) + for key, n in sorted(counts.items()): + typer.echo(f"{scope} {key}: {n}") + meetings = Counter(event.kind for event in events if event.kind.endswith("meeting")) + for kind, n in sorted(meetings.items()): + typer.echo(f"{kind}: {n}") + for tenure in tenures: + flags = " ".join( + name + for name, on in ( + ("left-censored", tenure.left_censored), + ("right-censored", tenure.right_censored), + ) + if on + ) + holders = ", ".join(tenure.holders) + typer.echo( + f"tenure {tenure.body} {tenure.key} " + f"{tenure.first_year}-{tenure.last_year} {holders} {flags}".rstrip() + ) + + +@app.command() +def events( + refresh: bool = typer.Option( + False, "--refresh", help="Re-fetch public-inspection documents." + ), + ama_url: str = typer.Option( + "", "--ama-url", help="AMA page listing RUC or CPT Panel meeting dates." + ), + body: str = typer.Option( + "", "--body", help="RUC or CPT_PANEL, required with --ama-url." + ), + write: bool = typer.Option(False, "--write", help="Store market.event."), +) -> None: + """Rule releases at public inspection, RUC and CPT Panel meetings, and + RUC seat tenures. Tenures are printed, not stored.""" + if ama_url and body not in ("RUC", "CPT_PANEL"): + typer.echo("pass --body RUC or CPT_PANEL with --ama-url") + raise typer.Exit(2) + store = _store() + con = _read() + try: + found, skipped, held = events_api.build( + store, con, refresh=refresh, cache=http.hoard_root() + ) + finally: + con.close() + ama_note = "" + ama_ok = False + if ama_url: + from pfs.seats import is_refusal + + try: + page = events_api.fetch_ama(ama_url) + except http.Refused as exc: + ama_note = str(exc) + else: + if is_refusal(page): + ama_note = "AMA page is a refusal" + else: + found.extend(events_api.ama_schedule(page, body)) + if events_api.both_calendars(page): + other = "CPT_PANEL" if body == "RUC" else "RUC" + found.extend(events_api.ama_schedule(page, other)) + ama_ok = True + _echo_events(found, skipped, held) + if ama_note: + typer.echo(f"gap: ama:events: {ama_note}") + if not write: + typer.echo("dry run: pass --write") + return + from cms.ingest_log import log_ingest + + with _batch() as batch: + ensure_tables(batch) + n = write_events(batch, found) + clear_gap(batch, "fr:events") + if ama_ok: + clear_gap(batch, "ama:events") + elif ama_note: + write_gap(batch, "ama:events", ama_note) + log_ingest( + batch, + module="market.events", + table_name="market.event", + rows=n, + source_file="fr:public-inspection", + ) + _publish() + typer.echo(f"stored: {n} events") + + @app.command() def gaps() -> None: """Sources that could not be read (market.source_gap).""" diff --git a/src/market/events.py b/src/market/events.py new file mode 100644 index 0000000..29afe76 --- /dev/null +++ b/src/market/events.py @@ -0,0 +1,603 @@ +"""Public-inspection releases, RUC and CPT Panel meetings, and seat tenures. + +Rule events are dated from the Federal Register public-inspection filing +when that record exists (16:00 America/New_York is after the close). A +missing record, the usual case before 2013, keeps the publication date and +says so. Family and specialty rows are copies of those program events. +Meeting mentions in the rule text are month precision. AMA schedule pages +supply a day. Tenures are computed for the report and are not stored. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, replace +from datetime import date, datetime, time +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +from market import http +from market.seed import SEAT_LABEL_KEYS, seat_key +from market.tables import MarketEvent +from pfs.codetables import ( + is_missing_table_error, + read_all_events, + read_exposures, + read_families, + read_seats, + read_specialty_impacts, +) +from pfs.seats import seats_by_specialty + +FR_PI = "https://www.federalregister.gov/api/v1/public-inspection-documents/{doc}.json" +_ET = ZoneInfo("America/New_York") +_CLOSE = time(16, 0) +_MONTHS = ( + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +) +_MONTH_NUM = {name: n for n, name in enumerate(_MONTHS, start=1)} +_MONTH_RE = "|".join(_MONTHS) +_MEETING_RE = re.compile( + rf"(?:(?P{_MONTH_RE})\s+(?P\d{{4}})\s+" + rf"(?PCPT Editorial Panel|CPT Panel|RUC)\s+meeting)" + rf"|(?:(?PCPT Editorial Panel|Panel|RUC)\s+" + rf"(?:at|during|in)\s+(?:its|their)\s+" + rf"(?P{_MONTH_RE})\s+(?P\d{{4}})\s+meeting)" +) +_AMA_DATE = re.compile( + r"\b(?P[A-Za-z]+\.?)\s+(?P\d{1,2})" + r"\s*[\u2013-]\s*(?:[A-Za-z]+\.?\s+)?\d{1,2},\s*(?P\d{4})\b" +) +_AMA_SECTION = re.compile( + r"CPT(?:®|\(R\))? Editorial Panel meetings" + r"|RVS Update Committee meetings" + r"|(? date: + text = value.isoformat() if isinstance(value, date) else str(value) + return date.fromisoformat(text[:10]) + + +def _gone(exc: Exception) -> bool: + if is_missing_table_error(exc): + return True + return "no such table" in str(exc).lower() + + +def _or_empty(fn, default): + try: + return fn() + except Exception as exc: + if _gone(exc): + return default + raise + + +def _chunks(items: Sequence[Any], n: int) -> list[Sequence[Any]]: + return [items[i : i + n] for i in range(0, len(items), n)] + + +def release(filed_at: str | None, published: date | str) -> tuple[date, bool, str]: + """``(ET date, ET time ≥ 16:00, note)`` for one public-inspection filing.""" + if not filed_at: + return _as_date(published), False, _PUBLICATION_NOTE + local = datetime.fromisoformat(filed_at).astimezone(_ET) + return local.date(), local.time() >= _CLOSE, "" + + +def rule_year(title: str) -> int | None: + """The calendar year named ``CY 2027`` or ``CY2027`` in a rule title.""" + match = _CY.search(title or "") + return int(match.group(1)) if match else None + + +def rule_events( + rules: Sequence[Mapping[str, Any]], + docs: Mapping[str, str], + pi: Mapping[str, str | None], +) -> tuple[list[MarketEvent], int]: + """Program events (target ``PFS``) and the count of skipped ``rule`` kinds.""" + events: list[MarketEvent] = [] + skipped = 0 + for rule in rules: + kind = _RULE_KIND.get(rule["kind"]) + if kind is None: + skipped += int(rule["kind"] == "rule") + continue + doc = docs.get(rule["item_key"]) or "" + filed = pi.get(doc) or "" + day, after, note = release(filed or None, rule["date_published"]) + events.append( + MarketEvent( + f"{kind}:{doc}", + kind, + day, + after, + "day", + "program", + "PFS", + rule["item_key"], + 0, + doc, + filed, + note, + ) + ) + return events, skipped + + +def family_rule_events( + events: Sequence[MarketEvent], + *, + exposure_item_keys: set[str], + lineage_item_keys: set[str], + family: str, +) -> list[MarketEvent]: + """Copies of rule events whose item anchors this family's exposure or lineage.""" + keys = set(exposure_item_keys) | set(lineage_item_keys) + return [ + replace(event, scope="family", target=family) + for event in events + if event.kind in _RULE_KIND.values() and event.item_key in keys + ] + + +def specialty_rule_events( + events: Sequence[MarketEvent], + impacts: Sequence[Any], + titles: Mapping[str, str], + specialty: str, + *, + min_abs: float = 0.01, +) -> list[MarketEvent]: + """Copies where CMS's TOTAL impact for the rule year clears ``min_abs``.""" + from pfs.commenters import specialty_key + + want = specialty_key(specialty) + out: list[MarketEvent] = [] + for event in events: + if event.kind not in _RULE_KIND.values(): + continue + year = rule_year(titles.get(event.item_key, "")) + if year is None: + continue + vintage = event.kind.removeprefix("rule-") + match = next( + ( + row + for row in impacts + if row.rule_year == year + and row.vintage == vintage + and row.setting == "TOTAL" + and specialty_key(row.specialty) == want + and row.impact_combined is not None + and abs(row.impact_combined) >= min_abs + ), + None, + ) + if match is None: + continue + note = f"CMS estimated impact {match.impact_combined:+.0%}" + out.append(replace(event, scope="specialty", target=want, note=note)) + return out + + +def meeting_mentions( + paragraphs: Iterable[tuple[str, int, str]], +) -> list[MarketEvent]: + """Month-precision program events. The lowest ``(item_key, p_id)`` wins.""" + chosen: dict[tuple[str, int, int], MarketEvent] = {} + for item_key, p_id, text in paragraphs: + for match in _MEETING_RE.finditer(text or ""): + month_name = match.group("m1") or match.group("m2") + year = int(match.group("y1") or match.group("y2")) + label = match.group("body1") or match.group("body2") + month = _MONTH_NUM[month_name] + kind = "panel-meeting" if "Panel" in label else "ruc-meeting" + event = MarketEvent( + f"{kind}:{year:04d}-{month:02d}", + kind, + date(year, month, 1), + False, + "month", + "program", + "PFS", + item_key, + int(p_id), + "", + "", + "", + ) + key = (kind, year, month) + prev = chosen.get(key) + if prev is None or (item_key, int(p_id)) < (prev.item_key, prev.p_id): + chosen[key] = event + return [chosen[key] for key in sorted(chosen)] + + +def _visible(text: str) -> str: + if "<" not in text: + return text + from pfs.seats import html_text + + return html_text(text.encode()) + + +def _month_number(token: str) -> int | None: + return _MONTH_LOOKUP.get(token.rstrip(".").lower()) + + +def _section(text: str, body: str) -> str: + """The body's part of a combined calendar; the whole page when it has no headings.""" + marks = list(_AMA_SECTION.finditer(text)) + if not marks: + return text + want_panel = body == "CPT_PANEL" + chunks: list[str] = [] + for i, mark in enumerate(marks): + if ("Panel" in mark.group(0)) != want_panel: + continue + end = marks[i + 1].start() if i + 1 < len(marks) else len(text) + chunks.append(text[mark.end() : end]) + return "\n".join(chunks) + + +def both_calendars(text: str) -> bool: + """True when *text* lists the CPT Panel and the RUC on one page.""" + heads = [match.group(0) for match in _AMA_SECTION.finditer(_visible(text))] + return any("Panel" in head for head in heads) and any( + "Panel" not in head for head in heads + ) + + +def ama_schedule(text: str, body: str) -> list[MarketEvent]: + """Day-precision meetings from ``" D–D, YYYY"`` ranges. + + Full and abbreviated months, and a range that crosses a month + (``April 29-May 1, 2027``), use the start day. A page with both + calendars contributes only that body's section. + """ + kind = _AMA_KIND.get(body) + if kind is None: + return [] + visible = _section(_visible(text), body) + found: dict[str, MarketEvent] = {} + for match in _AMA_DATE.finditer(visible): + month = _month_number(match.group("mon")) + if month is None: + continue + event_date = date(int(match.group("year")), month, int(match.group("day"))) + event_id = f"{kind}:{event_date.isoformat()}" + found.setdefault( + event_id, + MarketEvent( + event_id, + kind, + event_date, + False, + "day", + "program", + "PFS", + "", + 0, + "", + "", + "", + ), + ) + return [found[key] for key in sorted(found)] + + +def _year_keys(rows: Sequence[Any], year: int) -> dict[str, tuple[set[str], set[str]]]: + counted = seats_by_specialty(rows, year) + active = {key for key, counts in counted.items() if counts[0] + counts[1] > 0} + bags: dict[str, tuple[set[str], set[str]]] = {} + for row in rows: + if row.year != year: + continue + key = seat_key(row) + if key is None or (key not in active and key not in SEAT_LABEL_KEYS): + continue + holders, types = bags.setdefault(key, (set(), set())) + if row.organization: + holders.add(row.organization) + if row.seat_type: + types.add(row.seat_type) + return bags + + +def _runs(years: Iterable[int]) -> list[tuple[int, int]]: + ordered = sorted(years) + start = prev = ordered[0] + out: list[tuple[int, int]] = [] + for year in ordered[1:]: + if year == prev + 1: + prev = year + continue + out.append((start, prev)) + start = prev = year + out.append((start, prev)) + return out + + +def tenures(seats: Sequence[Any], *, body: str = "RUC") -> list[Tenure]: + """Consecutive roster years per held specialty, censored at the observed ends.""" + rows = [row for row in seats if row.body == body] + if not rows: + return [] + span = sorted({row.year for row in rows}) + held: dict[str, dict[int, tuple[set[str], set[str]]]] = {} + for year in span: + for key, bags in _year_keys(rows, year).items(): + held.setdefault(key, {})[year] = bags + first, last = span[0], span[-1] + out: list[Tenure] = [] + for key in sorted(held): + by_year = held[key] + for start, end in _runs(by_year): + holders: set[str] = set() + types: set[str] = set() + for year in range(start, end + 1): + year_holders, year_types = by_year[year] + holders.update(year_holders) + types.update(year_types) + out.append( + Tenure( + body, + key, + tuple(sorted(holders)), + tuple(sorted(types)), + start, + end, + start == first, + end == last, + ) + ) + return out + + +def day0(event: MarketEvent, calendar: Sequence[date]) -> date | None: + """First trading day on or after the event; the next one when filed after the close. + + Month precision has no day-zero. + """ + if event.precision != "day": + return None + days = sorted(day for day in calendar if day >= event.event_date) + if not days: + return None + if event.after_close and days[0] == event.event_date: + return days[1] if len(days) > 1 else None + return days[0] + + +def fetch_pi( + client: Any, + doc: str, + *, + pacer: http.Pacer, + cache: Path, + refresh: bool = False, +) -> str | None: + """Cached ``filed_at`` for one public-inspection document, or ``None`` on 404.""" + path = cache / "fr" / "pi" / f"{doc}.json" + if path.is_file() and not refresh: + payload = json.loads(path.read_text()) + return payload.get("filed_at") or None + url = FR_PI.format(doc=doc) + response = http.get(client, url, pacer=pacer, source="fr", ok_404=True) + if response.status_code == 404: + return None + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(response.content) + http.manifest(path, url) + payload = json.loads(response.content) + return payload.get("filed_at") or None + + +def fetch_ama(url: str, *, client: Any = None) -> str: + """The AMA page body. The caller decides whether it is a refusal.""" + own = client is None + if own: + client = http.client("ama") + try: + response = http.get(client, url, pacer=http.Pacer(0.5), source="ama") + return response.text + finally: + if own: + client.close() + + +def load_rules(store: Any) -> list[dict]: + """PFS rule items (``proposed``, ``final``, ``rule``). Corrections are already out.""" + from pfs.rounds import rule_items + + return rule_items(store, []) + + +def document_numbers(store: Any, keys: Iterable[str]) -> dict[str, str]: + """``item_key → document_number`` from ``fr_anchor_docs``.""" + wanted = [key for key in keys if key] + if not wanted: + return {} + con = store._con() + out: dict[str, str] = {} + try: + for chunk in _chunks(wanted, 400): + marks = ",".join("?" * len(chunk)) + rows = con.execute( + "SELECT item_key, document_number FROM fr_anchor_docs " + f"WHERE item_key IN ({marks})", + list(chunk), + ).fetchall() + for item, doc in rows: + if doc: + out[item] = doc + except Exception as exc: + if _gone(exc): + return {} + raise + return out + + +def load_meeting_paragraphs(store: Any) -> list[tuple[str, int, str]]: + """Anchors that mention a RUC or Panel meeting.""" + con = store._con() + try: + rows = con.execute( + "SELECT item_key, p_id, text FROM fr_anchors " + "WHERE text LIKE '%RUC meeting%' OR text LIKE '%Panel meeting%' " + "OR text LIKE '%Editorial Panel%'" + ).fetchall() + except Exception as exc: + if _gone(exc): + return [] + raise + return [(item, int(p_id), text) for item, p_id, text in rows] + + +def _family_events(con: Any, events: Sequence[MarketEvent]) -> list[MarketEvent]: + exposures = _or_empty(lambda: read_exposures(con), []) + families = _or_empty(lambda: read_families(con), []) + by_code = _or_empty(lambda: read_all_events(con), {}) + exposure: dict[str, set[str]] = {} + for row in exposures: + if row.item_key: + exposure.setdefault(row.family, set()).add(row.item_key) + codes: dict[str, set[str]] = {} + for row in families: + codes.setdefault(row.key, set()).add(row.code) + lineage: dict[str, set[str]] = {} + for fam, fam_codes in codes.items(): + keys = { + ev.item_key + for code in fam_codes + for ev in by_code.get(code, ()) + if ev.source == "fr" and ev.item_key + } + if keys: + lineage[fam] = keys + out: list[MarketEvent] = [] + for fam in sorted(set(exposure) | set(lineage)): + out.extend( + family_rule_events( + events, + exposure_item_keys=exposure.get(fam, set()), + lineage_item_keys=lineage.get(fam, set()), + family=fam, + ) + ) + return out + + +def _specialty_events( + con: Any, events: Sequence[MarketEvent], rules: Sequence[Mapping[str, Any]] +) -> list[MarketEvent]: + from pfs.commenters import specialty_vocab + + try: + names = specialty_vocab(con) + except Exception as exc: + if not _gone(exc): + raise + names = () + titles = {rule["item_key"]: rule.get("title", "") for rule in rules} + impacts = _or_empty(lambda: read_specialty_impacts(con), []) + out: list[MarketEvent] = [] + for spec in dict.fromkeys([*names, *SEAT_LABEL_KEYS]): + out.extend(specialty_rule_events(events, impacts, titles, spec)) + return out + + +def build( + store: Any, + con: Any, + *, + refresh: bool = False, + cache: Path | None = None, + client: Any = None, +) -> tuple[list[MarketEvent], int, list[Tenure]]: + """Program, family and specialty events, the skipped ``rule`` count, and tenures. + + One Federal Register client and one 0.5 s pacer cover every document. + A client passed in is left open. + """ + cache = cache or http.hoard_root() + own = client is None + if own: + client = http.client("fr") + try: + rules = load_rules(store) + docs = document_numbers(store, {rule["item_key"] for rule in rules}) + pacer = http.Pacer(0.5) + pi = { + doc: fetch_pi(client, doc, pacer=pacer, cache=cache, refresh=refresh) + for doc in sorted({d for d in docs.values() if d}) + } + events, skipped = rule_events(rules, docs, pi) + meetings = meeting_mentions(load_meeting_paragraphs(store)) + seats = _or_empty(lambda: read_seats(con), []) + return ( + [ + *events, + *_family_events(con, events), + *_specialty_events(con, events, rules), + *meetings, + ], + skipped, + tenures(seats), + ) + finally: + if own: + client.close() diff --git a/src/market/tables.py b/src/market/tables.py index 9d83f21..715934f 100644 --- a/src/market/tables.py +++ b/src/market/tables.py @@ -124,6 +124,11 @@ CREATE TABLE IF NOT EXISTS market.footprint ( level VARCHAR, key VARCHAR, congress INTEGER, district_id VARCHAR, year INTEGER, n_providers DOUBLE, n_sites DOUBLE, services DOUBLE, payment DOUBLE, method VARCHAR, source VARCHAR +); +CREATE TABLE IF NOT EXISTS market.event ( + event_id VARCHAR, kind VARCHAR, event_date DATE, after_close BOOLEAN, + precision VARCHAR, scope VARCHAR, target VARCHAR, item_key VARCHAR, + p_id INTEGER, document_number VARCHAR, filed_at VARCHAR, note VARCHAR ) """ @@ -514,6 +519,30 @@ class FootprintRow: source: str +@dataclass(frozen=True) +class MarketEvent: + """One public event the study can window (``market.event``). + + ``kind`` is ``rule-proposed``, ``rule-final``, ``ruc-meeting`` or + ``panel-meeting``. ``scope`` is ``program``, ``family`` or ``specialty``. + ``precision`` is ``day`` or ``month``. ``after_close`` is true when the + public-inspection filing was at or after 16:00 America/New_York. + """ + + event_id: str + kind: str + event_date: date + after_close: bool + precision: str + scope: str + target: str + item_key: str + p_id: int + document_number: str + filed_at: str + note: str + + def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int: """Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is a statement per row, far too slow for filing lists).""" @@ -1136,6 +1165,23 @@ def read_footprints(con: Any) -> list[FootprintRow]: ) +def write_events(con: Any, rows: Sequence[MarketEvent]) -> int: + """Replace every stored event. An empty batch deletes nothing.""" + ensure_tables(con) + if not rows: + return 0 + con.execute("DELETE FROM market.event") + return _insert(con, "market.event", rows) + + +def read_events(con: Any) -> list[MarketEvent]: + rows = _select(con, MarketEvent, "market.event") + return sorted( + (MarketEvent(*r) for r in rows), + key=lambda r: (r.event_id, r.scope, r.target), + ) + + # ── districts ──────────────────────────────────────────────────────── diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index a26e20b..29972ff 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -3044,6 +3044,196 @@ def test_district_view_cli_empty(env): assert "no footprint stored" in res.output +def _events_build(): + from market.events import MarketEvent, Tenure + + event = MarketEvent( + "rule-final:2022-23873", + "rule-final", + date(2022, 11, 2), + True, + "day", + "program", + "PFS", + "PETM225D", + 0, + "2022-23873", + "2022-11-02T16:15:00.000-04:00", + "", + ) + family = MarketEvent( + "rule-final:2022-23873", + "rule-final", + date(2022, 11, 2), + True, + "day", + "family", + "HOME-VISITS", + "PETM225D", + 0, + "2022-23873", + "2022-11-02T16:15:00.000-04:00", + "", + ) + tenure = Tenure( + "RUC", + "FAMILY PRACTICE", + ("American Academy of Family Physicians",), + ("permanent",), + 2019, + 2026, + True, + True, + ) + return [event, family], 4, [tenure] + + +def test_events_cli_offline_write(env, monkeypatch): + from market import events as events_api + + c, _, published, _ = env + monkeypatch.setattr(events_api, "build", lambda *a, **k: _events_build()) + res = runner.invoke(app, ["market", "events", "--write"]) + assert res.exit_code == 0, res.output + assert "skipped rule: 4" in res.output + assert "stored:" in res.output + assert "FAMILY PRACTICE" in res.output + assert _count(c, "market.event") == 2 + assert published == [True] + + +def test_events_cli_no_write_prints_counts(env, monkeypatch): + from market import events as events_api + + c, _, published, _ = env + monkeypatch.setattr(events_api, "build", lambda *a, **k: _events_build()) + res = runner.invoke(app, ["market", "events"]) + assert res.exit_code == 0, res.output + assert "skipped rule: 4" in res.output + assert "2 events" in res.output + assert "HOME-VISITS" in res.output + assert "dry run: pass --write" in res.output + assert published == [] + + +def test_events_cli_ama_refusal_gap(env, monkeypatch): + from market import events as events_api + + c, _, published, _ = env + monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, [])) + monkeypatch.setattr( + events_api, + "fetch_ama", + lambda *a, **k: (_ for _ in ()).throw(http.Refused("ama", 403, "denied")), + ) + res = runner.invoke( + app, + [ + "market", + "events", + "--ama-url", + "https://www.ama-assn.org/ruc-meetings", + "--body", + "RUC", + "--write", + ], + ) + assert res.exit_code == 0, res.output + assert "gap: ama:events:" in res.output + gaps = T.read_gaps(c) + assert any(g.source == "ama:events" for g in gaps) + assert published == [True] + + +def test_events_cli_ama_schedule_write(env, monkeypatch): + from market import events as events_api + + c, _, published, _ = env + monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, [])) + monkeypatch.setattr( + events_api, + "fetch_ama", + lambda *a, **k: "The RUC meets April 24–27, 2024.", + ) + res = runner.invoke( + app, + [ + "market", + "events", + "--ama-url", + "https://www.ama-assn.org/ruc-meetings", + "--body", + "RUC", + "--write", + ], + ) + assert res.exit_code == 0, res.output + rows = T.read_events(c) + assert any(r.event_id == "ruc-meeting:2024-04-24" for r in rows) + assert published == [True] + + +def test_events_cli_ama_calendar_stores_both_bodies(env, monkeypatch): + from market import events as events_api + + c, _, published, _ = env + page = ( + "CPT Editorial Panel meetings\nMay 6-8, 2027\n" + "AMA/Specialty Society RVS Update Committee meetings\n" + "Jan. 20-23, 2027\n" + ) + monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, [])) + monkeypatch.setattr(events_api, "fetch_ama", lambda *a, **k: page) + res = runner.invoke( + app, + [ + "market", + "events", + "--ama-url", + "https://www.ama-assn.org/about/cpt-editorial-panel/cpt-editorial-panel-ruc-meetings-calendar", + "--body", + "RUC", + "--write", + ], + ) + assert res.exit_code == 0, res.output + ids = {row.event_id for row in T.read_events(c)} + assert ids == {"ruc-meeting:2027-01-20", "panel-meeting:2027-05-06"} + assert published == [True] + + +def test_events_cli_ama_needs_a_body(env): + res = runner.invoke( + app, ["market", "events", "--ama-url", "https://example.test/ruc"] + ) + assert res.exit_code == 2 + assert "pass --body RUC or CPT_PANEL" in res.output + + +def test_events_cli_ama_page_is_a_refusal(env, monkeypatch): + from market import events as events_api + + c, _, published, _ = env + monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, [])) + monkeypatch.setattr(events_api, "fetch_ama", lambda *a, **k: "Just a moment") + res = runner.invoke( + app, + [ + "market", + "events", + "--ama-url", + "https://example.test/ruc", + "--body", + "RUC", + "--write", + ], + ) + assert res.exit_code == 0, res.output + assert "gap: ama:events: AMA page is a refusal" in res.output + assert any(g.source == "ama:events" for g in T.read_gaps(c)) + assert published == [True] + + def test_district_view_cli_needs_a_key(env): res = runner.invoke(app, ["market", "district-view"]) assert res.exit_code == 2 diff --git a/tests/market/fixtures/ama_schedule_snippet.html b/tests/market/fixtures/ama_schedule_snippet.html new file mode 100644 index 0000000..12af557 --- /dev/null +++ b/tests/market/fixtures/ama_schedule_snippet.html @@ -0,0 +1,8 @@ + + + +
+

The RUC meets April 24–27, 2024.

+
+ + diff --git a/tests/market/fixtures/fr_pi_2022-23873.json b/tests/market/fixtures/fr_pi_2022-23873.json new file mode 100644 index 0000000..261deeb --- /dev/null +++ b/tests/market/fixtures/fr_pi_2022-23873.json @@ -0,0 +1,6 @@ +{ + "document_number": "2022-23873", + "filed_at": "2022-11-02T16:15:00.000-04:00", + "publication_date": "2022-11-18", + "title": "Medicare and Medicaid Programs: CY 2023 Payment Policies under the Physician Fee Schedule and Other Changes to Part B Payment and Coverage Policies; Medicare Shared Savings Program Requirements; Medicare and Medicaid Provider Enrollment Policies, etc." +} diff --git a/tests/market/fixtures/fr_pi_2026-22743.json b/tests/market/fixtures/fr_pi_2026-22743.json new file mode 100644 index 0000000..6b13e4f --- /dev/null +++ b/tests/market/fixtures/fr_pi_2026-22743.json @@ -0,0 +1,6 @@ +{ + "document_number": "2026-22743", + "filed_at": "2026-11-04T17:00:00.000-05:00", + "publication_date": "2026-11-09", + "title": "Medicare and Medicaid Programs: Calendar Year 2027 Payment Policies under the Physician Fee Schedule and Other Changes to Part B Payment and Coverage Policies; Medicare Shared Savings Program Requirements; etc." +} diff --git a/tests/market/test_events.py b/tests/market/test_events.py new file mode 100644 index 0000000..11e8391 --- /dev/null +++ b/tests/market/test_events.py @@ -0,0 +1,697 @@ +"""Event windows: public-inspection releases, meetings and seat tenures.""" + +from __future__ import annotations + +import json +import sqlite3 +from datetime import date +from pathlib import Path + +import duckdb +import httpx +import pytest + +from market import events as E +from market import http +from pfs.codetables import ImpactRow, SeatRow + +FX = Path(__file__).parent / "fixtures" +CAL = [ + date(2026, 7, 14), + date(2026, 11, 4), + date(2026, 11, 5), + date(2026, 11, 6), + date(2026, 11, 9), +] +P193 = ( + "this code was recently reviewed at the April 2022 RUC meeting " + "and it no longer includes the SA046 supply." +) +P291 = ( + "At the February 2025 CPT Editorial Panel meeting, CPT code 20985 " + "was revised to remove image-less." +) +P584 = ( + "Remote Therapeutic Monitoring was valued by the RUC at its January 2021 meeting." +) + + +def _pi(name: str) -> dict: + return json.loads((FX / name).read_text()) + + +def _event(**kw) -> E.MarketEvent: + base = dict( + event_id="rule-final:2022-23873", + kind="rule-final", + event_date=date(2022, 11, 2), + after_close=True, + precision="day", + scope="program", + target="PFS", + item_key="PETM225D", + p_id=0, + document_number="2022-23873", + filed_at="2022-11-02T16:15:00.000-04:00", + note="", + ) + base.update(kw) + return E.MarketEvent(**base) + + +def _rule(item, kind, published, title="Medicare CY2023 PFS"): + return { + "item_key": item, + "title": title, + "date_published": published, + "kind": kind, + } + + +def test_after_close_filing_moves_day0(): + filed = _pi("fr_pi_2026-22743.json")["filed_at"] + assert filed == "2026-11-04T17:00:00.000-05:00" + day, after, note = E.release(filed, date(2026, 11, 9)) + assert (day, after, note) == (date(2026, 11, 4), True, "") + event = _event( + event_id="rule-final:2026-22743", + event_date=day, + after_close=after, + document_number="2026-22743", + filed_at=filed, + ) + assert E.day0(event, CAL) == date(2026, 11, 5) + + +def test_day0_after_close_friday_is_monday(): + event = _event(event_date=date(2026, 11, 6), after_close=True) + assert E.day0(event, CAL) == date(2026, 11, 9) + + +def test_before_close_same_day(): + day, after, note = E.release("2026-07-14T15:59:00.000-04:00", date(2026, 7, 16)) + assert (day, after, note) == (date(2026, 7, 14), False, "") + assert E.day0(_event(event_date=day, after_close=after), CAL) == date(2026, 7, 14) + + +def test_cy2023_final_release_not_publication(): + payload = _pi("fr_pi_2022-23873.json") + day, after, note = E.release(payload["filed_at"], date(2022, 11, 18)) + assert (day, after, note) == (date(2022, 11, 2), True, "") + assert day.isoformat() != payload["publication_date"] + + +def test_publication_fallback_caveat(): + published = date(2010, 11, 2) + assert E.release(None, published) == ( + published, + False, + "event date is FR publication; public-inspection time unknown", + ) + events, skipped = E.rule_events( + [_rule("OLD", "final", "2010-11-02", "CY 2010 PFS")], + {"OLD": "2010-00001"}, + {"2010-00001": None}, + ) + assert skipped == 0 + assert events[0].event_date == published + assert events[0].after_close is False + assert ( + events[0].note == "event date is FR publication; public-inspection time unknown" + ) + assert events[0].event_id == "rule-final:2010-00001" + assert events[0].scope == "program" and events[0].target == "PFS" + + +def test_rule_events_skip_undecidable_kind(): + events, skipped = E.rule_events( + [ + _rule("PETM225D", "final", "2022-11-18"), + _rule("NOPE", "rule", "2022-07-07", "CY 2023 PFS proposed"), + ], + {"PETM225D": "2022-23873", "NOPE": "2022-00001"}, + {"2022-23873": "2022-11-02T16:15:00.000-04:00", "2022-00001": None}, + ) + assert skipped == 1 + assert [e.event_id for e in events] == ["rule-final:2022-23873"] + assert events[0].kind == "rule-final" + assert events[0].item_key == "PETM225D" + + +def test_family_rule_events_from_exposure_and_lineage(): + final = _event() + lineage = _event( + event_id="rule-proposed:2011-1", + kind="rule-proposed", + item_key="LINE", + document_number="2011-1", + ) + other = _event(event_id="rule-final:9999-1", item_key="OTHER") + got = E.family_rule_events( + [final, other, lineage], + exposure_item_keys={"PETM225D"}, + lineage_item_keys={"LINE"}, + family="HOME-VISITS", + ) + assert [e.event_id for e in got] == [ + "rule-final:2022-23873", + "rule-proposed:2011-1", + ] + assert {e.scope for e in got} == {"family"} + assert {e.target for e in got} == {"HOME-VISITS"} + + +def test_specialty_rule_events_threshold(): + event = _event( + event_id="rule-final:2026-22743", + item_key="CY27", + document_number="2026-22743", + event_date=date(2026, 11, 4), + ) + impacts = [ + ImpactRow( + 2027, + "final", + "Family Practice", + "TOTAL", + 10, + 0.01, + 0.01, + 0, + 0.02, + "fr", + "CY27", + 1, + ), + ImpactRow( + 2027, + "final", + "Allergy/Immunology", + "TOTAL", + 1, + 0, + 0, + 0, + 0.004, + "fr", + "CY27", + 1, + ), + ImpactRow( + 2027, + "proposed", + "Family Practice", + "TOTAL", + 10, + 0.2, + 0.2, + 0, + 0.2, + "fr", + "CY27", + 1, + ), + ImpactRow( + 2027, + "final", + "Family Practice", + "Facility", + 10, + 0.5, + 0.5, + 0, + 0.5, + "fr", + "CY27", + 2, + ), + ] + titles = {"CY27": "Medicare CY2027 Physician Fee Schedule"} + got = E.specialty_rule_events([event], impacts, titles, "FAMILY PRACTICE") + assert len(got) == 1 + assert got[0].scope == "specialty" + assert got[0].target == "FAMILY PRACTICE" + assert got[0].note == "CMS estimated impact +2%" + assert E.specialty_rule_events([event], impacts, titles, "ALLERGY/IMMUNOLOGY") == [] + + +def test_meeting_mentions_from_fr(): + got = E.meeting_mentions( + [("5ITGVDJV", 193, P193), ("5ITGVDJV", 291, P291), ("N6MTT7VK", 584, P584)] + ) + by = {e.event_id: e for e in got} + assert set(by) == { + "ruc-meeting:2022-04", + "ruc-meeting:2021-01", + "panel-meeting:2025-02", + } + assert by["ruc-meeting:2022-04"].event_date == date(2022, 4, 1) + assert by["ruc-meeting:2022-04"].precision == "month" + assert by["ruc-meeting:2022-04"].item_key == "5ITGVDJV" + assert by["ruc-meeting:2022-04"].p_id == 193 + assert by["panel-meeting:2025-02"].p_id == 291 + assert by["ruc-meeting:2021-01"].item_key == "N6MTT7VK" + assert by["ruc-meeting:2022-04"].scope == "program" + assert by["ruc-meeting:2022-04"].target == "PFS" + assert E.day0(by["ruc-meeting:2022-04"], CAL) is None + + +def test_meeting_mentions_dedupe_keep_first_cite(): + got = E.meeting_mentions( + [ + ("B", 2, "discussed at the April 2022 RUC meeting"), + ("A", 9, "the April 2022 RUC meeting reviewed it"), + ("A", 3, "again the April 2022 RUC meeting"), + ] + ) + assert len(got) == 1 + assert (got[0].item_key, got[0].p_id) == ("A", 3) + + +def test_ama_schedule_day_dates(): + text = (FX / "ama_schedule_snippet.html").read_text() + got = E.ama_schedule(text, "RUC") + assert len(got) == 1 + assert got[0].event_date == date(2024, 4, 24) + assert got[0].precision == "day" + assert got[0].kind == "ruc-meeting" + assert got[0].event_id == "ruc-meeting:2024-04-24" + + +def test_ama_schedule_splits_abbreviated_calendars(): + text = """ + CPT Editorial Panel meetings + Feb. 4-6, 2027 (Los Angeles, CA) + May 6-8, 2027 (Chicago, IL) + Sept. 16-18, 2027 (Albuquerque, NM) + Foo 1-2, 2027 + AMA/Specialty Society RVS Update Committee meetings + Jan. 20-23, 2027 (Del Mar, CA) + April 29-May 1, 2027 (Chicago, IL) + Sept. 23-25, 2027 (Chicago, IL) + Updated Oct 8, 2026 + """ + assert E.both_calendars(text) + ruc = E.ama_schedule(text, "RUC") + panel = E.ama_schedule(text, "CPT_PANEL") + assert [e.event_date for e in ruc] == [ + date(2027, 1, 20), + date(2027, 4, 29), + date(2027, 9, 23), + ] + assert [e.event_date for e in panel] == [ + date(2027, 2, 4), + date(2027, 5, 6), + date(2027, 9, 16), + ] + assert {e.kind for e in ruc} == {"ruc-meeting"} + assert {e.kind for e in panel} == {"panel-meeting"} + assert not E.both_calendars("The RUC meets April 24–27, 2024.") + + +def test_tenures_span_holder_change(): + rows = [] + for year in range(2019, 2024): + rows.append( + SeatRow( + "RUC", + year, + "American Academy of Family Physicians", + "FAMILY PRACTICE", + "permanent", + "u", + "K", + "s", + "organization", + ) + ) + rows.append( + SeatRow( + "RUC", + year, + "Primary Care Rotating Seat", + "", + "rotating", + "u", + "K", + "s", + "organization", + ) + ) + for year in range(2019, 2023): + rows.append( + SeatRow( + "RUC", + year, + "American College of Physicians", + "INTERNAL MEDICINE", + "permanent", + "u", + "K", + "s", + "organization", + ) + ) + rows.append( + SeatRow( + "RUC", + 2023, + "American College of Physicians", + "INTERNAL MEDICINE", + "rotating", + "u", + "K", + "s", + "organization", + ) + ) + for year in range(2024, 2027): + rows.append( + SeatRow( + "RUC", + year, + "Family Medicine", + "FAMILY PRACTICE", + "permanent", + "u", + "K", + "s", + "specialty", + ) + ) + rows.append( + SeatRow( + "RUC", + 2026, + "Primary Care—additional rotating seat", + "", + "rotating", + "u", + "K", + "s", + "organization", + ) + ) + rows.append( + SeatRow("RUC", 2024, "RUC Chair", "", "chair", "u", "K", "s", "category") + ) + rows.append( + SeatRow( + "RUC", + 2024, + "American Medical Association", + "", + "other", + "u", + "K", + "s", + "organization", + ) + ) + got = {(t.key, t.first_year, t.last_year): t for t in E.tenures(rows)} + fp = got[("FAMILY PRACTICE", 2019, 2026)] + assert fp.holders == ("American Academy of Family Physicians", "Family Medicine") + assert fp.left_censored and fp.right_censored + assert fp.body == "RUC" + im = got[("INTERNAL MEDICINE", 2019, 2023)] + assert im.seat_types == ("permanent", "rotating") + assert ("PRIMARY CARE", 2019, 2023) in got + assert ("PRIMARY CARE", 2026, 2026) in got + assert {k for k, _, _ in got} == { + "FAMILY PRACTICE", + "INTERNAL MEDICINE", + "PRIMARY CARE", + } + + +def test_day0_month_precision_is_none(): + event = _event(precision="month", event_date=date(2022, 4, 1), after_close=False) + assert E.day0(event, CAL) is None + + +def test_fetch_pi_uses_cache_and_404(tmp_path, monkeypatch): + monkeypatch.setattr(http, "hoard_root", lambda: tmp_path) + cache = tmp_path + dest = cache / "fr" / "pi" / "2026-22743.json" + dest.parent.mkdir(parents=True) + dest.write_text((FX / "fr_pi_2026-22743.json").read_text()) + assert ( + E.fetch_pi(None, "2026-22743", pacer=http.Pacer(0), cache=cache, refresh=False) + == "2026-11-04T17:00:00.000-05:00" + ) + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/2010-00001.json"): + return httpx.Response(404) + return httpx.Response( + 200, + json={ + "document_number": "2022-23873", + "filed_at": "2022-11-02T16:15:00.000-04:00", + }, + ) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + assert ( + E.fetch_pi( + client, "2010-00001", pacer=http.Pacer(0), cache=cache, refresh=False + ) + is None + ) + assert ( + E.fetch_pi(client, "2022-23873", pacer=http.Pacer(0), cache=cache, refresh=True) + == "2022-11-02T16:15:00.000-04:00" + ) + assert (cache / "fr" / "pi" / "2022-23873.json").exists() + client.close() + + +def test_build_joins_rules_families_and_tenures(tmp_path, monkeypatch): + monkeypatch.setattr(http, "hoard_root", lambda: tmp_path) + rules = [ + _rule("PETM225D", "final", "2022-11-18", "CY 2023 PFS final"), + _rule("NOPE", "rule", "2022-07-07", "CY 2023 PFS"), + ] + monkeypatch.setattr(E, "load_rules", lambda store: rules) + + class Store: + def __init__(self): + self.c = sqlite3.connect(":memory:") + self.c.execute( + "CREATE TABLE fr_anchor_docs (item_key TEXT, document_number TEXT)" + ) + self.c.execute( + "CREATE TABLE fr_anchors (item_key TEXT, p_id INT, text TEXT)" + ) + self.c.executemany( + "INSERT INTO fr_anchor_docs VALUES (?, ?)", + [("PETM225D", "2022-23873"), ("NOPE", "2022-00001")], + ) + self.c.execute( + "INSERT INTO fr_anchors VALUES (?, ?, ?)", + ("5ITGVDJV", 193, P193), + ) + + def _con(self): + return self.c + + con = duckdb.connect(":memory:") + con.execute("CREATE SCHEMA pfs") + con.execute( + "CREATE TABLE pfs.code_exposure (code VARCHAR, family VARCHAR, " + "effective_date DATE, year INTEGER, kind VARCHAR, status VARCHAR, " + "prior_status VARCHAR, rvu_band INTEGER, non_fac_total DOUBLE, " + "delta_pct DOUBLE, item_key VARCHAR, p_id INTEGER, page INTEGER, " + "anchor_kind VARCHAR, correction BOOLEAN, note VARCHAR)" + ) + con.execute( + "INSERT INTO pfs.code_exposure VALUES " + "('99343', 'HOME-VISITS', DATE '2023-01-01', 2023, 'revalue', '', '', " + "0, 0, 0, 'PETM225D', 1, 1, 'final', false, '')" + ) + con.execute( + "CREATE TABLE pfs.code_family (key VARCHAR, name VARCHAR, code VARCHAR, " + "role VARCHAR, since INTEGER, until INTEGER, item_key VARCHAR, " + "p_id INTEGER, note VARCHAR)" + ) + con.execute( + "INSERT INTO pfs.code_family VALUES " + "('HOME-VISITS', 'Home', '99343', 'base', 2023, NULL, 'PETM225D', 1, '')" + ) + con.execute( + "CREATE TABLE pfs.code_event (code VARCHAR, year INTEGER, kind VARCHAR, " + "from_codes VARCHAR, to_codes VARCHAR, item_key VARCHAR, p_id INTEGER, " + "page INTEGER, source VARCHAR, anchored BOOLEAN, note VARCHAR)" + ) + con.execute( + "INSERT INTO pfs.code_event VALUES " + "('99343', 2023, 'revised', '', '', 'LINE', 2, 1, 'fr', true, '')" + ) + con.execute( + "CREATE TABLE pfs.specialty_impact (rule_year INTEGER, vintage VARCHAR, " + "specialty VARCHAR, setting VARCHAR, allowed_mil DOUBLE, " + "impact_work DOUBLE, impact_pe DOUBLE, impact_mp DOUBLE, " + "impact_combined DOUBLE, source VARCHAR, item_key VARCHAR, page INTEGER)" + ) + con.execute( + "INSERT INTO pfs.specialty_impact VALUES " + "(2023, 'final', 'Family Practice', 'TOTAL', 1, 0, 0, 0, 0.02, " + "'fr', 'PETM225D', 1)" + ) + con.execute( + "CREATE TABLE pfs.governance_seat (body VARCHAR, year INTEGER, " + "organization VARCHAR, specialty VARCHAR, seat_type VARCHAR, " + "source_url VARCHAR, item_key VARCHAR, snapshot VARCHAR, " + "holder_kind VARCHAR, n_seats INTEGER)" + ) + con.execute( + "INSERT INTO pfs.governance_seat VALUES " + "('RUC', 2019, 'American Academy of Family Physicians', " + "'FAMILY PRACTICE', 'permanent', 'u', 'K', 's', 'organization', 1)" + ) + con.execute("CREATE SCHEMA terminology") + con.execute( + "CREATE TABLE terminology.cms_acceptable_provider_specialty_codes " + "(specialty_description VARCHAR)" + ) + con.execute( + "INSERT INTO terminology.cms_acceptable_provider_specialty_codes " + "VALUES ('Family Practice')" + ) + + def filed(client, doc, **kwargs): + return { + "2022-23873": "2022-11-02T16:15:00.000-04:00", + "2022-00001": None, + }.get(doc) + + monkeypatch.setattr(E, "fetch_pi", filed) + store = Store() + events, skipped, tenures = E.build(store, con, cache=tmp_path) + store.c.close() + con.close() + assert skipped == 1 + ids = {(e.event_id, e.scope, e.target) for e in events} + assert ("rule-final:2022-23873", "program", "PFS") in ids + assert ("rule-final:2022-23873", "family", "HOME-VISITS") in ids + assert ("rule-final:2022-23873", "specialty", "FAMILY PRACTICE") in ids + assert ("ruc-meeting:2022-04", "program", "PFS") in ids + assert any(t.key == "FAMILY PRACTICE" and t.first_year == 2019 for t in tenures) + + +def test_load_rules_calls_rule_items(monkeypatch): + import pfs.rounds as rounds + + seen = {} + + def fake(store, anchors): + seen["store"] = store + seen["anchors"] = anchors + return [{"item_key": "K"}] + + monkeypatch.setattr(rounds, "rule_items", fake) + assert E.load_rules("bib") == [{"item_key": "K"}] + assert seen == {"store": "bib", "anchors": []} + + +def test_fetch_ama_opens_and_closes_client(monkeypatch): + seen = {} + + class Fake: + def close(self): + seen["closed"] = True + + def fake_client(source, key=None): + seen["source"] = source + return Fake() + + def fake_get(cl, url, *, pacer, source, **kwargs): + seen["url"] = url + seen["interval"] = pacer.min_interval + seen["get_source"] = source + + class Response: + text = "The RUC meets April 24–27, 2024." + + return Response() + + monkeypatch.setattr(http, "client", fake_client) + monkeypatch.setattr(http, "get", fake_get) + assert "April 24" in E.fetch_ama("https://example.test/ruc") + assert seen["source"] == "ama" + assert seen["get_source"] == "ama" + assert seen["interval"] == 0.5 + assert seen["closed"] is True + + +def test_edges_outside_the_release_calendar(): + meeting = _event(kind="ruc-meeting", precision="month") + assert E.specialty_rule_events([meeting], [], {}, "FAMILY PRACTICE") == [] + assert ( + E.specialty_rule_events([_event()], [], {"PETM225D": "Medicare PFS"}, "X") == [] + ) + assert E.ama_schedule("April 24–27, 2024", "NOPE") == [] + assert E.day0(_event(event_date=date(2030, 1, 2)), []) is None + + +def test_build_missing_tables_is_empty(tmp_path, monkeypatch): + monkeypatch.setattr(E, "load_rules", lambda store: []) + sql = sqlite3.connect(":memory:") + + class Store: + def _con(self): + return sql + + con = duckdb.connect(":memory:") + try: + events, skipped, tenures = E.build( + Store(), con, cache=tmp_path, client=object() + ) + assert E.document_numbers(Store(), []) == {} + assert E.document_numbers(Store(), {"ABSENT"}) == {} + assert E.load_meeting_paragraphs(Store()) == [] + finally: + sql.close() + con.close() + assert (events, skipped, tenures) == ([], 0, []) + + +def test_unexpected_reader_errors_propagate(tmp_path, monkeypatch): + class Boom: + def execute(self, *args, **kwargs): + raise RuntimeError("boom") + + class Store: + def _con(self): + return Boom() + + with pytest.raises(RuntimeError, match="boom"): + E.document_numbers(Store(), {"A"}) + with pytest.raises(RuntimeError, match="boom"): + E.load_meeting_paragraphs(Store()) + + monkeypatch.setattr(E, "load_rules", lambda store: []) + sql = sqlite3.connect(":memory:") + sql.execute("CREATE TABLE fr_anchors (item_key TEXT, p_id INT, text TEXT)") + + class Held: + def _con(self): + return sql + + try: + with pytest.raises(RuntimeError, match="boom"): + E.build(Held(), Boom(), cache=tmp_path, client=object()) + import pfs.commenters as commenters + + def vocab(con): + raise RuntimeError("vocab") + + monkeypatch.setattr(commenters, "specialty_vocab", vocab) + con = duckdb.connect(":memory:") + try: + with pytest.raises(RuntimeError, match="vocab"): + E.build(Held(), con, cache=tmp_path, client=object()) + finally: + con.close() + finally: + sql.close() diff --git a/tests/market/test_imports.py b/tests/market/test_imports.py index 8ce5fd9..63ff3f1 100644 --- a/tests/market/test_imports.py +++ b/tests/market/test_imports.py @@ -17,6 +17,7 @@ MODULES = [ "market.congress", "market.districts", "market.footprint", + "market.events", ] _PROBE = """ diff --git a/tests/market/test_tables.py b/tests/market/test_tables.py index 50b0c3f..72dd43c 100644 --- a/tests/market/test_tables.py +++ b/tests/market/test_tables.py @@ -55,6 +55,7 @@ def test_ensure_tables_idempotent(): "county_district", "entity_site", "footprint", + "event", } @@ -245,6 +246,20 @@ ENTITY_SITE = T.SiteRow( "cms-owners", "owners.csv", ) +EVENT = T.MarketEvent( + "rule-final:2022-23873", + "rule-final", + date(2022, 11, 2), + True, + "day", + "program", + "PFS", + "PETM225D", + 0, + "2022-23873", + "2022-11-02T16:15:00.000-04:00", + "", +) FOOTPRINT = T.FootprintRow( "family", "HOME-VISITS", @@ -327,6 +342,11 @@ ROUNDTRIP = { lambda c, r: T.write_footprints(c, [r]), lambda c: T.read_footprints(c), ), + "event": ( + EVENT, + lambda c, r: T.write_events(c, [r]), + lambda c: T.read_events(c), + ), "entity": ( ENTITY, lambda c, r: T.write_entities(c, [r]), @@ -397,6 +417,7 @@ def test_write_empty_is_noop(con): assert T.write_county_districts(con, []) == 0 assert T.write_entity_sites(con, []) == 0 assert T.write_footprints(con, []) == 0 + assert T.write_events(con, []) == 0 def test_lda_readers_filter_by_entity(con):