RUC seat list + Society:Person list and the CPT Panel Makeup paragraph are parsed, not model-read: one row per seat, rotating from the stars, special seats typed, a curated RUC_SEAT_SPECIALTY crosswalk (DIAGNOSTIC RADIOLOGY added to the vocabulary), holder_kind and n_seats columns, seats_by_specialty from distinct seats, per-year gap lines with ingest_log rows=0, and the model kept only as an opt-in cross-check. Stakeholder units: names (duplicates countable), no seat count, delta_lower_bound; reasons name the codes and the 1% share.
669 lines
24 KiB
Python
669 lines
24 KiB
Python
"""pfs.seats - RUC / CPT Editorial Panel seats from archived AMA rosters (T11)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import duckdb
|
|
import httpx
|
|
import pytest
|
|
|
|
from bib.store import Store
|
|
from pfs import seats
|
|
from pfs.codetables import SeatRow, ensure_tables, read_seats, write_seats
|
|
|
|
URL = "https://www.ama-assn.org/about/rvs-update-committee/ruc-members"
|
|
SPECS = ("GERIATRIC MEDICINE", "INTERNAL MEDICINE", "MULTISPECIALTY")
|
|
|
|
|
|
def _client(handler):
|
|
return httpx.Client(
|
|
transport=httpx.MockTransport(handler),
|
|
headers={"User-Agent": seats.USER_AGENT},
|
|
)
|
|
|
|
|
|
def test_user_agent_is_honest():
|
|
assert seats.USER_AGENT == "stack-pfs-dossier/1.0"
|
|
|
|
|
|
def test_cdx_snapshots_parse():
|
|
seen = []
|
|
|
|
def handler(req):
|
|
seen.append(req)
|
|
y = req.url.params["from"]
|
|
assert req.url.params["to"] == y
|
|
assert req.url.params["url"] == URL
|
|
assert req.url.params["output"] == "json"
|
|
assert req.url.params["filter"] == "statuscode:200"
|
|
assert req.url.params["limit"] == "1"
|
|
assert req.headers["user-agent"] == "stack-pfs-dossier/1.0"
|
|
if y == "2021":
|
|
return httpx.Response(200, json=[])
|
|
head = ["urlkey", "timestamp", "original", "mimetype", "statuscode"]
|
|
return httpx.Response(
|
|
200, json=[head, ["k", f"{y}0105123456", URL, "text/html", "200"]]
|
|
)
|
|
|
|
out = seats.cdx_snapshots(
|
|
_client(handler), URL, range(2020, 2023), sleep=lambda s: None
|
|
)
|
|
assert out == [(2020, "20200105123456"), (2022, "20220105123456")]
|
|
assert len(seen) == 3
|
|
|
|
|
|
def test_cdx_empty_body_is_no_snapshot():
|
|
client = _client(lambda req: httpx.Response(200, content=b""))
|
|
assert (
|
|
seats.cdx_snapshots(client, URL, range(2020, 2021), sleep=lambda s: None) == []
|
|
)
|
|
|
|
|
|
def test_get_backs_off_on_5xx_then_succeeds():
|
|
calls = []
|
|
naps = []
|
|
|
|
def handler(req):
|
|
calls.append(1)
|
|
return httpx.Response(503 if len(calls) < 3 else 200, content=b"ok")
|
|
|
|
r = seats.fetch_snapshot(_client(handler), "https://x/y", sleep=naps.append)
|
|
assert r == b"ok" and len(calls) == 3 and naps == [2.0, 4.0]
|
|
|
|
|
|
def test_get_backs_off_on_429_then_gives_up():
|
|
naps = []
|
|
client = _client(lambda req: httpx.Response(429))
|
|
with pytest.raises(httpx.HTTPStatusError) as e:
|
|
seats.fetch_snapshot(client, "https://x/y", sleep=naps.append)
|
|
assert e.value.response.status_code == 429 and len(naps) == 3
|
|
|
|
|
|
def test_403_is_not_retried():
|
|
naps = []
|
|
calls = []
|
|
|
|
def handler(req):
|
|
calls.append(1)
|
|
return httpx.Response(403)
|
|
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
seats.fetch_snapshot(_client(handler), "https://x/y", sleep=naps.append)
|
|
assert len(calls) == 1 and naps == []
|
|
|
|
|
|
def test_snapshot_url_is_the_raw_id_form():
|
|
assert (
|
|
seats.snapshot_url("20200105123456", URL)
|
|
== f"https://web.archive.org/web/20200105123456id_/{URL}"
|
|
)
|
|
|
|
|
|
def test_save_snapshot_never_overwrites(tmp_path):
|
|
p = tmp_path / "RUC" / "2020.html"
|
|
assert seats.save_snapshot(p, b"one") == p
|
|
assert seats.save_snapshot(p, b"one") == p # same bytes: no-op
|
|
q = seats.save_snapshot(p, b"two")
|
|
assert q != p and q.name.startswith("2020.") and q.suffix == ".html"
|
|
assert p.read_bytes() == b"one" and q.read_bytes() == b"two"
|
|
assert seats.save_snapshot(p, b"two") == q # same new bytes again
|
|
|
|
|
|
def test_html_text_drops_scripts_and_styles():
|
|
html = (
|
|
b"<html><head><style>.a{}</style><script>var x=1;</script></head>"
|
|
b"<body><h1>RUC Members</h1><p>American Medical Association</p>"
|
|
b"<ul><li>American Geriatrics Society</li></ul></body></html>"
|
|
)
|
|
t = seats.html_text(html)
|
|
assert "RUC Members" in t and "American Geriatrics Society" in t
|
|
assert "var x" not in t and ".a{}" not in t
|
|
|
|
|
|
def test_html_text_reads_only_main_when_the_page_has_one():
|
|
html = (
|
|
b"<html><body><nav>Chef Manny turkey</nav><main><h1>Composition</h1>"
|
|
b"<p>American Geriatrics Society</p><script>x()</script></main>"
|
|
b"<footer>Join the AMA</footer></body></html>"
|
|
)
|
|
t = seats.html_text(html)
|
|
assert "American Geriatrics Society" in t
|
|
assert "Chef Manny" not in t and "Join the AMA" not in t and "x()" not in t
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"text,refused",
|
|
[
|
|
("Access Denied. You don't have permission", True),
|
|
("Just a moment... verify you are human", True),
|
|
("", True),
|
|
("x" * 3000 + " captcha", False), # a long real page may mention it
|
|
("American Geriatrics Society holds a permanent seat. " * 20, False),
|
|
],
|
|
)
|
|
def test_is_refusal(text, refused):
|
|
assert seats.is_refusal(text) is refused
|
|
|
|
|
|
def _ask(reply):
|
|
calls = []
|
|
|
|
def ask(system, user, schema):
|
|
calls.append((system, user, schema))
|
|
return reply(user) if callable(reply) else reply
|
|
|
|
ask.calls = calls
|
|
return ask
|
|
|
|
|
|
def test_extract_seats_drops_unseen_org():
|
|
text = "Members: American Geriatrics Society (permanent). ACP (rotating)."
|
|
ask = _ask(
|
|
{
|
|
"seats": [
|
|
{
|
|
"organization": "American Geriatrics Society",
|
|
"seat_type": "permanent",
|
|
"specialty": "GERIATRIC MEDICINE",
|
|
},
|
|
{
|
|
"organization": "American Invented Society",
|
|
"seat_type": "permanent",
|
|
"specialty": "NONE",
|
|
},
|
|
]
|
|
}
|
|
)
|
|
out = seats.extract_seats(text, ask, specialties=SPECS)
|
|
assert out == [
|
|
{
|
|
"organization": "American Geriatrics Society",
|
|
"seat_type": "permanent",
|
|
"specialty": "GERIATRIC MEDICINE",
|
|
}
|
|
]
|
|
schema = ask.calls[0][2]
|
|
enum = schema["properties"]["seats"]["items"]["properties"]["specialty"]["enum"]
|
|
assert enum == [*SPECS, "NONE"]
|
|
assert seats.SEAT_PROMPT_VERSION == "seats-v1"
|
|
assert seats.SEAT_SCHEMA["properties"]["seats"]["type"] == "array"
|
|
|
|
|
|
def test_extract_seats_normalizes_dedupes_and_skips_people():
|
|
text = "Chair Jane Doe, MD. American College of Physicians. american college of physicians."
|
|
ask = _ask(
|
|
{
|
|
"seats": [
|
|
{
|
|
"organization": "Jane Doe, MD",
|
|
"seat_type": "chair",
|
|
"specialty": "NONE",
|
|
},
|
|
{
|
|
"organization": "American College of Physicians",
|
|
"seat_type": "bogus",
|
|
"specialty": "NOT A SPECIALTY",
|
|
},
|
|
{
|
|
"organization": "American College of Physicians",
|
|
"seat_type": "other",
|
|
"specialty": "NONE",
|
|
},
|
|
{"organization": "", "seat_type": "other", "specialty": "NONE"},
|
|
"junk",
|
|
]
|
|
}
|
|
)
|
|
out = seats.extract_seats(text, ask, specialties=SPECS)
|
|
assert out == [
|
|
{
|
|
"organization": "American College of Physicians",
|
|
"seat_type": "other",
|
|
"specialty": "",
|
|
}
|
|
]
|
|
|
|
|
|
def test_extract_seats_reads_long_text_in_chunks():
|
|
text = ("American Geriatrics Society. " * 400) + "Last Org Society."
|
|
ask = _ask(
|
|
lambda user: {
|
|
"seats": [
|
|
{
|
|
"organization": "Last Org Society"
|
|
if "Last Org" in user
|
|
else "American Geriatrics Society",
|
|
"seat_type": "rotating",
|
|
"specialty": "NONE",
|
|
}
|
|
]
|
|
}
|
|
)
|
|
out = seats.extract_seats(text, ask, specialties=SPECS)
|
|
assert len(ask.calls) > 1
|
|
assert {o["organization"] for o in out} == {
|
|
"Last Org Society",
|
|
"American Geriatrics Society",
|
|
}
|
|
|
|
|
|
def test_extract_seats_fails_closed_when_a_chunk_gets_no_reply():
|
|
ask = _ask(None)
|
|
with pytest.raises(seats.ExtractionFailed):
|
|
seats.extract_seats("American Geriatrics Society", ask, specialties=SPECS)
|
|
|
|
|
|
def test_extract_seats_malformed_reply_fails_closed():
|
|
ask = _ask({"nope": []})
|
|
with pytest.raises(seats.ExtractionFailed):
|
|
seats.extract_seats("American Geriatrics Society", ask, specialties=SPECS)
|
|
|
|
|
|
def test_cite_downloads_tag_and_attach(tmp_path):
|
|
store = Store(":memory:", storage_dir=tmp_path / "s")
|
|
try:
|
|
f = tmp_path / "2020.html"
|
|
f.write_bytes(b"<html>x</html>")
|
|
key = seats.cite(store, "CPT_PANEL", 2020, URL, "20200105123456", f)
|
|
again = seats.cite(store, "CPT_PANEL", 2020, URL, "20200105123456", f)
|
|
assert key == again
|
|
item = store.get(key)
|
|
assert item.url == seats.snapshot_url("20200105123456", URL)
|
|
for t in ("source:ama", "doc:cpt-panel-roster", "year:2020", "module:pfs"):
|
|
assert t in item.tags
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
def test_write_seats_replaces_each_year_and_reads_back():
|
|
con = duckdb.connect(":memory:")
|
|
ensure_tables(con)
|
|
a = SeatRow("RUC", 2020, "A", "S1", "permanent", URL, "K", "t1", "organization", 1)
|
|
b = SeatRow("RUC", 2021, "B", "S2", "rotating", URL, "K", "t2")
|
|
o = SeatRow("CPT_PANEL", 2020, "C", "", "other", URL, "K", "t3")
|
|
assert write_seats(con, "RUC", [a, b]) == 2
|
|
write_seats(con, "CPT_PANEL", [o])
|
|
a2 = SeatRow("RUC", 2020, "A2", "S1", "chair", URL, "K", "t1")
|
|
write_seats(con, "RUC", [a2])
|
|
assert read_seats(con, "RUC", 2020) == [a2]
|
|
assert read_seats(con, "RUC") == [a2, b]
|
|
assert read_seats(con) == [o, a2, b]
|
|
assert write_seats(con, "RUC", []) == 0
|
|
assert read_seats(con, "RUC", 2021)[0].holder_kind == ""
|
|
assert read_seats(con, "RUC", 2020)[0].n_seats == 1
|
|
|
|
|
|
def test_chunks_break_at_lines_and_split_a_giant_line():
|
|
lines = ["American Geriatrics Society seat line"] * 400
|
|
chunks = seats._chunks("\n".join(lines))
|
|
assert len(chunks) > 1 and all(len(c) <= seats._CHUNK for c in chunks)
|
|
assert "\n".join(chunks).count("seat line") == 400
|
|
giant = seats._chunks("x" * (seats._CHUNK * 2 + 10))
|
|
assert [len(c) for c in giant] == [seats._CHUNK, seats._CHUNK, 10]
|
|
assert seats._chunks("") == []
|
|
|
|
|
|
# ── R88: deterministic parsing of the two rosters ────────────────────
|
|
|
|
SPECIALS = """Chair
|
|
American Medical Association Representative
|
|
CPT Editorial Panel Representative
|
|
American Osteopathic Association Representative
|
|
Health Care Professionals Advisory Committee Representative
|
|
Practice Expense Review Committee Representative"""
|
|
|
|
|
|
def _page(intro, labels, tail, star="* Indicates rotating seat"):
|
|
return (
|
|
f"Composition of the committee\n{intro}\nThe members include some.\n"
|
|
f"{SPECIALS}\n{labels}\n{star}\nCurrent RUC Members\n{tail}\nStatic\n289\n"
|
|
"Copyright American Medical Association. All rights reserved."
|
|
)
|
|
|
|
|
|
OLD_LABELS = """Anesthesiology
|
|
Infectious Diseases*
|
|
Radiology
|
|
Primary Care*
|
|
Thoracic Surgery"""
|
|
OLD_TAIL = """RUC Chairperson: A Person, MD
|
|
American Medical Association: B Person, MD
|
|
CPT Editorial Panel: C Person, MD
|
|
Health Care Professional Advisory Committee: D Person, PhD
|
|
Practice Expense Subcommittee: E Person, MD
|
|
American College of Radiology: F Person, MD
|
|
American Osteopathic Association: G Person, DO
|
|
American Society of Anesthesiologists: H Person, MD
|
|
Infectious Diseases Society of America: I Person, MD
|
|
Primary Care Rotating Seat: J Person, MD
|
|
Society of Thoracic Surgeons: K Person, MD"""
|
|
OLD_INTRO = (
|
|
"The RUC represents the entire medical profession, with 8 of its 11 members "
|
|
"appointed by societies. The RUC also includes 2 seats that rotate on a "
|
|
"2-year basis."
|
|
)
|
|
|
|
|
|
def test_parse_ruc_society_list_with_rotation_from_the_seat_list():
|
|
got, notes = seats.parse_ruc(_page(OLD_INTRO, OLD_LABELS, OLD_TAIL))
|
|
by = {(g.organization): g for g in got}
|
|
assert len(got) == 11 and len({g.organization for g in got}) == 11
|
|
chair = by["RUC Chair"]
|
|
assert (chair.seat_type, chair.specialty, chair.holder_kind) == (
|
|
"chair",
|
|
"",
|
|
"category",
|
|
)
|
|
assert by["American Medical Association"].seat_type == "other"
|
|
assert by["American Osteopathic Association"].holder_kind == "organization"
|
|
assert by["CPT Editorial Panel"].specialty == ""
|
|
assert by["Health Care Professionals Advisory Committee"].seat_type == "hcpac"
|
|
assert by["Practice Expense Review Committee"].holder_kind == "category"
|
|
ids = by["Infectious Diseases Society of America"]
|
|
assert (ids.seat_type, ids.specialty, ids.holder_kind) == (
|
|
"rotating",
|
|
"INFECTIOUS DISEASE",
|
|
"organization",
|
|
)
|
|
assert by["Primary Care Rotating Seat"].seat_type == "rotating"
|
|
assert by["Primary Care Rotating Seat"].specialty == ""
|
|
acr = by["American College of Radiology"]
|
|
assert (acr.seat_type, acr.specialty) == ("permanent", "DIAGNOSTIC RADIOLOGY")
|
|
assert by["Society of Thoracic Surgeons"].specialty == "THORACIC SURGERY"
|
|
assert by["American Society of Anesthesiologists"].seat_type == "permanent"
|
|
assert sum(g.seat_type == "rotating" for g in got) == 2
|
|
assert notes == [] # 11 members, 2 rotating: all as the page says
|
|
|
|
|
|
def test_parse_ruc_person_star_decides_when_the_society_list_has_stars():
|
|
tail = OLD_TAIL.replace(
|
|
"Primary Care Rotating Seat: J Person, MD",
|
|
"Primary Care Rotating Seat: J Person, MD*",
|
|
)
|
|
tail = tail.replace(
|
|
"Infectious Diseases Society of America: I Person, MD",
|
|
"Infectious Diseases Society of America: I Person, MD",
|
|
)
|
|
labels = OLD_LABELS.replace("Infectious Diseases*", "Infectious Diseases").replace(
|
|
"Anesthesiology", "Anesthesiology*"
|
|
)
|
|
got, notes = seats.parse_ruc(_page(OLD_INTRO, labels, tail))
|
|
by = {g.organization: g for g in got}
|
|
assert by["Primary Care Rotating Seat"].seat_type == "rotating"
|
|
# the seat-list star on Anesthesiology is not applied through a link here
|
|
assert by["American Society of Anesthesiologists"].seat_type == "permanent"
|
|
assert by["Infectious Diseases Society of America"].seat_type == "permanent"
|
|
assert any("rotating" in n for n in notes) # 1 rotating seat vs the page's 2
|
|
|
|
|
|
def test_parse_ruc_names_specialties_not_societies_from_2024():
|
|
labels = "Anesthesiology\nRadiology\nHand Surgery*\nPrimary Care\u2014additional rotating seat*\nOncology*"
|
|
tail = "RUC Chairperson: A Person, MD\nRUC Vice Chair: B Person, MD\nC Person, MD\nD Person, MD"
|
|
intro = (
|
|
"The RUC represents the profession, with 5 of its 11 members appointed. "
|
|
"The RUC also includes three seats that rotate on a two-year basis."
|
|
)
|
|
got, notes = seats.parse_ruc(_page(intro, labels, tail))
|
|
assert len(got) == 11
|
|
by = {g.organization: g for g in got}
|
|
assert by["Radiology"].specialty == "DIAGNOSTIC RADIOLOGY"
|
|
assert by["Radiology"].holder_kind == "specialty"
|
|
assert by["Hand Surgery"].specialty == "HAND SURGERY"
|
|
assert by["Hand Surgery"].seat_type == "rotating"
|
|
pc = by["Primary Care\u2014additional rotating seat"]
|
|
assert (pc.specialty, pc.seat_type) == ("", "rotating")
|
|
assert by["Oncology"].specialty == "" # never forced to a near-miss
|
|
assert by["RUC Chair"].seat_type == "chair"
|
|
assert notes == []
|
|
|
|
|
|
def test_parse_ruc_cardiothoracic_and_unknown_labels():
|
|
labels = "Cardiothoracic Surgery\nCardiology\nAstrology"
|
|
got, _ = seats.parse_ruc(_page("", labels, "RUC Chairperson: A, MD"))
|
|
by = {g.organization: g.specialty for g in got}
|
|
assert by["Cardiothoracic Surgery"] == "THORACIC SURGERY"
|
|
assert by["Cardiology"] == "CARDIOLOGY"
|
|
assert by["Astrology"] == ""
|
|
|
|
|
|
def test_parse_ruc_note_when_member_count_differs_and_society_lines_end():
|
|
tail = OLD_TAIL + "\nAmerican Society of Examples: L Person, MD"
|
|
got, notes = seats.parse_ruc(_page(OLD_INTRO, OLD_LABELS, tail))
|
|
assert len(got) == 12
|
|
assert any("11 members" in n for n in notes)
|
|
assert any("seat list" in n for n in notes)
|
|
|
|
|
|
def test_parse_ruc_trailing_slash_and_primary_care_label_forms():
|
|
tail = (
|
|
"RUC Chairperson: A, MD\n"
|
|
"American College of Chest Physicians (CHEST)/American Thoracic Society (ATS)/: B, MD*\n"
|
|
"*Indicates rotating seat"
|
|
)
|
|
got, _ = seats.parse_ruc(_page("", "Pulmonary Medicine*", tail))
|
|
chest = [g for g in got if g.organization.startswith("American College of Chest")]
|
|
assert len(chest) == 1
|
|
assert chest[0].organization.endswith("(ATS)")
|
|
assert (chest[0].seat_type, chest[0].specialty) == ("rotating", "PULMONARY DISEASE")
|
|
|
|
|
|
def test_parse_ruc_unrecognised_layout_raises():
|
|
with pytest.raises(seats.ParseError):
|
|
seats.parse_ruc("A page about something else entirely.")
|
|
with pytest.raises(seats.ParseError):
|
|
seats.parse_ruc(
|
|
"Chair\nAmerican Medical Association Representative\nOnly two lines"
|
|
)
|
|
with pytest.raises(seats.ParseError):
|
|
seats.parse_ruc(f"{SPECIALS}\nAnesthesiology") # no end-of-list marker
|
|
|
|
|
|
CPT_TEXT = """Makeup of CPT Editorial Panel
|
|
The Panel is representative of all medical professionals, with 12 of its 21 members appointed by the
|
|
national medical specialty societies. In addition to the specialty seats, the Panel includes a seat for the Panel chair, vice chair, two seats for members of the CPT Health Care Professionals Advisory Committee, as well as representatives from these organizations:
|
|
One seat for the Blue Cross and Blue Shield Association
|
|
One seat for the America's Health Insurance Plans
|
|
One seat for the American Hospital Association
|
|
One seat for an at-large organizational member
|
|
One seat for an umbrella organization that represents private health care insurers
|
|
The AMA Board of Trustees selects all CPT Editorial Panel members. Invented text.
|
|
Current Panel members
|
|
Some Person, MD (chair)
|
|
Copyright American Medical Association. All rights reserved."""
|
|
|
|
|
|
def test_parse_cpt_makeup_counts_every_seat_and_ignores_the_footer():
|
|
got, notes = seats.parse_cpt(CPT_TEXT)
|
|
assert notes == []
|
|
assert sum(g.n_seats for g in got) == 21
|
|
by = {g.organization: g for g in got}
|
|
assert by["national medical specialty societies"].n_seats == 12
|
|
assert by["national medical specialty societies"].holder_kind == "category"
|
|
assert by["Panel chair"].seat_type == "chair"
|
|
assert by["Panel vice chair"].n_seats == 1
|
|
hc = by["CPT Health Care Professionals Advisory Committee"]
|
|
assert (hc.n_seats, hc.seat_type) == (2, "hcpac")
|
|
for org in (
|
|
"Blue Cross and Blue Shield Association",
|
|
"America's Health Insurance Plans",
|
|
"American Hospital Association",
|
|
):
|
|
assert by[org].holder_kind == "organization" and by[org].n_seats == 1
|
|
assert by["at-large organizational member"].holder_kind == "category"
|
|
assert (
|
|
by[
|
|
"umbrella organization that represents private health care insurers"
|
|
].holder_kind
|
|
== "category"
|
|
)
|
|
assert "American Medical Association" not in by
|
|
assert all(g.specialty == "" for g in got)
|
|
# a second capture of the same text yields the same rows
|
|
assert seats.parse_cpt(CPT_TEXT + "\nmore boilerplate")[0] == got
|
|
|
|
|
|
def test_parse_cpt_note_when_total_differs_and_unrecognised_raises():
|
|
odd = CPT_TEXT.replace("12 of its 21 members", "12 of its 25 members")
|
|
_, notes = seats.parse_cpt(odd)
|
|
assert any("25 members" in n for n in notes)
|
|
with pytest.raises(seats.ParseError):
|
|
seats.parse_cpt("Nothing about a panel here.")
|
|
with pytest.raises(seats.ParseError):
|
|
seats.parse_cpt(
|
|
"The Panel is representative of all.\nThe AMA Board of Trustees selects x"
|
|
)
|
|
|
|
|
|
def test_to_rows_carry_kind_and_seat_count():
|
|
items = [
|
|
seats.ParsedSeat("Panel chair", "", "chair", "category", 1),
|
|
seats.ParsedSeat(
|
|
"national medical specialty societies", "", "other", "category", 12
|
|
),
|
|
]
|
|
rows = seats.to_rows(
|
|
"CPT_PANEL", 2026, items, source_url=URL, item_key="K", snapshot="t"
|
|
)
|
|
assert rows[1] == SeatRow(
|
|
"CPT_PANEL",
|
|
2026,
|
|
"national medical specialty societies",
|
|
"",
|
|
"other",
|
|
URL,
|
|
"K",
|
|
"t",
|
|
"category",
|
|
12,
|
|
)
|
|
|
|
|
|
def test_ruc_crosswalk_is_curated_and_never_forced():
|
|
xw = seats.RUC_SEAT_SPECIALTY
|
|
assert xw["radiology"] == "DIAGNOSTIC RADIOLOGY"
|
|
assert xw["american college of radiology"] == "DIAGNOSTIC RADIOLOGY"
|
|
assert xw["cardiothoracic surgery"] == "THORACIC SURGERY"
|
|
assert xw["oncology"] == "" and xw["primary care"] == ""
|
|
assert len(xw) >= 40
|
|
|
|
|
|
def _seat(body, year, org, spec, typ, n=1, key="K", kind="organization"):
|
|
return SeatRow(body, year, org, spec, typ, URL, key, "t", kind, n)
|
|
|
|
|
|
def test_seats_by_specialty_counts_distinct_seats():
|
|
rows = [
|
|
_seat(
|
|
"RUC",
|
|
2023,
|
|
"American Geriatrics Society",
|
|
"GERIATRIC MEDICINE",
|
|
"permanent",
|
|
key="K1",
|
|
),
|
|
_seat(
|
|
"RUC",
|
|
2023,
|
|
"American Geriatrics Society",
|
|
"GERIATRIC MEDICINE",
|
|
"permanent",
|
|
key="K1",
|
|
),
|
|
_seat(
|
|
"RUC",
|
|
2023,
|
|
"Renal Physicians Association",
|
|
"NEPHROLOGY",
|
|
"rotating",
|
|
key="K1",
|
|
),
|
|
_seat("RUC", 2023, "Nephrology", "NEPHROLOGY", "rotating", key="K2"),
|
|
_seat("RUC", 2022, "Old", "GERIATRIC MEDICINE", "permanent"),
|
|
_seat("RUC", 2023, "RUC Chair", "", "chair"),
|
|
_seat(
|
|
"CPT_PANEL", 2023, "Some Panel Seat", "NEPHROLOGY", "other", n=2, key="K3"
|
|
),
|
|
]
|
|
got = seats.seats_by_specialty(rows, 2023)
|
|
assert got["GERIATRIC MEDICINE"] == (
|
|
1,
|
|
0,
|
|
0,
|
|
("American Geriatrics Society",),
|
|
("K1",),
|
|
)
|
|
assert got["NEPHROLOGY"] == (
|
|
0,
|
|
2,
|
|
2,
|
|
("Nephrology", "Renal Physicians Association", "Some Panel Seat"),
|
|
("K1", "K2", "K3"),
|
|
)
|
|
assert "" not in got and len(got) == 2
|
|
|
|
|
|
def test_cdx_raises_without_a_gaps_dict_and_records_gaps_with_one():
|
|
client = _client(lambda req: httpx.Response(403))
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
seats.cdx_snapshots(client, URL, range(2020, 2022), sleep=lambda s: None)
|
|
gaps = {}
|
|
assert (
|
|
seats.cdx_snapshots(
|
|
client, URL, range(2020, 2022), sleep=lambda s: None, gaps=gaps
|
|
)
|
|
== []
|
|
)
|
|
assert gaps == {2020: "403", 2021: "403"}
|
|
|
|
def boom(req):
|
|
raise httpx.ConnectError("down")
|
|
|
|
gaps = {}
|
|
seats.cdx_snapshots(_client(boom), URL, [2020], sleep=lambda s: None, gaps=gaps)
|
|
assert gaps == {2020: "ConnectError"}
|
|
|
|
|
|
def test_parse_ruc_rejects_a_short_or_misordered_seat_list():
|
|
with pytest.raises(seats.ParseError):
|
|
seats.parse_ruc(
|
|
"Chair\nAmerican Medical Association Representative\nX\n* Indicates rotating seat"
|
|
)
|
|
wrong = SPECIALS.replace("CPT Editorial Panel Representative", "Something Else")
|
|
with pytest.raises(seats.ParseError):
|
|
seats.parse_ruc(f"{wrong}\nAnesthesiology\n* Indicates rotating seat")
|
|
|
|
|
|
def test_parse_ruc_stops_the_member_list_at_bare_names_and_skips_duplicate_labels():
|
|
labels = "Anesthesiology\nOsteopathic Medicine\nPhysical Therapy\nGastroenterology*\nHospital Medicine*"
|
|
tail = (
|
|
"RUC Chairperson: A Person, MD\nC Person, MD\n"
|
|
"A story title: with a colon that is page furniture"
|
|
)
|
|
got, notes = seats.parse_ruc(_page("", labels, tail))
|
|
names = [g.organization for g in got]
|
|
assert len(got) == 9 # 6 special seats + anesthesiology, gastroenterology, hospital
|
|
assert "Osteopathic Medicine" not in names and "Physical Therapy" not in names
|
|
by = {g.organization: g for g in got}
|
|
assert by["Gastroenterology"].specialty == "GASTROENTEROLOGY"
|
|
assert by["Hospital Medicine"].specialty == "HOSPITALIST"
|
|
assert not any("societies" in n for n in notes)
|
|
|
|
|
|
def test_crosswalk_covers_the_societies_seen_on_the_2019_2023_pages():
|
|
for k in (
|
|
"american college of rheumatology",
|
|
"society for cardiovascular angiography and interventions",
|
|
"interventional cardiology",
|
|
"gastroenterology",
|
|
"hospital medicine",
|
|
):
|
|
assert seats.RUC_SEAT_SPECIALTY[k]
|
|
assert seats._key(
|
|
"The Society for Cardiovascular Angiography and Interventions"
|
|
) in (seats.RUC_SEAT_SPECIALTY)
|
|
|
|
|
|
def test_parse_ruc_skips_lines_before_the_first_member_entry():
|
|
tail = "A heading line\nRUC Chairperson: A, MD\nAmerican Society of Examples: B, MD"
|
|
got, _ = seats.parse_ruc(_page("", "Anesthesiology", tail))
|
|
assert "American Society of Examples" in [g.organization for g in got]
|