330 lines
11 KiB
Python
330 lines
11 KiB
Python
"""pfs.psps - Physician/Supplier Procedure Summary by specialty (T10)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import duckdb
|
|
import httpx
|
|
import pytest
|
|
|
|
from bib.store import Store
|
|
from pfs.codetables import PspsRow, ensure_tables
|
|
from pfs.psps import (
|
|
API,
|
|
DATASETS,
|
|
cache_path,
|
|
cite,
|
|
fetch_code_year,
|
|
load_cache,
|
|
newest_cache,
|
|
read_by_specialty,
|
|
save_cache,
|
|
to_rows,
|
|
write,
|
|
)
|
|
|
|
|
|
def rec(**kw):
|
|
base = {
|
|
"HCPCS_CD": "99350",
|
|
"HCPCS_INITIAL_MODIFIER_CD": "",
|
|
"PROVIDER_SPEC_CD": "50",
|
|
"PSPS_SUBMITTED_SERVICE_CNT": "10",
|
|
"PSPS_ALLOWED_CHARGE_AMT": "100.5",
|
|
"PSPS_NCH_PAYMENT_AMT": "80.25",
|
|
"PSPS_ERROR_IND_CD": "0",
|
|
}
|
|
base.update(kw)
|
|
return base
|
|
|
|
|
|
NAMES = {"50": "Nurse Practitioner", "08": "Family Practice"}
|
|
|
|
|
|
class TestDatasets:
|
|
def test_2010_to_2024_and_no_2025(self):
|
|
assert sorted(DATASETS) == list(range(2010, 2025))
|
|
assert DATASETS[2023] == "b72adfb6-22cf-4241-9c64-050ad9061e03"
|
|
|
|
|
|
class TestToRows:
|
|
def test_psps_aggregates_by_specialty(self):
|
|
records = [
|
|
rec(),
|
|
rec(HCPCS_INITIAL_MODIFIER_CD="95", PSPS_SUBMITTED_SERVICE_CNT="5"),
|
|
rec(PROVIDER_SPEC_CD="08", PSPS_ALLOWED_CHARGE_AMT="*"),
|
|
rec(PROVIDER_SPEC_CD="08", PSPS_ALLOWED_CHARGE_AMT="7"),
|
|
rec(PSPS_SUBMITTED_SERVICE_CNT="999", PSPS_ERROR_IND_CD="1"),
|
|
rec(HCPCS_CD="99349"),
|
|
]
|
|
rows = to_rows(
|
|
2023, "99350", records, dataset_id="d", item_key="K", names=NAMES
|
|
)
|
|
by = {r.specialty_cd: r for r in rows}
|
|
assert set(by) == {"08", "50"}
|
|
np_ = by["50"]
|
|
assert (np_.services, np_.allowed, np_.paid) == (15.0, 201.0, 160.5)
|
|
assert np_.specialty == "Nurse Practitioner"
|
|
assert (np_.year, np_.hcpcs, np_.dataset_id, np_.item_key) == (
|
|
2023,
|
|
"99350",
|
|
"d",
|
|
"K",
|
|
)
|
|
fp = by["08"]
|
|
assert fp.services == 20.0
|
|
assert fp.allowed is None # a suppressed part makes the sum None
|
|
|
|
def test_2010_2019_unprefixed_schema(self):
|
|
# real key names of the 2010-2019 files: no PSPS_ prefix
|
|
def old(**kw):
|
|
base = {
|
|
"HCPCS_CD": "99343",
|
|
"PROVIDER_SPEC_CD": "11",
|
|
"SUBMITTED_SERVICE_CNT": "76",
|
|
"ALLOWED_CHARGE_AMT": "8588.72",
|
|
"NCH_PAYMENT_AMT": "6141.23",
|
|
"ERROR_IND_CD": "0",
|
|
}
|
|
base.update(kw)
|
|
return base
|
|
|
|
records = [
|
|
old(),
|
|
old(SUBMITTED_SERVICE_CNT="4"),
|
|
old(
|
|
SUBMITTED_SERVICE_CNT="", ALLOWED_CHARGE_AMT="", NCH_PAYMENT_AMT="87.23"
|
|
),
|
|
old(ERROR_IND_CD="4", SUBMITTED_SERVICE_CNT="999", NCH_PAYMENT_AMT="999"),
|
|
]
|
|
(r,) = to_rows(2019, "99343", records, dataset_id="d", item_key="K", names={})
|
|
assert r.services is None and r.allowed is None
|
|
assert (r.services_reported, r.n_suppressed) == (80.0, 1)
|
|
assert r.paid == pytest.approx(6141.23 * 2 + 87.23)
|
|
|
|
def test_reported_lines_and_suppression_count_kept_beside_the_strict_sum(self):
|
|
records = [
|
|
rec(),
|
|
rec(PSPS_SUBMITTED_SERVICE_CNT="*", PSPS_ALLOWED_CHARGE_AMT="*"),
|
|
rec(PROVIDER_SPEC_CD="08", PSPS_SUBMITTED_SERVICE_CNT="*"),
|
|
]
|
|
by = {
|
|
r.specialty_cd: r
|
|
for r in to_rows(
|
|
2023, "99350", records, dataset_id="d", item_key="K", names=NAMES
|
|
)
|
|
}
|
|
np_, fp = by["50"], by["08"]
|
|
assert np_.services is None and np_.services_reported == 10.0
|
|
assert np_.n_suppressed == 1
|
|
assert fp.services is None and fp.services_reported is None
|
|
assert fp.n_suppressed == 1
|
|
clean = to_rows(2023, "99350", [rec()], dataset_id="d", item_key="K", names={})
|
|
assert (
|
|
clean[0].services,
|
|
clean[0].services_reported,
|
|
clean[0].n_suppressed,
|
|
) == (
|
|
10.0,
|
|
10.0,
|
|
0,
|
|
)
|
|
|
|
def test_unknown_specialty_name_blank_and_bad_numbers_none(self):
|
|
rows = to_rows(
|
|
2023,
|
|
"99350",
|
|
[
|
|
rec(
|
|
PROVIDER_SPEC_CD="ZZ",
|
|
PSPS_SUBMITTED_SERVICE_CNT="",
|
|
PSPS_NCH_PAYMENT_AMT="x",
|
|
)
|
|
],
|
|
dataset_id="d",
|
|
item_key="K",
|
|
names=NAMES,
|
|
)
|
|
assert rows[0].specialty == ""
|
|
assert rows[0].services is None and rows[0].paid is None
|
|
|
|
def test_missing_error_indicator_counts_as_clean(self):
|
|
r = rec()
|
|
del r["PSPS_ERROR_IND_CD"]
|
|
assert to_rows(2023, "99350", [r], dataset_id="d", item_key="K", names={})
|
|
|
|
|
|
def _paged(pages, seen, statuses=None):
|
|
it = iter(pages)
|
|
st = iter(statuses or [])
|
|
|
|
def handler(request):
|
|
seen.append(str(request.url))
|
|
code = next(st, 200)
|
|
if code != 200:
|
|
return httpx.Response(code)
|
|
return httpx.Response(200, json=next(it))
|
|
|
|
return httpx.Client(transport=httpx.MockTransport(handler))
|
|
|
|
|
|
class TestFetch:
|
|
def test_filters_by_code_and_pages_until_short(self):
|
|
seen: list[str] = []
|
|
client = _paged([[rec()] * 2, [rec()]], seen)
|
|
out = fetch_code_year(client, 2023, "99350", page_size=2)
|
|
assert len(out) == 3
|
|
assert "filter%5BHCPCS_CD%5D=99350" in seen[0]
|
|
assert "offset=2" in seen[1]
|
|
assert seen[0].startswith(API.format(id=DATASETS[2023]))
|
|
|
|
def test_5xx_is_retried_with_backoff(self):
|
|
seen: list[str] = []
|
|
naps: list[float] = []
|
|
client = _paged([[rec()]], seen, statuses=[503, 502])
|
|
out = fetch_code_year(client, 2023, "99350", sleep=naps.append)
|
|
assert len(out) == 1 and len(seen) == 3
|
|
assert naps == [2.0, 4.0]
|
|
|
|
def test_persistent_5xx_raises(self):
|
|
client = httpx.Client(
|
|
transport=httpx.MockTransport(lambda r: httpx.Response(503))
|
|
)
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
fetch_code_year(client, 2023, "99350", sleep=lambda s: None)
|
|
|
|
def test_4xx_is_not_retried(self):
|
|
seen: list[str] = []
|
|
client = _paged([], seen, statuses=[429])
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
fetch_code_year(client, 2023, "99350", sleep=lambda s: None)
|
|
assert len(seen) == 1
|
|
|
|
def test_unknown_year(self):
|
|
with pytest.raises(KeyError):
|
|
fetch_code_year(httpx.Client(), 1999, "99350")
|
|
|
|
def test_default_sleep_is_time_sleep(self, monkeypatch):
|
|
import time
|
|
|
|
naps: list[float] = []
|
|
monkeypatch.setattr(time, "sleep", naps.append)
|
|
client = _paged([[rec()]], [], statuses=[500])
|
|
fetch_code_year(client, 2023, "99350")
|
|
assert naps == [2.0]
|
|
|
|
|
|
class TestCache:
|
|
def test_path_layout(self, tmp_path):
|
|
assert (
|
|
cache_path(2023, "99350", root=tmp_path) == tmp_path / "2023" / "99350.json"
|
|
)
|
|
|
|
def test_default_root_is_next_to_the_primary_db(self):
|
|
p = cache_path(2023, "99350")
|
|
assert (
|
|
p.parts[-4:] == ("psps", "2023", "99350.json")[-4:]
|
|
or p.name == "99350.json"
|
|
)
|
|
assert "psps" in p.parts
|
|
|
|
def test_roundtrip(self, tmp_path):
|
|
p = tmp_path / "2023" / "99350.json"
|
|
assert save_cache(p, [rec()]) == p
|
|
assert load_cache(p) == [rec()]
|
|
|
|
def test_hoard_never_overwrites_a_cached_file(self, tmp_path):
|
|
p = tmp_path / "2023" / "99350.json"
|
|
save_cache(p, [rec()])
|
|
before = p.read_bytes()
|
|
# same bytes: a no-op on the same path
|
|
assert save_cache(p, [rec()]) == p
|
|
# different bytes: the old file is untouched, new one beside it
|
|
q = save_cache(p, [rec(PSPS_SUBMITTED_SERVICE_CNT="11")])
|
|
assert q != p and p.read_bytes() == before
|
|
assert (
|
|
q.name.startswith("99350.") and q.suffix == ".json" and len(q.stem) == 6 + 8
|
|
)
|
|
assert json.loads(q.read_text())[0]["PSPS_SUBMITTED_SERVICE_CNT"] == "11"
|
|
# the same different payload again does not rewrite or add a file
|
|
assert save_cache(p, [rec(PSPS_SUBMITTED_SERVICE_CNT="11")]) == q
|
|
assert len(list(p.parent.iterdir())) == 2
|
|
|
|
def test_newest_cache_prefers_latest_write(self, tmp_path):
|
|
import os
|
|
|
|
p = tmp_path / "2023" / "99350.json"
|
|
assert newest_cache(p) == p # nothing cached: the canonical path
|
|
save_cache(p, [rec()])
|
|
os.utime(p, (1, 1))
|
|
q = save_cache(p, [rec(PSPS_SUBMITTED_SERVICE_CNT="11")])
|
|
assert newest_cache(p) == q
|
|
assert load_cache(p)[0]["PSPS_SUBMITTED_SERVICE_CNT"] == "11"
|
|
|
|
|
|
class TestCite:
|
|
def test_one_bib_source_per_year(self):
|
|
store = Store(":memory:")
|
|
try:
|
|
k = cite(store, 2023)
|
|
assert cite(store, 2023) == k
|
|
assert cite(store, 2022) != k
|
|
item = store.get(k)
|
|
assert "CY2023" in item.title
|
|
for t in ("module:pfs", "table:pfs.utilization_specialty", "year:2023"):
|
|
assert t in item.tags
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def con():
|
|
c = duckdb.connect(":memory:")
|
|
ensure_tables(c)
|
|
yield c
|
|
c.close()
|
|
|
|
|
|
class TestWriteRead:
|
|
def _row(self, y, code, cd="50", n=1.0):
|
|
return PspsRow(y, code, cd, "NP", n, 2.0, 3.0, "d", "K")
|
|
|
|
def test_write_replaces_year_and_code_only(self, con):
|
|
assert write(con, 2023, "99350", [self._row(2023, "99350")]) == 1
|
|
write(con, 2023, "99349", [self._row(2023, "99349")])
|
|
write(con, 2022, "99350", [self._row(2022, "99350")])
|
|
assert write(con, 2023, "99350", [self._row(2023, "99350", n=9.0)]) == 1
|
|
rows = read_by_specialty(con, ["99350", "99349"])
|
|
assert [(r.year, r.hcpcs, r.services) for r in rows] == [
|
|
(2022, "99350", 1.0),
|
|
(2023, "99349", 1.0),
|
|
(2023, "99350", 9.0),
|
|
]
|
|
|
|
def test_old_shape_table_gains_the_suppression_columns(self):
|
|
c = duckdb.connect(":memory:")
|
|
c.execute("CREATE SCHEMA pfs")
|
|
c.execute(
|
|
"CREATE TABLE pfs.utilization_specialty (year INTEGER, hcpcs VARCHAR, "
|
|
"specialty_cd VARCHAR, specialty VARCHAR, services DOUBLE, allowed DOUBLE, "
|
|
"paid DOUBLE, dataset_id VARCHAR, item_key VARCHAR)"
|
|
)
|
|
c.execute(
|
|
"INSERT INTO pfs.utilization_specialty VALUES (2020,'A','1','X',1,2,3,'d','K')"
|
|
)
|
|
row = PspsRow(2021, "A", "1", "X", None, None, 3.0, "d", "K", 4.0, 2)
|
|
assert write(c, 2021, "A", [row]) == 1
|
|
got = read_by_specialty(c, ["A"])
|
|
assert got[0].services_reported is None and got[1] == row
|
|
c.close()
|
|
|
|
def test_read_filters_years_and_handles_empty(self, con):
|
|
write(con, 2023, "99350", [self._row(2023, "99350")])
|
|
write(con, 2022, "99350", [self._row(2022, "99350")])
|
|
assert [r.year for r in read_by_specialty(con, ["99350"], years=[2022])] == [
|
|
2022
|
|
]
|
|
assert read_by_specialty(con, []) == []
|
|
assert write(con, 2021, "99350", []) == 0
|