Also: the CPT descriptor text is blanked in skin_substitute_hcpcs_reference.csv; level()/ndc11() reject fullwidth digits and trailing newlines, expand() caps ranges, appl_id() rejects unknown kinds.
135 lines
4.3 KiB
Python
135 lines
4.3 KiB
Python
"""The skin-substitute ingest reads the same ASP parser as the drug tier, and
|
|
its output is unchanged (1,974 rows, 2009-Q1..2025-Q4)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import importlib.util
|
|
import math
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
ASP_DIR = ROOT / "data" / "cms" / "asp"
|
|
SKIN_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv"
|
|
REF_CSV = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv"
|
|
|
|
_spec = importlib.util.spec_from_file_location(
|
|
"_ingest_asp", ROOT / "dev" / "scripts" / "ingest_asp.py"
|
|
)
|
|
ingest = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(ingest)
|
|
|
|
needs_data = pytest.mark.skipif(
|
|
not (SKIN_CSV.exists() and any(ASP_DIR.glob("asp_*.zip"))),
|
|
reason="needs the hoarded data/cms/asp files",
|
|
)
|
|
|
|
|
|
def _legacy_pick(names: list[str]) -> str | None:
|
|
"""The selection the dev script made before the shared parser (frozen)."""
|
|
skip = ["noc", "crosswalk", "ndc", "not payable", "not_payable"]
|
|
cand = [
|
|
n
|
|
for n in names
|
|
if not any(s in n.lower() for s in skip)
|
|
and n.lower().endswith((".csv", ".xls", ".xlsx"))
|
|
]
|
|
csvs = [c for c in cand if c.lower().endswith(".csv")]
|
|
if csvs:
|
|
for c in csvs:
|
|
if "payment limit" in c.lower() or "pricing" in c.lower():
|
|
return c
|
|
return csvs[0]
|
|
xlss = [c for c in cand if c.lower().endswith((".xls", ".xlsx"))]
|
|
if xlss:
|
|
for c in xlss:
|
|
low = c.lower()
|
|
if "payment limit" in low or "pricing" in low or "asp" in low:
|
|
return c
|
|
return xlss[0]
|
|
return None
|
|
|
|
|
|
@needs_data
|
|
def test_pick_pricing_file_over_every_hoarded_zip():
|
|
from drugs import asp
|
|
|
|
n = 0
|
|
for z in sorted(ASP_DIR.glob("asp_*.zip")):
|
|
names = zipfile.ZipFile(z).namelist()
|
|
want = _legacy_pick(names)
|
|
assert ingest.pick_pricing_file(names) == want, z.name
|
|
assert asp.pick_pricing_file(names) == want, z.name
|
|
n += 1
|
|
assert n >= 86
|
|
|
|
|
|
@needs_data
|
|
def test_skin_rows_match_hoarded_csv():
|
|
df = ingest.build_skin_df()
|
|
got = sorted(
|
|
df.to_dict("records"),
|
|
key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
|
|
)
|
|
with SKIN_CSV.open(newline="") as fh:
|
|
want = sorted(
|
|
csv.DictReader(fh),
|
|
key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
|
|
)
|
|
assert len(want) == 1974
|
|
assert len(got) == len(want), "an extra or missing skin row appeared"
|
|
for g, w in zip(got, want, strict=True):
|
|
for col in (
|
|
"quarter",
|
|
"hcpcs_code",
|
|
"short_description",
|
|
"dosage_or_unit",
|
|
"source_file",
|
|
"source_url",
|
|
):
|
|
gv = g[col]
|
|
gv = "" if gv is None or (isinstance(gv, float) and math.isnan(gv)) else gv
|
|
assert gv == w[col], (g["quarter"], g["hcpcs_code"], col)
|
|
gp = g["payment_limit"]
|
|
if w["payment_limit"] == "":
|
|
assert gp is None or math.isnan(gp)
|
|
else:
|
|
assert abs(gp - float(w["payment_limit"])) < 1e-9
|
|
assert abs(g["asp_per_unit"] - float(w["asp_per_unit"])) < 1e-9
|
|
|
|
|
|
def test_reference_csv_carries_no_cpt_descriptor_text():
|
|
with REF_CSV.open(newline="") as fh:
|
|
rows = list(csv.DictReader(fh))
|
|
assert len(rows) == 34
|
|
assert list(rows[0]) == [
|
|
"hcpcs_code",
|
|
"short_descriptor",
|
|
"code_type",
|
|
"body_site",
|
|
"wound_size",
|
|
"effective_date",
|
|
"termination_date",
|
|
"status",
|
|
"notes",
|
|
]
|
|
cpt = [r for r in rows if "CPT" in r["code_type"]]
|
|
assert {r["hcpcs_code"] for r in cpt} >= {"15271", "15278", "15330", "15430"}
|
|
assert all(r["short_descriptor"] == "" for r in cpt)
|
|
# CMS's own HCPCS Level II text stays
|
|
level2 = {r["hcpcs_code"]: r["short_descriptor"] for r in rows if r not in cpt}
|
|
assert level2["C5271"].startswith("Low cost skin substitute graft")
|
|
assert level2["G0440"] and level2["C9363"]
|
|
assert all(r["short_descriptor"] for r in rows if r not in cpt)
|
|
|
|
|
|
def test_refuses_to_replace_table_with_nothing():
|
|
import pandas as pd
|
|
|
|
with pytest.raises(RuntimeError, match="no skin-substitute rows"):
|
|
ingest.require_rows(pd.DataFrame())
|
|
ingest.require_rows(pd.DataFrame({"a": [1]}))
|