Files
stack/tests/dev/test_ingest_asp_parity.py
kert 7c8c0e72bd feat(drugs): all-code quarterly ASP payment, not-payable and NOC rows with provenance; skin subs read the same parser
Also: the CPT descriptor text is blanked in skin_substitute_hcpcs_reference.csv; level()/ndc11() reject fullwidth digits and trailing newlines, expand() caps ranges, appl_id() rejects unknown kinds.
2026-11-13 04:34:01 -05:00

135 lines
4.3 KiB
Python

"""The skin-substitute ingest reads the same ASP parser as the drug tier, and
its output is unchanged (1,974 rows, 2009-Q1..2025-Q4)."""
from __future__ import annotations
import csv
import importlib.util
import math
import zipfile
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
ASP_DIR = ROOT / "data" / "cms" / "asp"
SKIN_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv"
REF_CSV = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv"
_spec = importlib.util.spec_from_file_location(
"_ingest_asp", ROOT / "dev" / "scripts" / "ingest_asp.py"
)
ingest = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(ingest)
needs_data = pytest.mark.skipif(
not (SKIN_CSV.exists() and any(ASP_DIR.glob("asp_*.zip"))),
reason="needs the hoarded data/cms/asp files",
)
def _legacy_pick(names: list[str]) -> str | None:
"""The selection the dev script made before the shared parser (frozen)."""
skip = ["noc", "crosswalk", "ndc", "not payable", "not_payable"]
cand = [
n
for n in names
if not any(s in n.lower() for s in skip)
and n.lower().endswith((".csv", ".xls", ".xlsx"))
]
csvs = [c for c in cand if c.lower().endswith(".csv")]
if csvs:
for c in csvs:
if "payment limit" in c.lower() or "pricing" in c.lower():
return c
return csvs[0]
xlss = [c for c in cand if c.lower().endswith((".xls", ".xlsx"))]
if xlss:
for c in xlss:
low = c.lower()
if "payment limit" in low or "pricing" in low or "asp" in low:
return c
return xlss[0]
return None
@needs_data
def test_pick_pricing_file_over_every_hoarded_zip():
from drugs import asp
n = 0
for z in sorted(ASP_DIR.glob("asp_*.zip")):
names = zipfile.ZipFile(z).namelist()
want = _legacy_pick(names)
assert ingest.pick_pricing_file(names) == want, z.name
assert asp.pick_pricing_file(names) == want, z.name
n += 1
assert n >= 86
@needs_data
def test_skin_rows_match_hoarded_csv():
df = ingest.build_skin_df()
got = sorted(
df.to_dict("records"),
key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
)
with SKIN_CSV.open(newline="") as fh:
want = sorted(
csv.DictReader(fh),
key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
)
assert len(want) == 1974
assert len(got) == len(want), "an extra or missing skin row appeared"
for g, w in zip(got, want, strict=True):
for col in (
"quarter",
"hcpcs_code",
"short_description",
"dosage_or_unit",
"source_file",
"source_url",
):
gv = g[col]
gv = "" if gv is None or (isinstance(gv, float) and math.isnan(gv)) else gv
assert gv == w[col], (g["quarter"], g["hcpcs_code"], col)
gp = g["payment_limit"]
if w["payment_limit"] == "":
assert gp is None or math.isnan(gp)
else:
assert abs(gp - float(w["payment_limit"])) < 1e-9
assert abs(g["asp_per_unit"] - float(w["asp_per_unit"])) < 1e-9
def test_reference_csv_carries_no_cpt_descriptor_text():
with REF_CSV.open(newline="") as fh:
rows = list(csv.DictReader(fh))
assert len(rows) == 34
assert list(rows[0]) == [
"hcpcs_code",
"short_descriptor",
"code_type",
"body_site",
"wound_size",
"effective_date",
"termination_date",
"status",
"notes",
]
cpt = [r for r in rows if "CPT" in r["code_type"]]
assert {r["hcpcs_code"] for r in cpt} >= {"15271", "15278", "15330", "15430"}
assert all(r["short_descriptor"] == "" for r in cpt)
# CMS's own HCPCS Level II text stays
level2 = {r["hcpcs_code"]: r["short_descriptor"] for r in rows if r not in cpt}
assert level2["C5271"].startswith("Low cost skin substitute graft")
assert level2["G0440"] and level2["C9363"]
assert all(r["short_descriptor"] for r in rows if r not in cpt)
def test_refuses_to_replace_table_with_nothing():
import pandas as pd
with pytest.raises(RuntimeError, match="no skin-substitute rows"):
ingest.require_rows(pd.DataFrame())
ingest.require_rows(pd.DataFrame({"a": [1]}))