feat(drugs): all-code quarterly ASP payment, not-payable and NOC rows with provenance; skin subs read the same parser

Also: the CPT descriptor text is blanked in skin_substitute_hcpcs_reference.csv; level()/ndc11() reject fullwidth digits and trailing newlines, expand() caps ranges, appl_id() rejects unknown kinds.
This commit is contained in:
kert
2026-11-13 04:34:01 -05:00
parent 6cb1b53973
commit 7c8c0e72bd
13 changed files with 2360 additions and 288 deletions

View File

@@ -1,12 +1,12 @@
"hcpcs_code","short_descriptor","code_type","body_site","wound_size","effective_date","termination_date","status","notes"
"15271","Skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Replaced CPT 15330"
"15272","Skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Add-on to 15271"
"15273","Skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15331"
"15274","Skin substitute graft, trunk/arms/legs, >=100 sq cm; each addl 100 sq cm","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Add-on to 15273"
"15275","Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; first 25 sq cm or less","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Replaced CPT 15335"
"15276","Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Add-on to 15275"
"15277","Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15336"
"15278","Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Add-on to 15277"
"15271","","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Replaced CPT 15330"
"15272","","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Add-on to 15271"
"15273","","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15331"
"15274","","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Add-on to 15273"
"15275","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Replaced CPT 15335"
"15276","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Add-on to 15275"
"15277","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15336"
"15278","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Add-on to 15277"
"C5271","Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less","Application (OPPS Low-Cost)","Trunk/arms/legs","<100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026 per CY 2026 OPPS final rule"
"C5272","Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm","Application (OPPS Low-Cost)","Trunk/arms/legs","<100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026"
"C5273","Low cost skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm","Application (OPPS Low-Cost)","Trunk/arms/legs",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026"
@@ -15,15 +15,15 @@
"C5276","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026"
"C5277","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026"
"C5278","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026"
"15330","Acellular dermal allograft, trunk/arms/legs; first 100 sq cm or less","Application (CPT - Historical)","Trunk/arms/legs","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15271/15272 in 2012"
"15331","Acellular dermal allograft, trunk/arms/legs; each addl 100 sq cm","Application (CPT - Historical)","Trunk/arms/legs",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15273/15274 in 2012"
"15335","Acellular dermal allograft, face/scalp/hands/feet; first 100 sq cm or less","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15275/15276 in 2012"
"15336","Acellular dermal allograft, face/scalp/hands/feet; each addl 100 sq cm","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15277/15278 in 2012"
"15340","Tissue cultured allogeneic skin substitute; first 25 sq cm or less","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 tissue-cultured allogeneic graft code"
"15360","Tissue cultured autograft; first 25 sq cm or less","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 autologous cultured skin code"
"15400","Xenograft; first 100 sq cm or less","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft application code"
"15401","Xenograft; each addl 100 sq cm","Application (CPT - Historical)","",">100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft add-on code"
"15430","Acellular xenograft implant; first 100 sq cm or less","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 acellular xenograft code"
"15330","","Application (CPT - Historical)","Trunk/arms/legs","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15271/15272 in 2012"
"15331","","Application (CPT - Historical)","Trunk/arms/legs",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15273/15274 in 2012"
"15335","","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15275/15276 in 2012"
"15336","","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15277/15278 in 2012"
"15340","","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 tissue-cultured allogeneic graft code"
"15360","","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 autologous cultured skin code"
"15400","","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft application code"
"15401","","Application (CPT - Historical)","",">100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft add-on code"
"15430","","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 acellular xenograft code"
"G0440","Skin substitute, tissue cultured; first 25 sq cm (OPPS)","Application (OPPS - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Transitional OPPS code for pass-through products"
"C9363","Integra Meshed Bilayer Wound Matrix, per sq cm","Product (C-code Historical)","","","","","Deleted","Transitional pass-through C-code; replaced by Q-code"
"C9358","SurgiMend Collagen Matrix (fetal), per 0.5 sq cm","Product (C-code Historical)","","","","","Active","Bovine collagen - fetal origin"
1 hcpcs_code short_descriptor code_type body_site wound_size effective_date termination_date status notes
2 15271 Skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less Application (CPT) Trunk/arms/legs <100 sq cm 2012-01-01 Active Replaced CPT 15330
3 15272 Skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm Application (CPT) Trunk/arms/legs <100 sq cm 2012-01-01 Active Add-on to 15271
4 15273 Skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm Application (CPT) Trunk/arms/legs >=100 sq cm 2012-01-01 Active Replaced CPT 15331
5 15274 Skin substitute graft, trunk/arms/legs, >=100 sq cm; each addl 100 sq cm Application (CPT) Trunk/arms/legs >=100 sq cm 2012-01-01 Active Add-on to 15273
6 15275 Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; first 25 sq cm or less Application (CPT) Face/scalp/hands/feet/digits/genitalia <100 sq cm 2012-01-01 Active Replaced CPT 15335
7 15276 Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm Application (CPT) Face/scalp/hands/feet/digits/genitalia <100 sq cm 2012-01-01 Active Add-on to 15275
8 15277 Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm Application (CPT) Face/scalp/hands/feet/digits/genitalia >=100 sq cm 2012-01-01 Active Replaced CPT 15336
9 15278 Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm Application (CPT) Face/scalp/hands/feet/digits/genitalia >=100 sq cm 2012-01-01 Active Add-on to 15277
10 C5271 Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less Application (OPPS Low-Cost) Trunk/arms/legs <100 sq cm 2014-01-01 2025-12-31 Deleted Deleted effective Jan 1 2026 per CY 2026 OPPS final rule
11 C5272 Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm Application (OPPS Low-Cost) Trunk/arms/legs <100 sq cm 2014-01-01 2025-12-31 Deleted Add-on. Deleted effective Jan 1 2026
12 C5273 Low cost skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm Application (OPPS Low-Cost) Trunk/arms/legs >=100 sq cm 2014-01-01 2025-12-31 Deleted Deleted effective Jan 1 2026
15 C5276 Low cost skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm Application (OPPS Low-Cost) Face/scalp/hands/feet/digits/genitalia <100 sq cm 2014-01-01 2025-12-31 Deleted Add-on. Deleted effective Jan 1 2026
16 C5277 Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm Application (OPPS Low-Cost) Face/scalp/hands/feet/digits/genitalia >=100 sq cm 2014-01-01 2025-12-31 Deleted Deleted effective Jan 1 2026
17 C5278 Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm Application (OPPS Low-Cost) Face/scalp/hands/feet/digits/genitalia >=100 sq cm 2014-01-01 2025-12-31 Deleted Add-on. Deleted effective Jan 1 2026
18 15330 Acellular dermal allograft, trunk/arms/legs; first 100 sq cm or less Application (CPT - Historical) Trunk/arms/legs <=100 sq cm 2006-01-01 2011-12-31 Deleted Replaced by 15271/15272 in 2012
19 15331 Acellular dermal allograft, trunk/arms/legs; each addl 100 sq cm Application (CPT - Historical) Trunk/arms/legs >100 sq cm 2006-01-01 2011-12-31 Deleted Replaced by 15273/15274 in 2012
20 15335 Acellular dermal allograft, face/scalp/hands/feet; first 100 sq cm or less Application (CPT - Historical) Face/scalp/hands/feet/digits/genitalia <=100 sq cm 2006-01-01 2011-12-31 Deleted Replaced by 15275/15276 in 2012
21 15336 Acellular dermal allograft, face/scalp/hands/feet; each addl 100 sq cm Application (CPT - Historical) Face/scalp/hands/feet/digits/genitalia >100 sq cm 2006-01-01 2011-12-31 Deleted Replaced by 15277/15278 in 2012
22 15340 Tissue cultured allogeneic skin substitute; first 25 sq cm or less Application (CPT - Historical) <=25 sq cm 2011-12-31 Deleted Pre-2012 tissue-cultured allogeneic graft code
23 15360 Tissue cultured autograft; first 25 sq cm or less Application (CPT - Historical) <=25 sq cm 2011-12-31 Deleted Pre-2012 autologous cultured skin code
24 15400 Xenograft; first 100 sq cm or less Application (CPT - Historical) <=100 sq cm 2011-12-31 Deleted Pre-2012 xenograft application code
25 15401 Xenograft; each addl 100 sq cm Application (CPT - Historical) >100 sq cm 2011-12-31 Deleted Pre-2012 xenograft add-on code
26 15430 Acellular xenograft implant; first 100 sq cm or less Application (CPT - Historical) <=100 sq cm 2011-12-31 Deleted Pre-2012 acellular xenograft code
27 G0440 Skin substitute, tissue cultured; first 25 sq cm (OPPS) Application (OPPS - Historical) <=25 sq cm 2011-12-31 Deleted Transitional OPPS code for pass-through products
28 C9363 Integra Meshed Bilayer Wound Matrix, per sq cm Product (C-code Historical) Deleted Transitional pass-through C-code; replaced by Q-code
29 C9358 SurgiMend Collagen Matrix (fetal), per 0.5 sq cm Product (C-code Historical) Active Bovine collagen - fetal origin

View File

@@ -1,7 +1,8 @@
"""Ingest CMS Average Sales Price (ASP) Drug Pricing Files.
Reads all quarterly ASP ZIP files from data/cms/asp/, normalises column
names across 20+ years of CMS format drift, filters **strictly** to
Reads all quarterly ASP ZIP files from data/cms/asp/ through the drug tier's
parser (``drugs.asp.parse_quarter_zip``: column-name drift, section-508 CSV
before its XLS twin, one file per quarter), filters **strictly** to
skin substitute HCPCS codes defined in data/cms/skin_substitute_hcpcs.csv,
and produces:
@@ -15,8 +16,8 @@ quarterly ZIP and the file within it that was parsed.
Source URLs are read from data/cms/asp_source_urls.csv (canonical URLs
provided from CMS.gov, one per quarter 2005-Q1 through 2026-Q2).
Column mapping rationale
------------------------
Column mapping rationale (the map itself now lives in ``drugs.asp.COL_MAP``)
---------------------------------------------------------------------------
CMS has changed column names across 20+ years of ASP files:
2005–2006: "HCPCS Code", "Short Description", "HCPCS Code Dosage",
@@ -53,23 +54,34 @@ Usage:
from __future__ import annotations
import csv
import io
import re
import zipfile
from pathlib import Path
import pandas as pd
from drugs.asp import newest_per_quarter, parse_quarter_zip, pick_pricing_file # noqa: F401
# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
ROOT = Path(__file__).resolve().parents[2]
ASP_DIR = ROOT / "data" / "cms" / "asp"
HCPCS_REF = ROOT / "data" / "cms" / "skin_substitute_hcpcs.csv"
HCPCS_REFERENCE = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv"
URL_REF = ROOT / "data" / "cms" / "asp_source_urls.csv"
DUCKDB_PATH = ROOT / "data" / "aco.duckdb"
OUTPUT_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv"
OUT_COLS = [
"quarter",
"hcpcs_code",
"short_description",
"asp_per_unit",
"payment_limit",
"dosage_or_unit",
"source_url",
"source_file",
]
def _load_source_urls() -> dict[str, str]:
"""Load canonical source URLs: {quarter: url}."""
@@ -81,142 +93,74 @@ def _load_source_urls() -> dict[str, str]:
return urls
# ---------------------------------------------------------------------------
# Column normalisation
# ---------------------------------------------------------------------------
# Mapping from observed CMS column names (lowercased) to our standard names.
# Each entry documents which CMS file years use that variant.
COL_MAP = {
# HCPCS code — consistent across all years
"hcpcs code": "hcpcs_code",
"hcpcs_code": "hcpcs_code",
# Short description — consistent
"short description": "short_description",
"short_description": "short_description",
# Dosage/unit — varies by year
"hcpcs code dosage": "dosage_or_unit", # 2005-2006, 2020-2025
"hcpcs_code_dosage": "dosage_or_unit",
"dosage": "dosage_or_unit", # 2007-2019
"dosage or unit": "dosage_or_unit", # 2026+
# Payment limit — consistent (ASP + 6%)
"payment limit": "payment_limit",
"payment_limit": "payment_limit",
}
KEEP_COLS = ["hcpcs_code", "short_description", "dosage_or_unit", "payment_limit"]
def require_rows(df: pd.DataFrame) -> None:
"""Refuse to replace ``skin_subs.asp_quarterly`` with nothing: a parser
fault must not publish an empty table over the replica."""
if df.empty:
raise RuntimeError(
"no skin-substitute rows parsed; leaving skin_subs.asp_quarterly as it is"
)
def normalise_columns(df: pd.DataFrame) -> pd.DataFrame:
"""Lower-case columns, apply COL_MAP, keep only KEEP_COLS."""
df.columns = [str(c).strip().lower() for c in df.columns]
df = df.rename(columns=COL_MAP)
present = [c for c in KEEP_COLS if c in df.columns]
return df[present]
# ---------------------------------------------------------------------------
# Readers
# ---------------------------------------------------------------------------
def _find_header_row(lines: list[str]) -> int:
"""Return the 0-based row index of the column-header row.
The header row starts with 'HCPCS Code,' (possibly quoted).
Must NOT match note lines that incidentally mention 'HCPCS code'.
"""
for i, line in enumerate(lines):
stripped = line.strip().lower()
if stripped.startswith("hcpcs code,") or stripped.startswith('"hcpcs code",'):
return i
return -1
def read_csv_from_zip(zf: zipfile.ZipFile, name: str) -> pd.DataFrame:
"""Read a section-508 CSV inside a ZIP, skipping preamble rows."""
raw = zf.read(name).decode("latin-1")
lines = raw.splitlines()
hdr = _find_header_row(lines)
if hdr < 0:
return pd.DataFrame()
buf = "\n".join(lines[hdr:])
df = pd.read_csv(io.StringIO(buf), dtype=str, on_bad_lines="skip")
return normalise_columns(df)
def read_xls_from_zip(zf: zipfile.ZipFile, name: str) -> pd.DataFrame:
"""Read .xls inside a ZIP, skipping preamble rows."""
import xlrd
data = zf.read(name)
wb = xlrd.open_workbook(file_contents=data)
sh = wb.sheet_by_index(0)
hdr_row = -1
for r in range(min(30, sh.nrows)):
first_cell = str(sh.cell_value(r, 0)).strip().lower()
if first_cell == "hcpcs code":
hdr_row = r
break
if hdr_row < 0:
return pd.DataFrame()
headers = [str(sh.cell_value(hdr_row, c)).strip() for c in range(sh.ncols)]
rows = []
for r in range(hdr_row + 1, sh.nrows):
row = [str(sh.cell_value(r, c)).strip() for c in range(sh.ncols)]
if any(row):
rows.append(row)
df = pd.DataFrame(rows, columns=headers)
return normalise_columns(df)
def pick_pricing_file(names: list[str]) -> str | None:
"""From a list of files in a ZIP, pick the ASP pricing file.
Prefer CSV (section 508) over XLS. Skip NOC, crosswalk, NDC,
and "not payable" files.
"""
skip_patterns = ["noc", "crosswalk", "ndc", "not payable", "not_payable"]
candidates: list[str] = []
for n in names:
low = n.lower()
if any(s in low for s in skip_patterns):
continue
if low.endswith((".csv", ".xls", ".xlsx")):
candidates.append(n)
csvs = [c for c in candidates if c.lower().endswith(".csv")]
if csvs:
for c in csvs:
low = c.lower()
if "payment limit" in low or "pricing" in low:
return c
return csvs[0]
xlss = [c for c in candidates if c.lower().endswith((".xls", ".xlsx"))]
if xlss:
for c in xlss:
low = c.lower()
if "payment limit" in low or "pricing" in low or "asp" in low:
return c
return xlss[0]
return None
# ---------------------------------------------------------------------------
# Numeric cleaning
# ---------------------------------------------------------------------------
def clean_numeric(series: pd.Series) -> pd.Series:
"""Strip $ signs and convert to float, coercing errors to NaN."""
return (
series.astype(str)
.str.replace("$", "", regex=False)
.str.replace(",", "", regex=False)
.str.strip()
.pipe(pd.to_numeric, errors="coerce")
def build_skin_df() -> pd.DataFrame:
"""Every payment-limit row of a skin-substitute code (strict filter: only
codes in skin_substitute_hcpcs.csv), one file per quarter, with
``asp_per_unit = payment_limit / 1.06`` (CMS payment limit = ASP + 6% for
drugs/biologicals; no longer meaningful for the flat 2026+ rate, kept for
consistency)."""
skin_codes = set(
pd.read_csv(HCPCS_REF, dtype=str)["hcpcs_code"].str.strip().str.upper()
)
print(f"Loaded {len(skin_codes)} skin-substitute HCPCS codes from reference")
source_urls = _load_source_urls()
print(f"Loaded {len(source_urls)} source URLs")
zips = newest_per_quarter(ASP_DIR.glob("asp_*.zip"))
print(f"Found {len(zips)} quarterly ZIP files\n")
records: list[dict[str, object]] = []
seen_q4: set[str] = set()
total = 0
for quarter, zpath in sorted(zips.items()):
try:
rows, _ = parse_quarter_zip(
zpath, quarter, source_url=source_urls.get(quarter, "")
)
except Exception as e: # a bad zip must not hide the other quarters
print(f" ERR {quarter}: {e}")
continue
payable = [r for r in rows if r.list_kind == "payment"]
if not payable:
print(f" SKIP {quarter}: no pricing rows found in ZIP")
continue
total += len(payable)
print(f" OK {quarter}: {len(payable):>5} rows from {payable[0].source_file}")
for r in payable:
if r.hcpcs in skin_codes:
records.append(
{
"quarter": quarter,
"hcpcs_code": r.hcpcs,
"short_description": r.short_desc,
"payment_limit": r.payment_limit,
"dosage_or_unit": r.dosage,
"source_url": r.source_url,
"source_file": r.source_file,
}
)
elif r.hcpcs.startswith("Q4") and len(r.hcpcs) in (4, 5):
seen_q4.add(r.hcpcs)
print(f"\nTotal rows across all quarters: {total}")
print(f"Skin substitute rows (strict filter): {len(records)}")
if seen_q4:
print(f" Excluded Q4 codes NOT in skin sub universe: {sorted(seen_q4)}")
skin_df = pd.DataFrame.from_records(records)
if skin_df.empty:
return pd.DataFrame(columns=OUT_COLS)
skin_df["payment_limit"] = pd.to_numeric(skin_df["payment_limit"], errors="coerce")
# ASP per unit = payment_limit / 1.06
skin_df["asp_per_unit"] = (skin_df["payment_limit"] / 1.06).round(3)
return skin_df[OUT_COLS].sort_values(["quarter", "hcpcs_code"])
# ---------------------------------------------------------------------------
@@ -225,106 +169,17 @@ def clean_numeric(series: pd.Series) -> pd.Series:
def main() -> None:
# Load skin-substitute HCPCS codes (strict filter — no regex fallback)
hcpcs_ref = pd.read_csv(HCPCS_REF, dtype=str)
skin_codes = set(hcpcs_ref["hcpcs_code"].str.strip().str.upper())
print(f"Loaded {len(skin_codes)} skin-substitute HCPCS codes from reference")
# Load canonical source URLs
source_urls = _load_source_urls()
print(f"Loaded {len(source_urls)} source URLs")
# Ingest all quarters
zips = sorted(ASP_DIR.glob("asp_*.zip"))
print(f"Found {len(zips)} quarterly ZIP files\n")
frames: list[pd.DataFrame] = []
for zpath in zips:
m = re.match(r"asp_(\d{4}-Q\d)", zpath.name)
if not m:
print(f" SKIP {zpath.name} (cannot parse quarter)")
continue
quarter = m.group(1)
try:
with zipfile.ZipFile(zpath) as zf:
target = pick_pricing_file(zf.namelist())
if target is None:
print(f" SKIP {quarter}: no pricing file found in ZIP")
continue
if target.lower().endswith(".csv"):
df = read_csv_from_zip(zf, target)
else:
df = read_xls_from_zip(zf, target)
if df.empty:
print(f" SKIP {quarter}: empty after parsing {target}")
continue
df["quarter"] = quarter
df["source_file"] = target
df["source_url"] = source_urls.get(quarter, "")
frames.append(df)
print(f" OK {quarter}: {len(df):>5} rows from {target}")
except Exception as e:
print(f" ERR {quarter}: {e}")
if not frames:
raise RuntimeError("No data parsed from any file")
all_df = pd.concat(frames, ignore_index=True)
print(f"\nTotal rows across all quarters: {len(all_df)}")
# Normalise HCPCS codes
all_df["hcpcs_code"] = all_df["hcpcs_code"].astype(str).str.strip().str.upper()
# Filter STRICTLY to skin substitute code universe
# NO regex fallback — only codes in skin_substitute_hcpcs.csv
skin_df = all_df[all_df["hcpcs_code"].isin(skin_codes)].copy()
print(f"Skin substitute rows (strict filter): {len(skin_df)}")
# Show what we excluded
q4_but_not_skin = all_df[
all_df["hcpcs_code"].str.match(r"^Q4\d{2,3}$")
& ~all_df["hcpcs_code"].isin(skin_codes)
]["hcpcs_code"].unique()
if len(q4_but_not_skin) > 0:
print(
f" Excluded Q4 codes NOT in skin sub universe: {sorted(q4_but_not_skin)}"
)
if not skin_df.empty:
skin_df["payment_limit"] = clean_numeric(skin_df["payment_limit"])
# ASP per unit = payment_limit / 1.06
# (CMS payment limit = ASP + 6% for drugs/biologicals)
skin_df["asp_per_unit"] = (skin_df["payment_limit"] / 1.06).round(3)
# Reorder columns
out_cols = [
"quarter",
"hcpcs_code",
"short_description",
"asp_per_unit",
"payment_limit",
"dosage_or_unit",
"source_url",
"source_file",
]
for c in out_cols:
if c not in skin_df.columns:
skin_df[c] = None
skin_df = skin_df[out_cols].sort_values(["quarter", "hcpcs_code"])
skin_df = build_skin_df()
require_rows(skin_df)
# Write CSV
skin_df.to_csv(OUTPUT_CSV, index=False)
print(f"\nWrote {len(skin_df)} rows to {OUTPUT_CSV}")
if len(skin_df) > 0:
quarters = sorted(skin_df["quarter"].unique())
codes = sorted(skin_df["hcpcs_code"].unique())
print(f" Quarters: {len(quarters)} ({quarters[0]} to {quarters[-1]})")
print(f" Unique HCPCS codes: {len(codes)}")
quarters = sorted(skin_df["quarter"].unique())
codes = sorted(skin_df["hcpcs_code"].unique())
print(f" Quarters: {len(quarters)} ({quarters[0]} to {quarters[-1]})")
print(f" Unique HCPCS codes: {len(codes)}")
# Load into DuckDB
print(f"\nLoading into DuckDB at {DUCKDB_PATH} ...")
@@ -335,24 +190,26 @@ def main() -> None:
with duckdb_batch("aco") as con:
con.execute("CREATE SCHEMA IF NOT EXISTS skin_subs")
con.execute("DROP TABLE IF EXISTS skin_subs.asp_quarterly")
if not skin_df.empty:
con.execute(
"""
CREATE TABLE skin_subs.asp_quarterly AS
SELECT * FROM read_csv_auto(?, header=true)
""",
[str(OUTPUT_CSV)],
)
count = con.execute("SELECT count(*) FROM skin_subs.asp_quarterly").fetchone()[
0
]
print(f" Loaded {count} rows into skin_subs.asp_quarterly")
if HCPCS_REFERENCE.exists():
# the scrubbed reference (CPT descriptors blanked; codes and
# HCPCS Level II text kept)
con.execute("DROP TABLE IF EXISTS skin_subs.hcpcs_reference")
con.execute(
"""
CREATE TABLE skin_subs.asp_quarterly AS
SELECT * FROM read_csv_auto(?, header=true)
""",
[str(OUTPUT_CSV)],
"CREATE TABLE skin_subs.hcpcs_reference AS "
"SELECT * FROM read_csv_auto(?, header=true)",
[str(HCPCS_REFERENCE)],
)
count = con.execute(
"SELECT count(*) FROM skin_subs.asp_quarterly"
).fetchone()[0]
print(f" Loaded {count} rows into skin_subs.asp_quarterly")
else:
print(" WARNING: No skin substitute data found in ASP files")
print(
" This may be expected — skin subs may not appear in main ASP pricing files"
)
print(" They may be in separate NOC or tissue coding files")
# Refresh the notebook read replica (#510).
print(f"replica → {publish_replica('aco')}")

View File

@@ -2,12 +2,18 @@
from __future__ import annotations
import re
import zipfile
from typing import Any
import typer
from drugs import asp as A
from drugs import http as dh
from drugs import tables as T
from drugs.tables import table_counts
from market.tables import read_gaps, write_gap
from market.http import Refused
from market.tables import clear_gap, read_gaps, write_gap
from pfs.codetables import is_missing_table_error
app = typer.Typer(no_args_is_help=True)
@@ -89,3 +95,162 @@ def status() -> None:
typer.echo("no gaps recorded")
for g in gaps:
typer.echo(f"gap: {g.source}: {g.note} ({g.recorded_on})")
# ── asp ──────────────────────────────────────────────────────────────
ASP_GAP = "cms-asp:asp"
def _cite(store: Any, url: str, title: str, table: str) -> str:
if not url:
return ""
return dh.cite_file(store, url=url, title=title, provider="cms-asp", table=table)
def _quarters_option(values: list[str] | None) -> list[str]:
for v in values or []:
if not re.fullmatch(r"[0-9]{4}-Q[1-4]", v):
raise typer.BadParameter(f"{v!r} is not a quarter like 2026-Q4")
return list(values or [])
@app.command()
def asp(
fetch: bool = typer.Option(False, "--fetch", help="Hoard the page and its zips."),
refresh: bool = typer.Option(False, "--refresh", help="Re-download every zip."),
quarter: list[str] = typer.Option(None, "--quarter", help="Only these quarters."),
dry_run: bool = typer.Option(
False, "--dry-run", help="List the links; change nothing."
),
write: bool = typer.Option(
False, "--write", help="Write drugs.asp_payment and drugs.asp_noc."
),
) -> None:
"""Quarterly Part B payment limits for every code (payable, not-payable
and NOC rows) from the hoarded CMS ASP files."""
sel = set(_quarters_option(quarter))
write = write and not dry_run
cl = None
if fetch:
cl = dh.client("cms")
try:
A.fetch_page(cl)
except Refused as exc:
_gap(ASP_GAP, f"{A.ASP_PAGE} refused: {exc}", write=write)
return
page = A.hoarded_page()
links = [
lk
for lk in (A.parse_asp_page(page.read_text(errors="replace")) if page else [])
if lk.kind in ("pricing", "noc", "crosswalk") and (not sel or lk.quarter in sel)
]
steps = A.plan_fetch(links, refresh=refresh)
if dry_run:
if not links:
typer.echo("no page hoarded; run with --fetch")
for st in steps:
typer.echo(f"{st.link.kind} {st.link.quarter} {st.action} {st.link.url}")
return
refusal = ""
if fetch:
for st in steps:
if st.action == "have":
continue
try:
A.apply_plan(cl, st)
except Refused as exc:
refusal = f"refused: {exc}"
break
typer.echo(f"{st.action} {st.link.kind} {st.link.quarter}")
def pick(kind: str, prefix: str) -> dict[str, Any]:
got = A.newest_per_quarter(A.hoard_dir(kind).glob(f"{prefix}_*.zip"))
return {q: p for q, p in got.items() if not sel or q in sel}
pricing, nocs, xwalks = (
pick("pricing", "asp"),
pick("noc", "noc"),
pick("crosswalk", "xwalk"),
)
url_of = {(lk.quarter, lk.kind): lk.url for lk in links}
canon = A.canonical_urls()
store = _store() if write else None
def source(q: str, kind: str, title: str, table: str) -> tuple[str, str]:
url = url_of.get((q, kind)) or (canon.get(q, "") if kind == "pricing" else "")
return url, (_cite(store, url, title, table) if write else "")
payments: dict[str, list[T.AspRow]] = {}
noc_rows: dict[str, list[T.NocRow]] = {}
files: dict[tuple[str, str], Any] = {}
unreadable: list[str] = []
for q, path in sorted(pricing.items()):
url, key = source(
q, "pricing", f"CMS ASP pricing file {q}", "drugs.asp_payment"
)
try:
rows, bundled = A.parse_quarter_zip(path, q, source_url=url, item_key=key)
except zipfile.BadZipFile:
unreadable.append(f"{q} pricing unreadable")
continue
payments[q] = rows
files[(q, "drugs.asp_payment")] = path
if bundled and q not in nocs:
noc_rows[q] = bundled
files[(q, "drugs.asp_noc")] = path
for q, path in sorted(nocs.items()):
url, key = source(q, "noc", f"CMS ASP NOC pricing file {q}", "drugs.asp_noc")
try:
_, rows = A.parse_quarter_zip(path, q, source_url=url, item_key=key)
except zipfile.BadZipFile:
unreadable.append(f"{q} noc unreadable")
continue
noc_rows[q] = rows
files[(q, "drugs.asp_noc")] = path
for q in sorted(payments):
rows = payments[q]
kinds = [r.list_kind for r in rows]
typer.echo(
f"{q}: {kinds.count('payment')} payment, {kinds.count('not-payable')} not-payable,"
f" {len(noc_rows.get(q, []))} noc"
)
for q in sorted(set(noc_rows) - set(payments)):
typer.echo(f"{q}: {len(noc_rows[q])} noc")
expected = sorted(sel) if sel else A.expected_quarters(A.today())
typer.echo(
f"pricing {len(payments)}/{len(expected)} quarters,"
f" {sum(len(r) for r in payments.values())} rows,"
f" {sum(r.list_kind == 'not-payable' for v in payments.values() for r in v)} not-payable;"
f" noc {len(noc_rows)} quarters, {sum(len(r) for r in noc_rows.values())} rows"
)
have = {
"pricing": set(pricing),
"noc": set(nocs) | set(noc_rows),
"crosswalk": set(xwalks),
}
listed = {(lk.quarter, lk.kind) for lk in links}
pairs = unreadable + A.gap_pairs(expected, have, listed)
note = "; ".join(x for x in (refusal, A.gap_note(pairs)) if x)
if note:
typer.echo(f"gap: {ASP_GAP}: {note}")
if not write:
return
from cms.ingest_log import log_ingest
with _batch() as con:
T.ensure_tables(con)
for q, rows in payments.items():
T.write_asp(con, q, rows)
for q, nrows in noc_rows.items():
T.write_noc(con, q, nrows)
for (q, table), path in sorted(files.items()):
n = len(payments[q]) if table == "drugs.asp_payment" else len(noc_rows[q])
log_ingest(
con, module="drugs.asp", table_name=table, rows=n, source_file=str(path)
)
if note:
write_gap(con, ASP_GAP, note)
else:
clear_gap(con, ASP_GAP)
_publish()

594
src/drugs/asp.py Normal file
View File

@@ -0,0 +1,594 @@
"""CMS Medicare Part B drug payment-limit ("ASP") files, every quarter.
One parser serves two readers: ``stack drugs asp`` (all codes, plus the
"drugs not payable" lists and the not-otherwise-classified drugs) and the
skin-substitute ingest (``dev/scripts/ingest_asp.py``).
Rules held here:
* CMS's short descriptions of CPT (Level I) codes are AMA text and are never
kept: a licensed code's description is ``""``. HCPCS Level II text is CMS's
own and is kept.
* A zip holds one member per kind (pricing, not-payable, NOC); when CMS ships
both a section-508 ``.csv`` and its ``.xls`` twin, the ``.csv`` is read and
the twin is not, so no row is counted twice.
* Files are hoarded under ``data/cms/asp`` and never overwritten; an updated
posting lands beside the old file and is the one read.
* A quarter with no file is a gap, never a zero.
"""
from __future__ import annotations
import csv
import io
import json
import re
import zipfile
from collections.abc import Callable, Iterable, Sequence
from dataclasses import dataclass
from datetime import date
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, urljoin, urlparse
from drugs import http as dh
from drugs.codes import level, licensed
from drugs.tables import ( # noqa: F401 (re-exported)
AspRow,
NocRow,
read_asp,
read_noc,
write_asp,
write_noc,
)
ASP_PAGE = "https://www.cms.gov/medicare/payment/part-b-drugs/asp-pricing-files"
CMS = "https://www.cms.gov"
FIRST_QUARTER = "2005-Q1"
KINDS = ("pricing", "noc", "crosswalk", "vaccine", "other")
def today() -> date:
return date.today()
# ── the page ─────────────────────────────────────────────────────────
@dataclass(frozen=True)
class AspLink:
quarter: str
kind: str
url: str
text: str
class _Anchors(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.found: list[tuple[str, str]] = []
self._href: str | None = None
self._text: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag == "a":
self._href = dict(attrs).get("href")
self._text = []
def handle_data(self, data: str) -> None:
if self._href is not None:
self._text.append(data)
def handle_endtag(self, tag: str) -> None:
if tag == "a" and self._href is not None:
text = re.sub(r"\s+", " ", "".join(self._text).replace("\xa0", " "))
self.found.append((self._href, text.strip()))
self._href = None
_MONTH_Q = {"jan": 1, "apr": 2, "jul": 3, "oct": 4}
_TEXT_Q = re.compile(r"\b(jan|apr|jul|oct)[a-z]*\.?\s+(\d{4})\b")
_HREF_Q = re.compile(r"(january|april|july|october)[-_ ]?(\d{4})")
_HREF_YQ = re.compile(r"(\d{4})[-_ ](january|april|july|october)")
def _quarter_of(text: str, href: str) -> str:
m = _TEXT_Q.search(text.lower())
if m:
return f"{m.group(2)}-Q{_MONTH_Q[m.group(1)]}"
low = href.lower()
m = _HREF_Q.search(low)
if m:
return f"{m.group(2)}-Q{_MONTH_Q[m.group(1)[:3]]}"
m = _HREF_YQ.search(low)
if m:
return f"{m.group(1)}-Q{_MONTH_Q[m.group(2)[:3]]}"
return ""
_KIND_MARKS = (
("crosswalk", ("crosswalk", "ndc")),
("noc", ("noc",)),
("pricing", ("asp pricing", "payment limit")),
)
def _kind_of(text: str) -> str:
"""The first kind the link text names: a bundle such as "ASP Pricing
File/ASP NOC Pricing File/ASP NDC-HCPCS Crosswalk" is its pricing zip."""
low = text.lower()
if "vaccine" in low:
return "vaccine"
best, where = "other", len(low) + 1
for kind, marks in _KIND_MARKS:
for mark in marks:
i = low.find(mark)
if 0 <= i < where:
best, where = kind, i
return best
def parse_asp_page(html: str) -> list[AspLink]:
"""Every zip link of the ASP pricing-files page, with its quarter and
kind. Vaccine and unrecognised links have quarter ``""``."""
p = _Anchors()
p.feed(html)
out: list[AspLink] = []
for href, text in p.found:
url = direct_url(href)
if not urlparse(url).path.lower().endswith(".zip"):
continue
kind = _kind_of(text)
quarter = _quarter_of(text, url)
if kind in ("vaccine", "other") or not quarter:
kind, quarter = ("vaccine" if kind == "vaccine" else "other"), ""
out.append(AspLink(quarter, kind, url, text))
return out
def direct_url(href: str) -> str:
"""The file URL behind a CMS click-through link
(``.../license.asp?file=<p>`` or ``.../license/ama?file=<p>`` ->
``https://www.cms.gov<p>``); a relative href is made absolute."""
parts = urlparse(href)
if parts.path.lower().endswith(("/license.asp", "/license/ama")):
target = parse_qs(parts.query).get("file", [""])[0]
if target:
return urljoin(CMS, target)
return urljoin(CMS, href)
def page_path() -> Path:
return dh.path("drugs", "cms", "asp-pricing-files.html")
def fetch_page(cl: Any) -> Path:
"""Hoard the page (a changed page lands beside the old one)."""
return dh.fetch(
cl, ASP_PAGE, page_path(), provider="cms", source="cms-asp", refresh=True
)
def hoarded_page() -> Path | None:
p = dh.newest(page_path())
return p if p.is_file() else None
# ── hoard layout ─────────────────────────────────────────────────────
_DIRS = {
"pricing": ("asp", "asp"),
"noc": ("asp/noc", "noc"),
"crosswalk": ("asp/xwalk", "xwalk"),
}
def hoard_dir(kind: str) -> Path:
if kind not in _DIRS:
raise ValueError(f"no hoard for kind {kind!r}")
return dh.path("cms", *_DIRS[kind][0].split("/"))
def hoard_path(link: AspLink) -> Path:
return hoard_dir(link.kind) / f"{_DIRS[link.kind][1]}_{link.quarter}.zip"
_ZIP_Q = re.compile(r"^(?:asp|noc|xwalk)_(\d{4}-Q[1-4])(?:\.[0-9a-f]{8})?\.zip$")
def newest_per_quarter(paths: Iterable[Path]) -> dict[str, Path]:
"""One path per quarter: the newest copy (``market.http.newest``'s order:
write time, then name), so an updated posting beside the original wins and
a quarter is never read twice."""
best: dict[str, Path] = {}
for p in paths:
m = _ZIP_Q.match(p.name)
if not m:
continue
q = m.group(1)
key = (p.stat().st_mtime_ns, p.name)
if q not in best or key > (best[q].stat().st_mtime_ns, best[q].name):
best[q] = p
return best
def expected_quarters(today: date) -> list[str]:
"""Every quarter from 2005-Q1 to the one containing *today*."""
out: list[str] = []
y, q = 2005, 1
while (y, q) <= (today.year, (today.month - 1) // 3 + 1):
out.append(f"{y}-Q{q}")
y, q = (y + 1, 1) if q == 4 else (y, q + 1)
return out
# ── the manifest and what to fetch ──────────────────────────────────
def manifest_urls() -> set[str]:
mf = dh.data_root() / "drugs" / dh.MANIFEST
if not mf.exists():
return set()
return {json.loads(x)["url"] for x in mf.read_text().splitlines() if x.strip()}
def canonical_urls() -> dict[str, str]:
"""``data/cms/asp_source_urls.csv``: the URL each of the first 86 pricing
zips was downloaded from."""
p = dh.path("cms", "asp_source_urls.csv")
if not p.exists():
return {}
with p.open(newline="") as fh:
return {r["quarter"]: r["url"] for r in csv.DictReader(fh)}
@dataclass(frozen=True)
class Step:
link: AspLink
dest: Path
action: str # have | adopt | fetch | fetch-beside
def plan_fetch(links: Sequence[AspLink], *, refresh: bool) -> list[Step]:
"""What to do with each pricing, NOC and crosswalk link: ``have`` (its
URL is in the manifest), ``adopt`` (the file on disk was downloaded from
this URL, per ``asp_source_urls.csv``; record it), ``fetch`` (nothing on
disk) or ``fetch-beside`` (a different posting of a hoarded quarter, or
*refresh*; it lands beside the old file)."""
known = manifest_urls()
canon = canonical_urls()
steps: list[Step] = []
for link in links:
if link.kind not in _DIRS or not link.quarter:
continue
dest = hoard_path(link)
if not dest.exists():
action = "fetch"
elif refresh:
action = "fetch-beside"
elif link.url in known:
action = "have"
elif link.kind == "pricing" and canon.get(link.quarter) == link.url:
action = "adopt"
else:
action = "fetch-beside"
steps.append(Step(link, dest, action))
return steps
def apply_plan(cl: Any, step: Step) -> Path:
"""Carry out one step; returns the hoarded path (the newest copy)."""
if step.action == "have":
return dh.newest(step.dest)
if step.action == "adopt":
dh.manifest(
step.dest, step.link.url, "adopted: downloaded from asp_source_urls.csv"
)
return step.dest
return dh.fetch(
cl,
step.link.url,
step.dest,
provider="cms",
source="cms-asp",
refresh=True,
note=f"{step.link.kind} {step.link.quarter}",
)
def gap_pairs(
expected: Sequence[str],
have: dict[str, set[str]],
listed: set[tuple[str, str]],
) -> list[str]:
"""``"<quarter> <kind>"`` for every expected quarter with no pricing file
and every NOC or crosswalk file the page lists but the hoard lacks."""
pairs = [f"{q} pricing" for q in expected if q not in have["pricing"]]
for q, kind in sorted(listed):
if kind != "pricing" and q not in have.get(kind, set()):
pairs.append(f"{q} {kind}")
return pairs
def gap_note(pairs: Sequence[str], show: int = 12) -> str:
if not pairs:
return ""
note = "no hoarded file for: " + ", ".join(pairs[:show])
if len(pairs) > show:
note += f" (+{len(pairs) - show} more)"
return note
# ── reading a file ───────────────────────────────────────────────────
#: observed CMS column names (lower-cased) -> ours
COL_MAP = {
"hcpcs code": "hcpcs_code",
"hcpcs_code": "hcpcs_code",
"short description": "short_description",
"short_description": "short_description",
"long description": "short_description", # the not-payable list
"hcpcs code dosage": "dosage_or_unit",
"hcpcs_code_dosage": "dosage_or_unit",
"dosage": "dosage_or_unit",
"dosage or unit": "dosage_or_unit",
"payment limit": "payment_limit",
"payment_limit": "payment_limit",
"co-insurance percentage": "coinsurance_pct",
"notes": "notes",
"manufacturer": "manufacturer", # the not-payable list
"drug generic name (trade name)": "drug_name", # NOC files
"drug generic name": "drug_name",
"drug name (generic)": "drug_name", # the 2005 NOC sheet
}
_HEADER_STARTS = (
"hcpcs code,",
'"hcpcs code",',
"drug generic name",
'"drug generic name',
"drug name (generic)",
)
def find_header_row(lines: Sequence[str]) -> int:
"""0-based index of the column-header row, or -1. The header starts with
``HCPCS Code,`` (or ``Drug Generic Name`` in NOC files); a note line that
merely mentions HCPCS codes does not match."""
for i, line in enumerate(lines):
if line.strip().lower().startswith(_HEADER_STARTS):
return i
return -1
def _skip_member(low: str) -> bool:
return any(
s in low for s in ("noc", "crosswalk", "ndc", "not payable", "not_payable")
)
def pick_pricing_file(names: Sequence[str]) -> str | None:
"""The ASP pricing member of a zip: ``.csv`` before ``.xls``; NOC,
crosswalk, NDC and "not payable" members are skipped."""
candidates = [
n
for n in names
if not _skip_member(n.lower()) and n.lower().endswith((".csv", ".xls", ".xlsx"))
]
csvs = [c for c in candidates if c.lower().endswith(".csv")]
if csvs:
for c in csvs:
low = c.lower()
if "payment limit" in low or "pricing" in low:
return c
return csvs[0]
xlss = [c for c in candidates if c.lower().endswith((".xls", ".xlsx"))]
if xlss:
for c in xlss:
low = c.lower()
if "payment limit" in low or "pricing" in low or "asp" in low:
return c
return xlss[0]
return None
_MEMBER_MARKS = {
"not-payable": ("not payable", "not_payable", "notpayable"),
"noc": ("noc",),
}
def pick_member(names: Sequence[str], kind: str) -> str | None:
"""The one member of kind ``"not-payable"`` or ``"noc"``: its ``.csv``
when CMS ships a twin, else its ``.xls``."""
if kind not in _MEMBER_MARKS:
raise ValueError(f"no member kind {kind!r}")
marks = _MEMBER_MARKS[kind]
found = [
n
for n in names
if n.lower().endswith((".csv", ".xls", ".xlsx"))
and any(m in n.lower() for m in marks)
and not ("crosswalk" in n.lower() or "ndc" in n.lower())
]
csvs = [n for n in found if n.lower().endswith(".csv")]
return (csvs or found or [None])[0]
def _cell(v: Any) -> str:
if isinstance(v, float) and v == int(v):
return str(int(v))
return str(v).strip()
def read_table(
zf: zipfile.ZipFile,
name: str,
*,
open_workbook: Callable[..., Any] | None = None,
sheet: str = "",
) -> list[dict[str, str]]:
"""The rows of a zip member as ``{our column: text}`` (only the columns of
:data:`COL_MAP`); preamble rows are skipped and blank rows dropped. A CSV
is latin-1 with its header row found; an ``.xls`` goes through
``xlrd.open_workbook`` (injectable for tests): its first sheet, or with
*sheet* the first sheet whose name contains that text (the 2005 files keep
the NOC table on a second sheet)."""
if name.lower().endswith((".xls", ".xlsx")):
return _read_xls(zf, name, open_workbook, sheet)
lines = zf.read(name).decode("latin-1").splitlines()
hdr = find_header_row(lines)
if hdr < 0:
return []
reader = csv.reader(io.StringIO("\n".join(lines[hdr:]), newline=""))
header = [COL_MAP.get(c.strip().lower()) for c in next(reader)]
rows: list[dict[str, str]] = []
for raw in reader:
if len(raw) > len(header):
if any(c.strip() for c in raw[len(header) :]):
continue # more fields than columns: a malformed line
raw = raw[: len(header)]
row: dict[str, str] = {}
for col, val in zip(header, raw, strict=False):
if col and col not in row:
row[col] = val
if any(v.strip() for v in row.values()):
rows.append(row)
return rows
def _read_xls(
zf: zipfile.ZipFile,
name: str,
open_workbook: Callable[..., Any] | None,
sheet: str,
) -> list[dict[str, str]]:
if open_workbook is None:
import xlrd
open_workbook = xlrd.open_workbook
wb = open_workbook(file_contents=zf.read(name))
index = 0
if sheet:
found = [i for i, n in enumerate(wb.sheet_names()) if sheet in n.lower()]
if not found:
return []
index = found[0]
sh = wb.sheet_by_index(index)
hdr = -1
for r in range(min(30, sh.nrows)):
first = str(sh.cell_value(r, 0)).strip().lower()
if first == "hcpcs code" or first.startswith(
("drug generic name", "drug name (generic)")
):
hdr = r
break
if hdr < 0:
return []
header = [
COL_MAP.get(_cell(sh.cell_value(hdr, c)).lower()) for c in range(sh.ncols)
]
rows: list[dict[str, str]] = []
for r in range(hdr + 1, sh.nrows):
vals = [_cell(sh.cell_value(r, c)) for c in range(sh.ncols)]
if not any(vals):
continue
row: dict[str, str] = {}
for col, val in zip(header, vals, strict=True):
if col and col not in row:
row[col] = val
rows.append(row)
return rows
def _number(raw: str | None) -> float | None:
s = (raw or "").replace("$", "").replace(",", "").strip()
if not s:
return None
try:
return float(s)
except ValueError:
return None
def _asp_rows(
rows: Sequence[dict[str, str]],
*,
quarter: str,
list_kind: str,
member: str,
source_url: str,
item_key: str,
) -> list[AspRow]:
out: list[AspRow] = []
for r in rows:
code = r.get("hcpcs_code", "").strip().upper()
if not code:
continue
payable = list_kind == "payment"
out.append(
AspRow(
quarter=quarter,
hcpcs=code,
level=level(code),
list_kind=list_kind,
dosage=r.get("dosage_or_unit", ""),
payment_limit=_number(r.get("payment_limit")) if payable else None,
coinsurance_pct=_number(r.get("coinsurance_pct")) if payable else None,
short_desc="" if licensed(code) else r.get("short_description", ""),
notes=r.get("notes", ""),
source_file=member,
source_url=source_url,
item_key=item_key,
manufacturer=r.get("manufacturer", "").strip() if not payable else "",
)
)
return out
def parse_quarter_zip(
path: Path,
quarter: str,
*,
source_url: str,
item_key: str = "",
open_workbook: Callable[..., Any] | None = None,
) -> tuple[list[AspRow], list[NocRow]]:
"""The payment, not-payable and NOC rows of one quarter's zip: one member
per kind. A licensed (CPT/CDT) code gets ``short_desc=""``."""
asp: list[AspRow] = []
noc: list[NocRow] = []
with zipfile.ZipFile(path) as zf:
names = zf.namelist()
pricing = pick_pricing_file(names)
for kind, member in (
("payment", pricing),
("not-payable", pick_member(names, "not-payable")),
):
if member:
asp += _asp_rows(
read_table(zf, member, open_workbook=open_workbook),
quarter=quarter,
list_kind=kind,
member=member,
source_url=source_url,
item_key=item_key,
)
member = pick_member(names, "noc")
table: list[dict[str, str]] = []
if member:
table = read_table(zf, member, open_workbook=open_workbook)
elif pricing and pricing.lower().endswith((".xls", ".xlsx")):
table = read_table(zf, pricing, open_workbook=open_workbook, sheet="noc")
member = f"{pricing}#NOC"
if member:
for r in table:
name = r.get("drug_name", "").strip()
if name:
noc.append(
NocRow(
quarter=quarter,
drug_name=name,
dosage=r.get("dosage_or_unit", ""),
payment_limit=_number(r.get("payment_limit")),
source_file=member,
source_url=source_url,
item_key=item_key,
)
)
return asp, noc

View File

@@ -5,18 +5,18 @@ from __future__ import annotations
import re
from datetime import date, timedelta
_L1 = re.compile(r"^\d{4}[0-9A-Z]$")
_L2 = re.compile(r"^[ABCEGHJKLMPQRSTUV]\d{4}$")
_CDT = re.compile(r"^D\d{4}$")
_L1 = re.compile(r"[0-9]{4}[0-9A-Z]")
_L2 = re.compile(r"[ABCEGHJKLMPQRSTUV][0-9]{4}")
_CDT = re.compile(r"D[0-9]{4}")
def level(code: str) -> str:
"""``"I"`` (CPT), ``"II"`` (HCPCS Level II), ``"CDT"`` or ``""``."""
if _L1.match(code):
if _L1.fullmatch(code):
return "I"
if _L2.match(code):
if _L2.fullmatch(code):
return "II"
if _CDT.match(code):
if _CDT.fullmatch(code):
return "CDT"
return ""
@@ -30,9 +30,9 @@ def ndc11(raw: str) -> str | None:
"""The 11-digit NDC of a 4-4-2, 5-3-2, 5-4-1 or 5-4-2 hyphenated code, or
of 11 bare digits; anything else is ``None``."""
s = raw.strip()
if re.fullmatch(r"\d{11}", s):
if re.fullmatch(r"[0-9]{11}", s):
return s
m = re.fullmatch(r"(\d+)-(\d+)-(\d+)", s)
m = re.fullmatch(r"([0-9]+)-([0-9]+)-([0-9]+)", s)
if not m:
return None
a, b, c = m.groups()
@@ -48,27 +48,36 @@ def ndc11(raw: str) -> str | None:
return None
_APPL = {"N": "NDA", "A": "ANDA", "BLA": "BLA"}
_APPL = {"N": "NDA", "NDA": "NDA", "A": "ANDA", "ANDA": "ANDA", "BLA": "BLA"}
def appl_id(kind: str, number: str) -> str:
"""``("N","21436")`` -> ``NDA021436``; an id already spelled out passes."""
if re.fullmatch(r"[A-Z]+\d{6}", kind):
if re.fullmatch(r"[A-Z]+[0-9]{6}", kind):
return kind
return f"{_APPL.get(kind, kind)}{int(number):06d}"
if kind not in _APPL:
raise ValueError(f"unknown application kind {kind!r}")
return f"{_APPL[kind]}{int(number):06d}"
#: the longest range :func:`expand` will list; a wider one is a data error
MAX_RANGE = 500
def expand(lo: str, hi: str) -> list[str]:
"""Every code from *lo* to *hi* when they share a letter prefix and a
numeric tail; otherwise just *lo*."""
a = re.fullmatch(r"([A-Z]+)(\d+)", lo)
b = re.fullmatch(r"([A-Z]+)(\d+)", hi)
numeric tail; otherwise just *lo*. A range wider than :data:`MAX_RANGE`
codes raises ``ValueError``."""
a = re.fullmatch(r"([A-Z]+)([0-9]+)", lo)
b = re.fullmatch(r"([A-Z]+)([0-9]+)", hi)
if not a or not b or a.group(1) != b.group(1):
return [lo]
width = len(a.group(2))
start, end = int(a.group(2)), int(b.group(2))
if end < start or len(b.group(2)) != width:
return [lo]
if end - start + 1 > MAX_RANGE:
raise ValueError(f"range too large: {lo}-{hi} is {end - start + 1} codes")
return [f"{a.group(1)}{n:0{width}d}" for n in range(start, end + 1)]

View File

@@ -7,11 +7,35 @@ tables. Nothing opens a database at import.
from __future__ import annotations
from collections.abc import Iterable, Sequence
from dataclasses import astuple, fields
from dataclasses import astuple, dataclass, fields
from typing import Any
_DDL = """
CREATE SCHEMA IF NOT EXISTS drugs
CREATE SCHEMA IF NOT EXISTS drugs;
CREATE TABLE IF NOT EXISTS drugs.asp_payment (
quarter VARCHAR,
hcpcs VARCHAR,
level VARCHAR,
list_kind VARCHAR,
dosage VARCHAR,
payment_limit DOUBLE,
coinsurance_pct DOUBLE,
short_desc VARCHAR,
notes VARCHAR,
source_file VARCHAR,
source_url VARCHAR,
item_key VARCHAR,
manufacturer VARCHAR
);
CREATE TABLE IF NOT EXISTS drugs.asp_noc (
quarter VARCHAR,
drug_name VARCHAR,
dosage VARCHAR,
payment_limit DOUBLE,
source_file VARCHAR,
source_url VARCHAR,
item_key VARCHAR
)
"""
@@ -61,3 +85,79 @@ def _select(con: Any, cls: type, table: str, where: str = "", params: Sequence =
def _chunks(items: Sequence[Any], n: int = 500) -> Iterable[Sequence[Any]]:
for i in range(0, len(items), n):
yield items[i : i + n]
# ── asp_payment / asp_noc ────────────────────────────────────────────
@dataclass(frozen=True)
class AspRow:
"""One line of a quarterly Part B payment-limit file. ``list_kind`` is
``payment`` or ``not-payable`` (no Medicaid rebate agreement; carries the
file's manufacturer). ``short_desc`` is ``""`` for a licensed (CPT/CDT)
code. ``payment_limit`` is the CMS payment allowance (ASP + 6%), never
the ASP itself."""
quarter: str
hcpcs: str
level: str
list_kind: str
dosage: str
payment_limit: float | None
coinsurance_pct: float | None
short_desc: str
notes: str
source_file: str
source_url: str
item_key: str
manufacturer: str = ""
@dataclass(frozen=True)
class NocRow:
"""One line of a quarterly not-otherwise-classified drug pricing file."""
quarter: str
drug_name: str
dosage: str
payment_limit: float | None
source_file: str
source_url: str
item_key: str
def write_asp(con: Any, quarter: str, rows: Sequence[AspRow]) -> int:
"""Replace *quarter* of ``drugs.asp_payment`` with *rows*."""
ensure_tables(con)
con.execute("DELETE FROM drugs.asp_payment WHERE quarter = ?", [quarter])
return _insert(con, "drugs.asp_payment", rows)
def read_asp(
con: Any, *, hcpcs: str | None = None, quarter: str | None = None
) -> list[AspRow]:
where, params = [], []
if hcpcs is not None:
where.append("hcpcs = ?")
params.append(hcpcs)
if quarter is not None:
where.append("quarter = ?")
params.append(quarter)
rows = _select(con, AspRow, "drugs.asp_payment", " AND ".join(where), params)
return sorted(
(AspRow(*r) for r in rows),
key=lambda r: (r.quarter, r.hcpcs, r.list_kind, r.dosage),
)
def write_noc(con: Any, quarter: str, rows: Sequence[NocRow]) -> int:
"""Replace *quarter* of ``drugs.asp_noc`` with *rows*."""
ensure_tables(con)
con.execute("DELETE FROM drugs.asp_noc WHERE quarter = ?", [quarter])
return _insert(con, "drugs.asp_noc", rows)
def read_noc(con: Any, quarter: str | None = None) -> list[NocRow]:
where, params = ("quarter = ?", [quarter]) if quarter is not None else ("", [])
rows = _select(con, NocRow, "drugs.asp_noc", where, params)
return sorted(
(NocRow(*r) for r in rows), key=lambda r: (r.quarter, r.drug_name, r.dosage)
)

View File

@@ -2,12 +2,20 @@
from __future__ import annotations
import io
import json
import zipfile
import duckdb
import httpx
import pytest
from typer.testing import CliRunner
import cli.drugs as dg
from bib.store import Store
from cli import app
from drugs import asp as A
from drugs import http as dh
from drugs import tables as T
from market import tables as MT
@@ -140,3 +148,376 @@ def test_status_gap_read_error_propagates(env, monkeypatch):
monkeypatch.setattr(dg, "_read", lambda: _Con())
assert isinstance(runner.invoke(app, ["drugs", "status"]).exception, RuntimeError)
# ── stack drugs asp ──────────────────────────────────────────────────
PRICING = (
"Payment Allowance Limits,,,,,,,,,,,\r\n"
"HCPCS Code,Short Description,HCPCS Code Dosage,Payment Limit,"
"Co-insurance Percentage,Vaccine AWP%,Vaccine Limit,Blood AWP%,Blood limit,"
"Clotting Factor,Notes\r\n"
"90371,x,1 ML,140.209,20.000,,,,,,\r\n"
"J9035,Bevacizumab injection,10 MG,66.123,20.000,,,,,,\r\n"
)
NP = (
"HCPCS Code,Long Description,Manufacturer\r\nJ7330,Chondrocytes implant,Vericel\r\n"
)
NOC = "Drug Generic Name (Trade Name),Dosage,Payment Limit,Notes\r\nExamplumab,10 MG,12.5,\r\n"
def _zip(members):
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
for n, b in members.items():
zf.writestr(n, b)
return buf.getvalue()
PAGE = """<html><body>
<a href="/files/zip/july-2026-medicare-part-b-payment-limit-files.zip">July 2026 Medicare Part B Payment Limit Files</a>
<a href="/files/zip/july-2026-ndc-hcpcs-crosswalk.zip">July 2026 NDC-HCPCS Crosswalk</a>
<a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/x/downloads/2018-April-ASP-Pricing-File.zip">April 2018 ASP Pricing File</a>
<a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip">April 2018 NOC Pricing File</a>
<a href="/files/zip/2026-2027-seasonal-vaccine-price-file.zip">2026-2027 Seasonal Vaccine Price File</a>
</body></html>"""
ROUTES = {
"/medicare/payment/part-b-drugs/asp-pricing-files": PAGE.encode(),
"/files/zip/july-2026-medicare-part-b-payment-limit-files.zip": _zip(
{
"section 508 version of July 2026 Payment Limit File.csv": PRICING,
"section 508 version of July 2026 Drugs Not Payable.csv": NP,
}
),
"/files/zip/july-2026-ndc-hcpcs-crosswalk.zip": _zip({"x.csv": "a"}),
"/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip": _zip({"NOC.csv": NOC}),
"/Medicare/x/downloads/2018-April-ASP-Pricing-File.zip": _zip({"ASP.csv": PRICING}),
}
class _NoWait:
def wait(self):
pass
@pytest.fixture
def asp_env(env, monkeypatch, tmp_path):
c, published = env
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
dh._MANIFESTS.clear()
store = Store(":memory:", storage_dir=tmp_path / "st")
monkeypatch.setattr(dg, "_store", lambda: store)
seen: list[httpx.Request] = []
def handler(req):
seen.append(req)
body = ROUTES.get(req.url.path)
return (
httpx.Response(200, content=body)
if body is not None
else httpx.Response(404)
)
monkeypatch.setattr(
dh,
"client",
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
)
monkeypatch.setattr(A, "today", lambda: __import__("datetime").date(2026, 8, 1))
return c, published, seen, tmp_path, store
def test_asp_cli_fetch_write(asp_env):
c, published, seen, root, store = asp_env
MT.write_gap(c, "cms-asp:asp", "stale")
res = runner.invoke(
app,
[
"drugs",
"asp",
"--fetch",
"--write",
"--quarter",
"2026-Q3",
"--quarter",
"2018-Q2",
],
)
assert res.exit_code == 0, res.output
pay = [r for r in T.read_asp(c) if r.quarter == "2026-Q3"]
assert {r.quarter for r in T.read_asp(c)} == {"2018-Q2", "2026-Q3"}
assert [(r.quarter, r.hcpcs, r.list_kind) for r in pay] == [
("2026-Q3", "90371", "payment"),
("2026-Q3", "J7330", "not-payable"),
("2026-Q3", "J9035", "payment"),
]
assert pay[0].short_desc == "" and pay[2].short_desc == "Bevacizumab injection"
assert pay[1].manufacturer == "Vericel"
assert all(
r.item_key
and r.source_url.startswith("https://www.cms.gov/files/zip/july-2026")
for r in pay
)
noc = T.read_noc(c)
assert [(n.quarter, n.drug_name, n.payment_limit) for n in noc] == [
("2018-Q2", "Examplumab", 12.5)
]
assert (
noc[0].source_url
== "https://www.cms.gov/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip"
)
assert {
k
for (k,) in c.execute(
"SELECT DISTINCT table_name FROM cms.ingest_log"
).fetchall()
} == {
"drugs.asp_payment",
"drugs.asp_noc",
}
assert published == [True]
assert [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"] == []
assert "stale" not in " ".join(g.note for g in MT.read_gaps(c))
# every request carried the honest user agent through the real client factory
hoard = root / "cms" / "asp"
assert (hoard / "asp_2026-Q3.zip").exists()
assert (hoard / "noc" / "noc_2018-Q2.zip").exists()
assert (hoard / "xwalk" / "xwalk_2026-Q3.zip").exists()
assert (root / "drugs" / "cms" / "asp-pricing-files.html").exists()
manifest = [
json.loads(x) for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
]
assert len(manifest) == 5
# vaccine zip is never fetched
assert not any("vaccine" in str(r.url) for r in seen)
assert "2026-Q3" in res.output and "not-payable" in res.output
cites = [i for i in store.list_items() if "ASP" in i.title]
assert len(cites) == 3
# a second run fetches nothing but the page
n = len(seen)
again = runner.invoke(
app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3", "--quarter", "2018-Q2"]
)
assert again.exit_code == 0, again.output
assert len(seen) == n + 1
def test_asp_cli_gap_note_written_for_missing(asp_env):
c, published, seen, root, store = asp_env
res = runner.invoke(
app,
[
"drugs",
"asp",
"--fetch",
"--write",
"--quarter",
"2026-Q3",
"--quarter",
"2026-Q2",
],
)
assert res.exit_code == 0, res.output
gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"]
assert len(gaps) == 1 and "2026-Q2 pricing" in gaps[0].note
assert "gap: cms-asp:asp: no hoarded file for: 2026-Q2 pricing" in res.output
def test_asp_cli_dry_run_lists_links(asp_env):
c, published, seen, root, store = asp_env
first = runner.invoke(app, ["drugs", "asp", "--fetch", "--dry-run"])
assert first.exit_code == 0, first.output
assert [r.url.path for r in seen] == [
"/medicare/payment/part-b-drugs/asp-pricing-files"
]
out = first.output
assert (
"pricing 2026-Q3 fetch https://www.cms.gov/files/zip/july-2026-medicare-part-b-payment-limit-files.zip"
in out
)
assert "noc 2018-Q2 fetch" in out and "crosswalk 2026-Q3 fetch" in out
assert "vaccine" not in out
assert not (root / "cms").exists()
assert T.table_counts(c) == {}
assert published == []
# --write is ignored by a dry run
res = runner.invoke(
app, ["drugs", "asp", "--dry-run", "--write", "--quarter", "2018-Q2"]
)
assert res.exit_code == 0 and published == [] and "noc 2018-Q2 fetch" in res.output
assert "crosswalk 2026-Q3" not in res.output
def test_asp_cli_refusal_prints_gap(asp_env, monkeypatch):
c, published, seen, root, store = asp_env
monkeypatch.setattr(
dh,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(lambda r: httpx.Response(403))
),
)
res = runner.invoke(app, ["drugs", "asp", "--fetch"])
assert res.exit_code == 0, res.output
lines = [x for x in res.output.splitlines() if x.strip()]
assert len(lines) == 1 and lines[0].startswith("gap: cms-asp:asp:")
assert published == []
assert T.table_counts(c) == {} and not (root / "cms").exists()
res = runner.invoke(app, ["drugs", "asp", "--fetch", "--write"])
assert [g.source for g in MT.read_gaps(c)] == ["cms-asp:asp"]
assert published == [True]
assert T.table_counts(c) == {}
def test_asp_cli_refusal_on_a_zip_stops_fetching(asp_env, monkeypatch):
c, published, seen, root, store = asp_env
def handler(req):
if req.url.path.endswith("asp-pricing-files"):
return httpx.Response(200, content=PAGE.encode())
return httpx.Response(403)
monkeypatch.setattr(
dh,
"client",
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
)
res = runner.invoke(
app, ["drugs", "asp", "--fetch", "--write", "--quarter", "2026-Q3"]
)
assert res.exit_code == 0, res.output
assert "gap: cms-asp:asp: refused" in res.output
assert "2026-Q3 pricing" in res.output
assert T.read_asp(c) == []
def test_asp_cli_parse_only_reads_hoard_without_cites(asp_env):
c, published, seen, root, store = asp_env
assert (
runner.invoke(
app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3"]
).exit_code
== 0
)
assert T.table_counts(c) == {}
seen.clear()
res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q3"])
assert res.exit_code == 0 and seen == []
assert "2026-Q3" in res.output and "3 payment" not in res.output
assert "2 payment" in res.output and "1 not-payable" in res.output
assert store.list_items() == [] and published == []
def test_asp_cli_unreadable_zip_is_a_gap(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp"
d.mkdir(parents=True)
(d / "asp_2026-Q3.zip").write_bytes(b"not a zip")
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
assert res.exit_code == 0, res.output
gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"]
assert gaps and "2026-Q3 pricing unreadable" in gaps[0].note
assert T.read_asp(c) == []
def test_asp_cli_source_url_falls_back_to_csv(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp"
d.mkdir(parents=True)
(d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING}))
(root / "cms" / "asp_source_urls.csv").write_text(
"quarter,url\n2026-Q3,https://cms.test/canon.zip\n"
)
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
assert res.exit_code == 0, res.output
assert {r.source_url for r in T.read_asp(c)} == {"https://cms.test/canon.zip"}
assert all(r.item_key for r in T.read_asp(c))
def test_asp_cli_no_url_no_cite(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp"
d.mkdir(parents=True)
(d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING}))
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
assert res.exit_code == 0, res.output
assert {(r.source_url, r.item_key) for r in T.read_asp(c)} == {("", "")}
def test_asp_cli_refresh_refetches(asp_env):
c, published, seen, root, store = asp_env
assert (
runner.invoke(
app, ["drugs", "asp", "--fetch", "--quarter", "2018-Q2"]
).exit_code
== 0
)
n = len(seen)
ROUTES_BEFORE = dict(ROUTES)
res = runner.invoke(
app, ["drugs", "asp", "--fetch", "--refresh", "--quarter", "2018-Q2"]
)
assert (
res.exit_code == 0 and len(seen) == n + 3
) # page, the pricing zip and the NOC zip
assert ROUTES == ROUTES_BEFORE
def test_asp_cli_unfiltered_uses_expected_quarters(asp_env):
c, published, seen, root, store = asp_env
res = runner.invoke(app, ["drugs", "asp"])
assert res.exit_code == 0, res.output
assert "gap: cms-asp:asp: no hoarded file for: 2005-Q1 pricing" in res.output
assert "(+" in res.output and published == []
assert seen == []
def test_asp_today_is_real_date():
import datetime
assert A.today() == datetime.date.today()
def test_asp_cli_rejects_bad_quarter(asp_env):
res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q5"])
assert res.exit_code != 0 and "not a quarter" in res.output
def test_asp_cli_unreadable_noc_zip_is_a_gap(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp" / "noc"
d.mkdir(parents=True)
(d / "noc_2018-Q2.zip").write_bytes(b"not a zip")
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2018-Q2"])
assert res.exit_code == 0, res.output
assert "2018-Q2 noc unreadable" in res.output
assert T.read_noc(c) == []
def test_asp_cli_bundled_noc_from_pricing_zip(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp"
d.mkdir(parents=True)
(d / "asp_2006-Q2.zip").write_bytes(
_zip({"ASPbyHCPCS.csv": PRICING, "ASPNOC.csv": NOC})
)
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2006-Q2"])
assert res.exit_code == 0, res.output
assert [n.drug_name for n in T.read_noc(c)] == ["Examplumab"]
assert "1 noc" in res.output
def test_asp_cli_dry_run_without_page(asp_env):
res = runner.invoke(app, ["drugs", "asp", "--dry-run"])
assert res.exit_code == 0 and "no page hoarded; run with --fetch" in res.output
def test_asp_cli_noc_only_quarter_is_listed(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp" / "noc"
d.mkdir(parents=True)
(d / "noc_2018-Q2.zip").write_bytes(_zip({"NOC.csv": NOC}))
res = runner.invoke(app, ["drugs", "asp", "--quarter", "2018-Q2"])
assert res.exit_code == 0, res.output
assert "2018-Q2: 1 noc" in res.output

View File

@@ -0,0 +1,134 @@
"""The skin-substitute ingest reads the same ASP parser as the drug tier, and
its output is unchanged (1,974 rows, 2009-Q1..2025-Q4)."""
from __future__ import annotations
import csv
import importlib.util
import math
import zipfile
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
ASP_DIR = ROOT / "data" / "cms" / "asp"
SKIN_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv"
REF_CSV = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv"
_spec = importlib.util.spec_from_file_location(
"_ingest_asp", ROOT / "dev" / "scripts" / "ingest_asp.py"
)
ingest = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(ingest)
needs_data = pytest.mark.skipif(
not (SKIN_CSV.exists() and any(ASP_DIR.glob("asp_*.zip"))),
reason="needs the hoarded data/cms/asp files",
)
def _legacy_pick(names: list[str]) -> str | None:
"""The selection the dev script made before the shared parser (frozen)."""
skip = ["noc", "crosswalk", "ndc", "not payable", "not_payable"]
cand = [
n
for n in names
if not any(s in n.lower() for s in skip)
and n.lower().endswith((".csv", ".xls", ".xlsx"))
]
csvs = [c for c in cand if c.lower().endswith(".csv")]
if csvs:
for c in csvs:
if "payment limit" in c.lower() or "pricing" in c.lower():
return c
return csvs[0]
xlss = [c for c in cand if c.lower().endswith((".xls", ".xlsx"))]
if xlss:
for c in xlss:
low = c.lower()
if "payment limit" in low or "pricing" in low or "asp" in low:
return c
return xlss[0]
return None
@needs_data
def test_pick_pricing_file_over_every_hoarded_zip():
from drugs import asp
n = 0
for z in sorted(ASP_DIR.glob("asp_*.zip")):
names = zipfile.ZipFile(z).namelist()
want = _legacy_pick(names)
assert ingest.pick_pricing_file(names) == want, z.name
assert asp.pick_pricing_file(names) == want, z.name
n += 1
assert n >= 86
@needs_data
def test_skin_rows_match_hoarded_csv():
df = ingest.build_skin_df()
got = sorted(
df.to_dict("records"),
key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
)
with SKIN_CSV.open(newline="") as fh:
want = sorted(
csv.DictReader(fh),
key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
)
assert len(want) == 1974
assert len(got) == len(want), "an extra or missing skin row appeared"
for g, w in zip(got, want, strict=True):
for col in (
"quarter",
"hcpcs_code",
"short_description",
"dosage_or_unit",
"source_file",
"source_url",
):
gv = g[col]
gv = "" if gv is None or (isinstance(gv, float) and math.isnan(gv)) else gv
assert gv == w[col], (g["quarter"], g["hcpcs_code"], col)
gp = g["payment_limit"]
if w["payment_limit"] == "":
assert gp is None or math.isnan(gp)
else:
assert abs(gp - float(w["payment_limit"])) < 1e-9
assert abs(g["asp_per_unit"] - float(w["asp_per_unit"])) < 1e-9
def test_reference_csv_carries_no_cpt_descriptor_text():
with REF_CSV.open(newline="") as fh:
rows = list(csv.DictReader(fh))
assert len(rows) == 34
assert list(rows[0]) == [
"hcpcs_code",
"short_descriptor",
"code_type",
"body_site",
"wound_size",
"effective_date",
"termination_date",
"status",
"notes",
]
cpt = [r for r in rows if "CPT" in r["code_type"]]
assert {r["hcpcs_code"] for r in cpt} >= {"15271", "15278", "15330", "15430"}
assert all(r["short_descriptor"] == "" for r in cpt)
# CMS's own HCPCS Level II text stays
level2 = {r["hcpcs_code"]: r["short_descriptor"] for r in rows if r not in cpt}
assert level2["C5271"].startswith("Low cost skin substitute graft")
assert level2["G0440"] and level2["C9363"]
assert all(r["short_descriptor"] for r in rows if r not in cpt)
def test_refuses_to_replace_table_with_nothing():
import pandas as pd
with pytest.raises(RuntimeError, match="no skin-substitute rows"):
ingest.require_rows(pd.DataFrame())
ingest.require_rows(pd.DataFrame({"a": [1]}))

View File

@@ -0,0 +1,17 @@
<!doctype html>
<html><head><title>ASP Pricing Files</title></head>
<body>
<nav><a href="/medicare/payment/part-b-drugs/asp-reporting">ASP Reporting</a></nav>
<main>
<ul>
<li><a href="/files/zip/october-2026-medicare-part-b-payment-limit-files-final.zip">October 2026 Medicare Part B Payment Limit Files</a></li>
<li><a href="/files/zip/october-2026-ndc-hcpcs-crosswalk-final.zip">October 2026 NDC-HCPCS Crosswalk</a></li>
<li><a href="/files/zip/2026-2027-seasonal-vaccine-price-file.zip">2026-2027 Seasonal Vaccine Price File</a></li>
<li><a href="/files/zip/2026-2027-ndc-hcpcs-crosswalk-seasonal-vaccines.zip">2026-2027 NDC-HCPCS Crosswalk for Seasonal Vaccines</a></li>
<li><a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/McrPartBDrugAvgSalesPrice/Downloads/2018-January-ASP-Pricing-File.zip">January 2018 ASP Pricing File&nbsp;- Updated 03/02/2018</a></li>
<li><a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/McrPartBDrugAvgSalesPrice/Downloads/2018-April-NOC-Pricing-File.zip">April 2018 NOC Pricing File - Updated 05/02/2018</a></li>
<li><a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/McrPartBDrugAvgSalesPrice/downloads/jan05aspbyhcpcsv6_033106.zip">Jan 2005 ASP Pricing File and ASP NOC Pricing File - Updated 03/31/2006</a></li>
<li><a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/McrPartBDrugAvgSalesPrice/downloads/HCPCSCrosswalk0105.zip">Jan 2005 ASP NDC-HCPCS Crosswalk</a></li>
</ul>
</main>
</body></html>

694
tests/drugs/test_asp.py Normal file
View File

@@ -0,0 +1,694 @@
import json
import os
import zipfile
from datetime import date
from pathlib import Path
import duckdb
import httpx
import pytest
from drugs import asp
from drugs import http as dh
from drugs import tables as T
FIXTURE = Path(__file__).parent / "fixtures" / "asp_page_head.html"
class _NoWait:
def wait(self):
pass
@pytest.fixture
def root(monkeypatch, tmp_path):
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
dh._MANIFESTS.clear()
return tmp_path
# ── the page ─────────────────────────────────────────────────────────
def test_parse_asp_page_kinds_and_quarters():
links = asp.parse_asp_page(FIXTURE.read_text())
got = {(lk.quarter, lk.kind): lk.url for lk in links}
assert got[("2026-Q4", "pricing")] == (
"https://www.cms.gov/files/zip/"
"october-2026-medicare-part-b-payment-limit-files-final.zip"
)
assert got[("2026-Q4", "crosswalk")] == (
"https://www.cms.gov/files/zip/october-2026-ndc-hcpcs-crosswalk-final.zip"
)
vaccine = [lk for lk in links if lk.kind == "vaccine"]
assert len(vaccine) == 2 and {lk.quarter for lk in vaccine} == {""}
assert ("2018-Q2", "noc") in got and ("2018-Q1", "pricing") in got
assert got[("2005-Q1", "pricing")] == (
"https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/"
"McrPartBDrugAvgSalesPrice/downloads/jan05aspbyhcpcsv6_033106.zip"
)
assert ("2005-Q1", "crosswalk") in got
assert len(links) == 8
assert links[0].text == "October 2026 Medicare Part B Payment Limit Files"
def test_parse_asp_page_edge_cases():
html = """
<a href="/files/zip/january-2023-asp-pricing-file.zip">2023 ASP Pricing File (ZIP)</a>
<a href="/files/zip/april-2023-noc-pricing-file.zip">2023 NOC&nbsp;Pricing File</a>
<a href="/x/2018-July-ASP-Pricing-File.zip">ASP Pricing File</a>
<a href="/files/zip/jan06.zip">January 2006 ASP Pricing File/ASP NOC Pricing File/ASP NDC-HCPCS Crosswalk</a>
<a href="/files/zip/jul06.zip">July 2006 ASP NDC-HCPCS Crosswalk - Updated</a>
<a href="/files/zip/oct09.zip">October 2009 ASP NOC Pricing File</a>
<a href="/files/zip/mystery.zip">Something else entirely</a>
<a href="/files/zip/no-month.zip">ASP Pricing File</a>
<a href="/files/pdf/guide.pdf">Guide to the files (PDF)</a>
<a>no href at all</a>
<a href="/files/zip/oct05.zip">Oct. 2005 ASP Pricing File and ASP NOC Pricing File</a>
"""
links = asp.parse_asp_page(html)
got = [(lk.quarter, lk.kind) for lk in links]
assert got == [
("2023-Q1", "pricing"),
("2023-Q2", "noc"),
("2018-Q3", "pricing"),
("2006-Q1", "pricing"),
("2006-Q3", "crosswalk"),
("2009-Q4", "noc"),
("", "other"),
("", "other"),
("2005-Q4", "pricing"),
]
assert links[1].text == "2023 NOC Pricing File"
def test_direct_url_unwraps_license():
p = "/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/x/downloads/a.zip"
want = "https://www.cms.gov" + p
assert asp.direct_url(f"https://www.cms.gov/apps/ama/license.asp?file={p}") == want
assert asp.direct_url(f"/apps/ama/license.asp?file={p}") == want
assert asp.direct_url(f"https://www.cms.gov/license/ama?file={p}") == want
assert asp.direct_url("/files/zip/a.zip") == "https://www.cms.gov/files/zip/a.zip"
assert asp.direct_url("https://x.test/a.zip") == "https://x.test/a.zip"
def test_hoard_path_and_newest_per_quarter(root):
def lk(kind):
return asp.AspLink("2026-Q2", kind, "u", "t")
assert asp.hoard_path(lk("pricing")) == root / "cms/asp/asp_2026-Q2.zip"
assert asp.hoard_path(lk("noc")) == root / "cms/asp/noc/noc_2026-Q2.zip"
assert asp.hoard_path(lk("crosswalk")) == root / "cms/asp/xwalk/xwalk_2026-Q2.zip"
with pytest.raises(ValueError):
asp.hoard_path(lk("vaccine"))
def test_one_zip_per_quarter_newest_wins(tmp_path):
a = tmp_path / "asp_2026-Q2.zip"
b = tmp_path / "asp_2026-Q2.ab12cd34.zip"
c = tmp_path / "asp_2026-Q1.zip"
junk = tmp_path / "asp_notes.zip"
for p in (a, b, c, junk):
p.write_bytes(b"x")
os.utime(a, (1_000, 1_000))
os.utime(b, (2_000, 2_000))
got = asp.newest_per_quarter([a, b, c, junk])
assert got == {"2026-Q1": c, "2026-Q2": b}
# the older copy still wins when it is the newer file
os.utime(a, (3_000, 3_000))
assert asp.newest_per_quarter([a, b])["2026-Q2"] == a
assert asp.newest_per_quarter([]) == {}
def test_expected_quarters():
q = asp.expected_quarters(date(2026, 11, 13))
assert q[0] == "2005-Q1" and q[-1] == "2026-Q4" and len(q) == 88
assert asp.expected_quarters(date(2005, 1, 1)) == ["2005-Q1"]
assert asp.expected_quarters(date(2005, 3, 31)) == ["2005-Q1"]
assert asp.expected_quarters(date(2006, 2, 1)) == [
"2005-Q1",
"2005-Q2",
"2005-Q3",
"2005-Q4",
"2006-Q1",
]
# ── the member logic ─────────────────────────────────────────────────
def test_col_map_covers_dev_script_and_new_columns():
dev = {
"hcpcs code": "hcpcs_code",
"hcpcs_code": "hcpcs_code",
"short description": "short_description",
"short_description": "short_description",
"hcpcs code dosage": "dosage_or_unit",
"hcpcs_code_dosage": "dosage_or_unit",
"dosage": "dosage_or_unit",
"dosage or unit": "dosage_or_unit",
"payment limit": "payment_limit",
"payment_limit": "payment_limit",
}
for k, v in dev.items():
assert asp.COL_MAP[k] == v
assert asp.COL_MAP["co-insurance percentage"] == "coinsurance_pct"
assert asp.COL_MAP["notes"] == "notes"
assert asp.COL_MAP["long description"] == "short_description"
assert asp.COL_MAP["manufacturer"] == "manufacturer"
assert asp.COL_MAP["drug generic name (trade name)"] == "drug_name"
def test_find_header_row():
assert asp.find_header_row(["a", "Note: HCPCS code x,y", "HCPCS Code,Foo"]) == 2
assert asp.find_header_row(['"HCPCS Code","Foo"']) == 0
assert asp.find_header_row(["", "Drug Generic Name (Trade Name),Dosage"]) == 1
assert asp.find_header_row(["Drug Name (Generic),Dosage"]) == 0
assert asp.find_header_row(["nothing here"]) == -1
assert asp.find_header_row([]) == -1
NAMES_2015Q3 = [
"Jul 15 ASP Pricing File - updated 090115.xls",
"section 508 version of Jul 15 ASP Pricing File - updated 090115.csv",
]
NAMES_2026Q2 = [
"April 2026 Drugs Not Payable Under Part B 030926.xls",
"April 2026 Medicare Part B Payment Limit File 032426.xls",
"section 508 version of April 2026 Drugs Not Payable Under Part B 030926.csv",
"section 508 version of April 2026 Medicare Part B Payment Limit File 032426.csv",
]
def test_pick_pricing_file_matches_dev_script():
assert asp.pick_pricing_file(["Jan05ASPbyHCPCSv6_033106.xls"]) == (
"Jan05ASPbyHCPCSv6_033106.xls"
)
assert asp.pick_pricing_file(NAMES_2015Q3) == NAMES_2015Q3[1]
assert asp.pick_pricing_file(
[
"January 2026 Medicare Part B Payment Limit File updated 022526.xls",
"section 508 version of January 2026 Medicare Part B Payment Limit "
"File updated 022526.csv",
]
).endswith("022526.csv")
assert asp.pick_pricing_file(NAMES_2026Q2) == NAMES_2026Q2[3]
assert asp.pick_pricing_file(["a.xls", "b.pdf"]) == "a.xls"
assert asp.pick_pricing_file(["ASP.xlsx", "other.xls"]) == "ASP.xlsx"
assert (
asp.pick_pricing_file(["other.csv", "payment limit.csv"]) == "payment limit.csv"
)
assert asp.pick_pricing_file(["x.csv"]) == "x.csv"
assert (
asp.pick_pricing_file(["Oct 2025 NOC Pricing File.csv", "readme.txt"]) is None
)
assert asp.pick_pricing_file(["Crosswalk.csv", "ndc.xls"]) is None
def test_pick_member_prefers_csv_over_its_xls_twin():
# P7: one member per kind per zip
assert asp.pick_member(NAMES_2026Q2, "not-payable") == NAMES_2026Q2[2]
assert asp.pick_member(NAMES_2026Q2, "noc") is None
names = [
"1 - Apr06_ASPbyHCPCS_122107.xls",
"1 - Apr06_ASPbyHCPCS_122107.csv",
"1 - Apr06_ASPNOC_20Sep06.xls",
"1 - Apr06_ASPNOC_20Sep06.csv",
"April06ASPNDC-HCPCSCrosswalk_20Mar06.xls",
]
assert asp.pick_member(names, "noc") == "1 - Apr06_ASPNOC_20Sep06.csv"
assert asp.pick_member(names[2:3] + names[4:], "noc") == names[2]
assert asp.pick_member(["Drugs_Not_Payable.xlsx"], "not-payable") == (
"Drugs_Not_Payable.xlsx"
)
assert asp.pick_member(["x.csv"], "noc") is None
with pytest.raises(ValueError):
asp.pick_member(names, "crosswalk")
# ── reading tables ───────────────────────────────────────────────────
PRICING_HEAD = (
"Payment Allowance Limits for Medicare Part B Drugs,,,,,,,,,,,\r\n"
",,,,,,,,,,,\r\n"
"Note 1: HCPCS code lines mention HCPCS code here,,,,,,,,,,,\r\n"
"HCPCS Code,Short Description,HCPCS Code Dosage,Payment Limit,"
"Co-insurance Percentage,Vaccine AWP%,Vaccine Limit,Blood AWP%,Blood limit,"
"Clotting Factor,Notes\r\n"
)
PRICING_ROWS = (
"90371,x,1 ML,140.209,20.000,,,,,,\r\n"
'J9035,Bevacizumab injection,10 MG,"$66.123",20.000,,,,,,\r\n'
)
NP_CSV = (
"Drugs Not Payable Under Part B,,,\r\n"
",,,\r\n"
"HCPCS Code,Long Description,Manufacturer,,,,\r\n"
'J7330," Autologous cultured chondrocytes, implant ",Vericel Corporation,,,,\r\n'
"90999,x,Acme,,,,\r\n"
",,,,,,\r\n"
)
NOC_CSV = (
"Payment Allowance Limits for NOC Drugs,,,\r\n"
",,,\r\n"
"Drug Generic Name (Trade Name),Dosage,Payment Limit,Notes\r\n"
"Examplumab,10 MG,12.5,\r\n"
"Vasopressin (Long Grove),1 UNIT,$2.462,\r\n"
",,,\r\n"
)
def make_zip(path: Path, members: dict[str, str | bytes]) -> Path:
with zipfile.ZipFile(path, "w") as zf:
for name, body in members.items():
zf.writestr(
name, body if isinstance(body, bytes) else body.encode("latin-1")
)
return path
def test_cpt_short_description_never_stored(tmp_path):
z = make_zip(
tmp_path / "asp_2026-Q2.zip",
{
"section 508 version of April 2026 Payment Limit File.csv": PRICING_HEAD
+ PRICING_ROWS
},
)
rows, noc = asp.parse_quarter_zip(
z, "2026-Q2", source_url="https://cms.test/a.zip", item_key="K1"
)
assert noc == []
by = {r.hcpcs: r for r in rows}
assert by["90371"].level == "I" and by["90371"].short_desc == ""
assert by["90371"].dosage == "1 ML" and by["90371"].payment_limit == 140.209
assert by["90371"].coinsurance_pct == 20.0
j = by["J9035"]
assert j.level == "II" and j.short_desc == "Bevacizumab injection"
assert j.payment_limit == 66.123 and j.list_kind == "payment"
assert j.quarter == "2026-Q2" and j.item_key == "K1"
assert j.source_url == "https://cms.test/a.zip"
assert j.source_file == "section 508 version of April 2026 Payment Limit File.csv"
assert "Bevacizumab" not in " ".join(r.short_desc for r in rows if r.level == "I")
def test_numbers_codes_and_blank_rows(tmp_path):
body = (
PRICING_HEAD
+ ' j9035 ,Bevacizumab,10 MG,"$1,066.50",,,,,,,see note\r\n'
+ "J0000,Unparseable,1 MG,n/a,abc,,,,,,\r\n"
+ "J0001,Short row\r\n"
+ ",,,,,,,,,,\r\n"
+ " ,blank code,,,,,,,,,\r\n"
+ "J0002,Too many,1 MG,1.0,20,,,,,,,extra,fields\r\n"
+ "D0120,dental,1,2.0,,,,,,,\r\n"
)
z = make_zip(tmp_path / "asp_2020-Q1.zip", {"ASP Pricing File.csv": body})
rows, _ = asp.parse_quarter_zip(z, "2020-Q1", source_url="")
by = {r.hcpcs: r for r in rows}
# a line with more fields than columns is malformed and skipped
assert set(by) == {"J9035", "J0000", "J0001", "D0120"}
assert by["J9035"].payment_limit == 1066.5 and by["J9035"].notes == "see note"
assert by["J0000"].payment_limit is None and by["J0000"].coinsurance_pct is None
assert by["J0001"].dosage == "" and by["J0001"].short_desc == "Short row"
assert by["D0120"].level == "CDT" and by["D0120"].short_desc == ""
assert by["J9035"].item_key == "" and by["J9035"].source_url == ""
def test_not_payable_rows_kept(tmp_path):
z = make_zip(
tmp_path / "asp_2026-Q2.zip",
{
"April 2026 Drugs Not Payable Under Part B 030926.xls": b"not read",
"section 508 version of April 2026 Drugs Not Payable Under Part B 030926.csv": NP_CSV,
"section 508 version of April 2026 Payment Limit File.csv": PRICING_HEAD
+ PRICING_ROWS,
},
)
rows, _ = asp.parse_quarter_zip(z, "2026-Q2", source_url="u")
np_rows = [r for r in rows if r.list_kind == "not-payable"]
assert len(np_rows) == 2 and len(rows) == 4
j = next(r for r in np_rows if r.hcpcs == "J7330")
assert j.payment_limit is None and j.coinsurance_pct is None
assert j.manufacturer == "Vericel Corporation"
assert j.short_desc == " Autologous cultured chondrocytes, implant "
assert j.source_file.endswith("Under Part B 030926.csv")
lvl1 = next(r for r in np_rows if r.hcpcs == "90999")
assert lvl1.level == "I" and lvl1.short_desc == "" and lvl1.manufacturer == "Acme"
assert all(r.manufacturer == "" for r in rows if r.list_kind == "payment")
def test_noc_member_parsed(tmp_path):
z = make_zip(
tmp_path / "noc_2025-Q4.zip",
{
"Oct 2025 NOC Pricing File 090425.xls": b"not read",
"section 508 version of Oct 2025 NOC Pricing File 090425.csv": NOC_CSV,
},
)
rows, noc = asp.parse_quarter_zip(
z, "2025-Q4", source_url="https://cms.test/n.zip", item_key="K2"
)
assert rows == []
assert [(n.drug_name, n.dosage, n.payment_limit) for n in noc] == [
("Examplumab", "10 MG", 12.5),
("Vasopressin (Long Grove)", "1 UNIT", 2.462),
]
assert noc[0].quarter == "2025-Q4" and noc[0].item_key == "K2"
assert noc[0].source_url == "https://cms.test/n.zip"
assert noc[0].source_file.endswith("090425.csv")
def test_bundled_zip_gives_both_and_one_member_per_kind(tmp_path):
# 2006-style zip: pricing, NOC and crosswalk, each as .xls and .csv twins
z = make_zip(
tmp_path / "asp_2006-Q2.zip",
{
"1 - Apr06_ASPbyHCPCS.xls": b"x",
"1 - Apr06_ASPbyHCPCS.csv": PRICING_HEAD + PRICING_ROWS,
"1 - Apr06_ASPNOC.xls": b"x",
"1 - Apr06_ASPNOC.csv": NOC_CSV,
"April06ASPNDC-HCPCSCrosswalk.csv": "HCPCS Code,NDC\r\nJ9035,1\r\n",
},
)
rows, noc = asp.parse_quarter_zip(z, "2006-Q2", source_url="")
assert [r.hcpcs for r in rows] == ["90371", "J9035"]
assert len(noc) == 2
def test_header_missing_gives_no_rows(tmp_path):
z = make_zip(
tmp_path / "asp_2020-Q1.zip", {"ASP Pricing File.csv": "a,b\r\n1,2\r\n"}
)
assert asp.parse_quarter_zip(z, "2020-Q1", source_url="") == ([], [])
z2 = make_zip(tmp_path / "asp_2020-Q2.zip", {"readme.txt": "hi"})
assert asp.parse_quarter_zip(z2, "2020-Q2", source_url="") == ([], [])
class _Sheet:
def __init__(self, grid):
self.grid = grid
self.nrows = len(grid)
self.ncols = max(len(r) for r in grid)
def cell_value(self, r, c):
row = self.grid[r]
return row[c] if c < len(row) else ""
class _Book:
def __init__(self, grid, sheets=None):
self.sheets = [("Sheet1", _Sheet(grid))] + [
(name, _Sheet(g)) for name, g in (sheets or [])
]
def sheet_names(self):
return [n for n, _ in self.sheets]
def sheet_by_index(self, i):
return self.sheets[i][1]
XLS_GRID = [
["Payment Allowance Limits"],
[""],
["HCPCS Code", "Short Description", "HCPCS Code Dosage", "Payment Limit"],
[90371.0, "x", "1 ML", 140.209],
["J9035 ", "Bevacizumab injection", "10 MG", 66.123],
["", "", "", ""],
["J0001", "Whole", "1 MG", 5.0],
]
def test_xls_member_through_fake_workbook(tmp_path):
z = make_zip(
tmp_path / "asp_2005-Q1.zip", {"Jan05ASPbyHCPCSv6_033106.xls": b"BIFF"}
)
seen = {}
def open_wb(**kw):
seen.update(kw)
return _Book(XLS_GRID)
rows, noc = asp.parse_quarter_zip(
z, "2005-Q1", source_url="u", open_workbook=open_wb
)
assert seen == {"file_contents": b"BIFF"}
assert noc == []
assert [
(r.hcpcs, r.level, r.short_desc, r.dosage, r.payment_limit) for r in rows
] == [
("90371", "I", "", "1 ML", 140.209),
("J9035", "II", "Bevacizumab injection", "10 MG", 66.123),
("J0001", "II", "Whole", "1 MG", 5.0),
]
csv_z = make_zip(
tmp_path / "asp_2005-Q2.zip",
{"ASP.csv": PRICING_HEAD + PRICING_ROWS},
)
csv_rows, _ = asp.parse_quarter_zip(csv_z, "2005-Q2", source_url="u")
assert [(r.hcpcs, r.dosage, r.payment_limit) for r in csv_rows] == [
(r.hcpcs, r.dosage, r.payment_limit) for r in rows[:2]
]
def test_xls_without_header_and_default_workbook(tmp_path, monkeypatch):
z = make_zip(tmp_path / "asp_2005-Q1.zip", {"ASP.xls": b"BIFF"})
assert asp.parse_quarter_zip(
z, "2005-Q1", source_url="", open_workbook=lambda **kw: _Book([["a"], ["b"]])
) == ([], [])
import sys
import types
fake = types.SimpleNamespace(open_workbook=lambda **kw: _Book(XLS_GRID))
monkeypatch.setitem(sys.modules, "xlrd", fake)
rows, _ = asp.parse_quarter_zip(z, "2005-Q1", source_url="")
assert len(rows) == 3
def test_noc_xls_member(tmp_path):
grid = [
["Drug Generic Name (Trade Name)", "Dosage", "Payment Limit"],
["Abatacept", "250 MG", 477.0],
["", "", ""],
]
z = make_zip(tmp_path / "noc_2008-Q1.zip", {"Jan08 NOC.xls": b"BIFF"})
_, noc = asp.parse_quarter_zip(
z, "2008-Q1", source_url="", open_workbook=lambda **kw: _Book(grid)
)
assert [(n.drug_name, n.dosage, n.payment_limit) for n in noc] == [
("Abatacept", "250 MG", 477.0)
]
# ── manifest and gaps ────────────────────────────────────────────────
def test_manifest_urls_and_known_urls(root):
assert asp.manifest_urls() == set()
d = root / "cms" / "asp"
d.mkdir(parents=True)
f = d / "asp_2026-Q1.zip"
f.write_bytes(b"z")
dh.manifest(f, "https://cms.test/q1.zip")
assert asp.manifest_urls() == {"https://cms.test/q1.zip"}
(root / "cms" / "asp_source_urls.csv").write_text(
"quarter,url\n2026-Q2,https://cms.test/q2.zip\n"
)
assert asp.canonical_urls() == {"2026-Q2": "https://cms.test/q2.zip"}
def test_plan_fetch_actions(root):
d = root / "cms" / "asp"
d.mkdir(parents=True)
(d / "asp_2026-Q1.zip").write_bytes(b"q1")
(d / "asp_2026-Q2.zip").write_bytes(b"q2")
(d / "asp_2025-Q4.zip").write_bytes(b"q4")
dh.manifest(d / "asp_2025-Q4.zip", "https://cms.test/2025q4.zip")
(root / "cms" / "asp_source_urls.csv").write_text(
"quarter,url\n2026-Q2,https://cms.test/q2-a.zip\n2026-Q1,https://cms.test/q1.zip\n"
)
def lk(q, kind, url):
return asp.AspLink(q, kind, url, "t")
links = [
lk("2025-Q4", "pricing", "https://cms.test/2025q4.zip"), # in manifest
lk("2026-Q1", "pricing", "https://cms.test/q1.zip"), # canonical, adopt
lk("2026-Q2", "pricing", "https://cms.test/q2-b.zip"), # updated posting
lk("2026-Q3", "pricing", "https://cms.test/q3.zip"), # new
lk("2026-Q3", "noc", "https://cms.test/n3.zip"),
lk("", "vaccine", "https://cms.test/v.zip"),
]
plan = asp.plan_fetch(links, refresh=False)
assert [(p.link.quarter, p.link.kind, p.action) for p in plan] == [
("2025-Q4", "pricing", "have"),
("2026-Q1", "pricing", "adopt"),
("2026-Q2", "pricing", "fetch-beside"),
("2026-Q3", "pricing", "fetch"),
("2026-Q3", "noc", "fetch"),
]
assert plan[3].dest == d / "asp_2026-Q3.zip"
forced = asp.plan_fetch(links, refresh=True)
assert {p.action for p in forced[:3]} == {"fetch-beside"}
assert forced[3].action == "fetch"
def test_gap_pairs_and_note():
have = {"pricing": {"2026-Q1", "2026-Q3"}, "noc": set(), "crosswalk": set()}
listed = {("2026-Q3", "noc"), ("2026-Q3", "crosswalk")}
exp = ["2026-Q1", "2026-Q2", "2026-Q3"]
pairs = asp.gap_pairs(exp, have, listed)
assert pairs == ["2026-Q2 pricing", "2026-Q3 crosswalk", "2026-Q3 noc"]
assert asp.gap_note(pairs).startswith("no hoarded file for: 2026-Q2 pricing")
assert asp.gap_note([]) == ""
many = [f"{2000 + i}-Q1 pricing" for i in range(40)]
note = asp.gap_note(many)
assert note.endswith("(+28 more)") and "2011-Q1 pricing" in note
def test_missing_quarter_is_gap_not_zero(tmp_path):
z1 = make_zip(
tmp_path / "asp_2026-Q1.zip", {"ASP.csv": PRICING_HEAD + PRICING_ROWS}
)
z3 = make_zip(
tmp_path / "asp_2026-Q3.zip", {"ASP.csv": PRICING_HEAD + PRICING_ROWS}
)
got = asp.newest_per_quarter([z1, z3])
have = {"pricing": set(got), "noc": set(), "crosswalk": set()}
pairs = asp.gap_pairs(["2026-Q1", "2026-Q2", "2026-Q3"], have, set())
assert "2026-Q2 pricing" in asp.gap_note(pairs)
con = duckdb.connect(":memory:")
for q, z in (("2026-Q1", z1), ("2026-Q3", z3)):
rows, _ = asp.parse_quarter_zip(z, q, source_url="")
asp.write_asp(con, q, rows)
assert asp.read_asp(con, quarter="2026-Q2") == []
assert len(asp.read_asp(con, quarter="2026-Q1")) == 2
# ── a transport-level updated posting lands beside ──────────────────
def test_updated_posting_lands_beside(root):
d = root / "cms" / "asp"
d.mkdir(parents=True)
old = d / "asp_2026-Q2.zip"
old.write_bytes(b"old posting")
dh.manifest(old, "https://cms.test/A.zip")
link = asp.AspLink("2026-Q2", "pricing", "https://cms.test/B.zip", "t")
(plan,) = asp.plan_fetch([link], refresh=False)
assert plan.action == "fetch-beside"
seen = []
def handler(req):
seen.append(str(req.url))
return httpx.Response(200, content=b"new posting")
cl = httpx.Client(transport=httpx.MockTransport(handler))
out = asp.apply_plan(cl, plan)
assert seen == ["https://cms.test/B.zip"]
assert out != old and out.read_bytes() == b"new posting"
assert old.read_bytes() == b"old posting"
urls = [
json.loads(x)["url"]
for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
]
assert urls == ["https://cms.test/A.zip", "https://cms.test/B.zip"]
# it is now the newest copy of the quarter
assert asp.newest_per_quarter(d.glob("asp_*.zip")) == {"2026-Q2": out}
# a second plan sees B in the manifest
(again,) = asp.plan_fetch([link], refresh=False)
assert again.action == "have"
def test_apply_plan_adopt_and_have(root):
d = root / "cms" / "asp"
d.mkdir(parents=True)
f = d / "asp_2026-Q1.zip"
f.write_bytes(b"q1")
(root / "cms" / "asp_source_urls.csv").write_text(
"quarter,url\n2026-Q1,https://cms.test/q1.zip\n"
)
link = asp.AspLink("2026-Q1", "pricing", "https://cms.test/q1.zip", "t")
(plan,) = asp.plan_fetch([link], refresh=False)
assert plan.action == "adopt"
boom = httpx.Client(transport=httpx.MockTransport(lambda r: pytest.fail("fetched")))
assert asp.apply_plan(boom, plan) == f
rec = json.loads((root / "drugs/MANIFEST.jsonl").read_text())
assert (
rec["url"] == "https://cms.test/q1.zip" and "asp_source_urls.csv" in rec["note"]
)
(plan2,) = asp.plan_fetch([link], refresh=False)
assert plan2.action == "have" and asp.apply_plan(boom, plan2) == f
def test_page_hoard(root):
seen = []
def handler(req):
seen.append(req.headers["user-agent"])
return httpx.Response(200, content=b"<html>one</html>")
cl = httpx.Client(
headers={"User-Agent": "ua"}, transport=httpx.MockTransport(handler)
)
p = asp.fetch_page(cl)
assert (
p == root / "drugs/cms/asp-pricing-files.html"
and p.read_bytes() == b"<html>one</html>"
)
assert asp.hoarded_page() == p
cl2 = httpx.Client(
transport=httpx.MockTransport(
lambda r: httpx.Response(200, content=b"<html>two</html>")
)
)
p2 = asp.fetch_page(cl2)
assert p2 != p and p.read_bytes() == b"<html>one</html>"
assert asp.hoarded_page() == p2
assert (
asp.ASP_PAGE
== "https://www.cms.gov/medicare/payment/part-b-drugs/asp-pricing-files"
)
def test_hoarded_page_absent(root):
assert asp.hoarded_page() is None
# ── tables ───────────────────────────────────────────────────────────
def test_write_noc_replaces_quarter():
con = duckdb.connect(":memory:")
a = T.NocRow("2025-Q4", "A", "1 MG", 1.0, "f", "u", "k")
b = T.NocRow("2025-Q3", "B", "1 MG", None, "f", "u", "k")
asp.write_noc(con, "2025-Q4", [a])
asp.write_noc(con, "2025-Q3", [b])
asp.write_noc(con, "2025-Q4", [T.NocRow("2025-Q4", "C", "", 2.0, "f", "u", "")])
assert [r.drug_name for r in asp.read_noc(con)] == ["B", "C"]
assert [r.drug_name for r in asp.read_noc(con, "2025-Q3")] == ["B"]
def test_2005_style_noc_sheet_inside_the_pricing_workbook(tmp_path):
noc_grid = [
["Note"],
["Drug Name (Generic)", "Dosage", "Payment Limit", "Notes"],
["Allopurinol Sodium", "500 MG", "455.714", "Revised"],
["Amikacin Sulfate", "50 MG", "$0.492", ""],
]
z = make_zip(tmp_path / "asp_2005-Q1.zip", {"Jan05ASPbyHCPCSv6.xls": b"BIFF"})
rows, noc = asp.parse_quarter_zip(
z,
"2005-Q1",
source_url="u",
open_workbook=lambda **kw: _Book(XLS_GRID, [("Jan05_NOC_rev", noc_grid)]),
)
assert len(rows) == 3
assert [(n.drug_name, n.payment_limit) for n in noc] == [
("Allopurinol Sodium", 455.714),
("Amikacin Sulfate", 0.492),
]
assert noc[0].source_file == "Jan05ASPbyHCPCSv6.xls#NOC"
# no NOC sheet: no NOC rows
_, none = asp.parse_quarter_zip(
z, "2005-Q1", source_url="u", open_workbook=lambda **kw: _Book(XLS_GRID)
)
assert none == []
def test_manufacturer_is_stripped(tmp_path):
body = "HCPCS Code,Long Description,Manufacturer\r\nJ0209,Some drug,\tHope Pharmaceuticals \r\n"
z = make_zip(tmp_path / "asp_2026-Q4.zip", {"Drugs Not Payable.csv": body})
(row,), _ = asp.parse_quarter_zip(z, "2026-Q4", source_url="")
assert row.manufacturer == "Hope Pharmaceuticals" and row.list_kind == "not-payable"

View File

@@ -1,5 +1,7 @@
from datetime import date
import pytest
from drugs import codes
@@ -34,7 +36,8 @@ def test_appl_id():
assert codes.appl_id("A", "90123") == "ANDA090123"
assert codes.appl_id("BLA", "125057") == "BLA125057"
assert codes.appl_id("NDA021436", "") == "NDA021436"
assert codes.appl_id("X", "5") == "X000005"
assert codes.appl_id("NDA", "21436") == "NDA021436"
assert codes.appl_id("ANDA", "90123") == "ANDA090123"
def test_expand():
@@ -54,3 +57,31 @@ def test_quarters_and_fy():
assert codes.quarter_end("2026-Q1") == date(2026, 3, 31)
assert codes.fy_effective("2026") == date(2025, 10, 1)
assert codes.fy_effective("2026-04") == date(2026, 4, 1)
def test_level_and_ndc_reject_trailing_newline_and_fullwidth_digits():
assert codes.level("J9035\n") == ""
assert codes.level("90371") == ""
assert codes.level("90371") == ""
assert codes.licensed("90371\n") is False
assert codes.ndc11("000214338011") is None
assert codes.ndc11("00021433801") is None
assert codes.ndc11("0002-1433-80") is None
assert codes.ndc11("0002-1433-80") == "00002143380"
def test_appl_id_rejects_unknown_kinds():
with pytest.raises(ValueError, match="unknown application kind"):
codes.appl_id("X", "5")
with pytest.raises(ValueError):
codes.appl_id("", "5")
def test_expand_caps_range_size():
assert len(codes.expand("J0001", "J0500")) == 500
assert codes.MAX_RANGE == 500
with pytest.raises(ValueError, match="range too large"):
codes.expand("J0001", "J9999")
with pytest.raises(ValueError):
codes.expand("J0001", "J0501")
assert codes.expand("J9001", "J9002") == ["J9001"]

View File

@@ -5,7 +5,7 @@ from __future__ import annotations
import subprocess
import sys
MODULES = ["drugs.http", "drugs.codes", "drugs.tables"]
MODULES = ["drugs.http", "drugs.codes", "drugs.tables", "drugs.asp"]
_PROBE = """
import builtins, importlib, sys

View File

@@ -1,4 +1,5 @@
import duckdb
import pytest
from drugs import tables as T
@@ -16,7 +17,7 @@ def test_table_names_empty():
con = duckdb.connect(":memory:")
assert T.table_counts(con) == {}
T.ensure_tables(con)
assert T.table_counts(con) == {}
assert T.table_counts(con) == {"drugs.asp_noc": 0, "drugs.asp_payment": 0}
def test_helpers():
@@ -37,3 +38,92 @@ def test_helpers():
1,
]
con.execute("CREATE TABLE drugs.x (a INTEGER)") if False else None
ROUNDTRIP = {
"asp_payment": (
T.write_asp,
T.read_asp,
[
T.AspRow(
"2026-Q2",
"J9035",
"II",
"payment",
"10 MG",
66.123,
20.0,
"Bevacizumab injection",
"n",
"f.csv",
"https://x.test/a.zip",
"K1",
),
T.AspRow(
"2026-Q2",
"J7330",
"II",
"not-payable",
"",
None,
None,
"Autologous cultured chondrocytes, implant",
"",
"np.csv",
"https://x.test/a.zip",
"K1",
"Vericel Corporation",
),
],
),
"asp_noc": (
T.write_noc,
T.read_noc,
[
T.NocRow(
"2026-Q2",
"Examplumab",
"10 MG",
12.5,
"n.csv",
"https://x.test/n.zip",
"K2",
),
T.NocRow("2026-Q2", "Other", "", None, "n.csv", "https://x.test/n.zip", ""),
],
),
}
@pytest.mark.parametrize("name", sorted(ROUNDTRIP))
def test_roundtrip(name):
write, read, rows = ROUNDTRIP[name]
con = duckdb.connect(":memory:")
assert write(con, "2026-Q2", rows) == len(rows)
assert sorted(read(con), key=repr) == sorted(rows, key=repr)
# writing a quarter again replaces it, other quarters are untouched
assert write(con, "2026-Q2", rows[:1]) == 1
assert read(con) == rows[:1]
assert T.table_counts(con)[f"drugs.{name}"] == 1
def test_write_empty_quarter_clears_it():
con = duckdb.connect(":memory:")
rows = ROUNDTRIP["asp_payment"][2]
T.write_asp(con, "2026-Q2", rows)
assert T.write_asp(con, "2026-Q2", []) == 0
assert T.read_asp(con) == []
def test_read_asp_filters():
con = duckdb.connect(":memory:")
a, b = ROUNDTRIP["asp_payment"][2]
c = T.AspRow(
"2026-Q1", "J9035", "II", "payment", "10 MG", 60.0, 20.0, "x", "", "f", "u", ""
)
T.write_asp(con, "2026-Q2", [a, b])
T.write_asp(con, "2026-Q1", [c])
assert [r.quarter for r in T.read_asp(con, hcpcs="J9035")] == ["2026-Q1", "2026-Q2"]
assert T.read_asp(con, quarter="2026-Q1") == [c]
assert T.read_asp(con, hcpcs="J7330", quarter="2026-Q2") == [b]
assert T.read_asp(con, hcpcs="ZZZZZ") == []