feat(drugs): all-code quarterly ASP payment, not-payable and NOC rows with provenance; skin subs read the same parser
Also: the CPT descriptor text is blanked in skin_substitute_hcpcs_reference.csv; level()/ndc11() reject fullwidth digits and trailing newlines, expand() caps ranges, appl_id() rejects unknown kinds.
This commit is contained in:
@@ -1,12 +1,12 @@
|
||||
"hcpcs_code","short_descriptor","code_type","body_site","wound_size","effective_date","termination_date","status","notes"
|
||||
"15271","Skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Replaced CPT 15330"
|
||||
"15272","Skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Add-on to 15271"
|
||||
"15273","Skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15331"
|
||||
"15274","Skin substitute graft, trunk/arms/legs, >=100 sq cm; each addl 100 sq cm","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Add-on to 15273"
|
||||
"15275","Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; first 25 sq cm or less","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Replaced CPT 15335"
|
||||
"15276","Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Add-on to 15275"
|
||||
"15277","Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15336"
|
||||
"15278","Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Add-on to 15277"
|
||||
"15271","","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Replaced CPT 15330"
|
||||
"15272","","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Add-on to 15271"
|
||||
"15273","","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15331"
|
||||
"15274","","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Add-on to 15273"
|
||||
"15275","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Replaced CPT 15335"
|
||||
"15276","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Add-on to 15275"
|
||||
"15277","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15336"
|
||||
"15278","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Add-on to 15277"
|
||||
"C5271","Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less","Application (OPPS Low-Cost)","Trunk/arms/legs","<100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026 per CY 2026 OPPS final rule"
|
||||
"C5272","Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm","Application (OPPS Low-Cost)","Trunk/arms/legs","<100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026"
|
||||
"C5273","Low cost skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm","Application (OPPS Low-Cost)","Trunk/arms/legs",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026"
|
||||
@@ -15,15 +15,15 @@
|
||||
"C5276","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026"
|
||||
"C5277","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026"
|
||||
"C5278","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026"
|
||||
"15330","Acellular dermal allograft, trunk/arms/legs; first 100 sq cm or less","Application (CPT - Historical)","Trunk/arms/legs","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15271/15272 in 2012"
|
||||
"15331","Acellular dermal allograft, trunk/arms/legs; each addl 100 sq cm","Application (CPT - Historical)","Trunk/arms/legs",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15273/15274 in 2012"
|
||||
"15335","Acellular dermal allograft, face/scalp/hands/feet; first 100 sq cm or less","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15275/15276 in 2012"
|
||||
"15336","Acellular dermal allograft, face/scalp/hands/feet; each addl 100 sq cm","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15277/15278 in 2012"
|
||||
"15340","Tissue cultured allogeneic skin substitute; first 25 sq cm or less","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 tissue-cultured allogeneic graft code"
|
||||
"15360","Tissue cultured autograft; first 25 sq cm or less","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 autologous cultured skin code"
|
||||
"15400","Xenograft; first 100 sq cm or less","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft application code"
|
||||
"15401","Xenograft; each addl 100 sq cm","Application (CPT - Historical)","",">100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft add-on code"
|
||||
"15430","Acellular xenograft implant; first 100 sq cm or less","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 acellular xenograft code"
|
||||
"15330","","Application (CPT - Historical)","Trunk/arms/legs","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15271/15272 in 2012"
|
||||
"15331","","Application (CPT - Historical)","Trunk/arms/legs",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15273/15274 in 2012"
|
||||
"15335","","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15275/15276 in 2012"
|
||||
"15336","","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15277/15278 in 2012"
|
||||
"15340","","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 tissue-cultured allogeneic graft code"
|
||||
"15360","","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 autologous cultured skin code"
|
||||
"15400","","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft application code"
|
||||
"15401","","Application (CPT - Historical)","",">100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft add-on code"
|
||||
"15430","","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 acellular xenograft code"
|
||||
"G0440","Skin substitute, tissue cultured; first 25 sq cm (OPPS)","Application (OPPS - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Transitional OPPS code for pass-through products"
|
||||
"C9363","Integra Meshed Bilayer Wound Matrix, per sq cm","Product (C-code Historical)","","","","","Deleted","Transitional pass-through C-code; replaced by Q-code"
|
||||
"C9358","SurgiMend Collagen Matrix (fetal), per 0.5 sq cm","Product (C-code Historical)","","","","","Active","Bovine collagen - fetal origin"
|
||||
|
||||
|
@@ -1,7 +1,8 @@
|
||||
"""Ingest CMS Average Sales Price (ASP) Drug Pricing Files.
|
||||
|
||||
Reads all quarterly ASP ZIP files from data/cms/asp/, normalises column
|
||||
names across 20+ years of CMS format drift, filters **strictly** to
|
||||
Reads all quarterly ASP ZIP files from data/cms/asp/ through the drug tier's
|
||||
parser (``drugs.asp.parse_quarter_zip``: column-name drift, section-508 CSV
|
||||
before its XLS twin, one file per quarter), filters **strictly** to
|
||||
skin substitute HCPCS codes defined in data/cms/skin_substitute_hcpcs.csv,
|
||||
and produces:
|
||||
|
||||
@@ -15,8 +16,8 @@ quarterly ZIP and the file within it that was parsed.
|
||||
Source URLs are read from data/cms/asp_source_urls.csv (canonical URLs
|
||||
provided from CMS.gov, one per quarter 2005-Q1 through 2026-Q2).
|
||||
|
||||
Column mapping rationale
|
||||
------------------------
|
||||
Column mapping rationale (the map itself now lives in ``drugs.asp.COL_MAP``)
|
||||
---------------------------------------------------------------------------
|
||||
CMS has changed column names across 20+ years of ASP files:
|
||||
|
||||
2005–2006: "HCPCS Code", "Short Description", "HCPCS Code Dosage",
|
||||
@@ -53,23 +54,34 @@ Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from drugs.asp import newest_per_quarter, parse_quarter_zip, pick_pricing_file # noqa: F401
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paths
|
||||
# ---------------------------------------------------------------------------
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
ASP_DIR = ROOT / "data" / "cms" / "asp"
|
||||
HCPCS_REF = ROOT / "data" / "cms" / "skin_substitute_hcpcs.csv"
|
||||
HCPCS_REFERENCE = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv"
|
||||
URL_REF = ROOT / "data" / "cms" / "asp_source_urls.csv"
|
||||
DUCKDB_PATH = ROOT / "data" / "aco.duckdb"
|
||||
OUTPUT_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv"
|
||||
|
||||
OUT_COLS = [
|
||||
"quarter",
|
||||
"hcpcs_code",
|
||||
"short_description",
|
||||
"asp_per_unit",
|
||||
"payment_limit",
|
||||
"dosage_or_unit",
|
||||
"source_url",
|
||||
"source_file",
|
||||
]
|
||||
|
||||
|
||||
def _load_source_urls() -> dict[str, str]:
|
||||
"""Load canonical source URLs: {quarter: url}."""
|
||||
@@ -81,142 +93,74 @@ def _load_source_urls() -> dict[str, str]:
|
||||
return urls
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Column normalisation
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mapping from observed CMS column names (lowercased) to our standard names.
|
||||
# Each entry documents which CMS file years use that variant.
|
||||
COL_MAP = {
|
||||
# HCPCS code — consistent across all years
|
||||
"hcpcs code": "hcpcs_code",
|
||||
"hcpcs_code": "hcpcs_code",
|
||||
# Short description — consistent
|
||||
"short description": "short_description",
|
||||
"short_description": "short_description",
|
||||
# Dosage/unit — varies by year
|
||||
"hcpcs code dosage": "dosage_or_unit", # 2005-2006, 2020-2025
|
||||
"hcpcs_code_dosage": "dosage_or_unit",
|
||||
"dosage": "dosage_or_unit", # 2007-2019
|
||||
"dosage or unit": "dosage_or_unit", # 2026+
|
||||
# Payment limit — consistent (ASP + 6%)
|
||||
"payment limit": "payment_limit",
|
||||
"payment_limit": "payment_limit",
|
||||
}
|
||||
|
||||
KEEP_COLS = ["hcpcs_code", "short_description", "dosage_or_unit", "payment_limit"]
|
||||
def require_rows(df: pd.DataFrame) -> None:
|
||||
"""Refuse to replace ``skin_subs.asp_quarterly`` with nothing: a parser
|
||||
fault must not publish an empty table over the replica."""
|
||||
if df.empty:
|
||||
raise RuntimeError(
|
||||
"no skin-substitute rows parsed; leaving skin_subs.asp_quarterly as it is"
|
||||
)
|
||||
|
||||
|
||||
def normalise_columns(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Lower-case columns, apply COL_MAP, keep only KEEP_COLS."""
|
||||
df.columns = [str(c).strip().lower() for c in df.columns]
|
||||
df = df.rename(columns=COL_MAP)
|
||||
present = [c for c in KEEP_COLS if c in df.columns]
|
||||
return df[present]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Readers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _find_header_row(lines: list[str]) -> int:
|
||||
"""Return the 0-based row index of the column-header row.
|
||||
|
||||
The header row starts with 'HCPCS Code,' (possibly quoted).
|
||||
Must NOT match note lines that incidentally mention 'HCPCS code'.
|
||||
"""
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.strip().lower()
|
||||
if stripped.startswith("hcpcs code,") or stripped.startswith('"hcpcs code",'):
|
||||
return i
|
||||
return -1
|
||||
|
||||
|
||||
def read_csv_from_zip(zf: zipfile.ZipFile, name: str) -> pd.DataFrame:
|
||||
"""Read a section-508 CSV inside a ZIP, skipping preamble rows."""
|
||||
raw = zf.read(name).decode("latin-1")
|
||||
lines = raw.splitlines()
|
||||
hdr = _find_header_row(lines)
|
||||
if hdr < 0:
|
||||
return pd.DataFrame()
|
||||
buf = "\n".join(lines[hdr:])
|
||||
df = pd.read_csv(io.StringIO(buf), dtype=str, on_bad_lines="skip")
|
||||
return normalise_columns(df)
|
||||
|
||||
|
||||
def read_xls_from_zip(zf: zipfile.ZipFile, name: str) -> pd.DataFrame:
|
||||
"""Read .xls inside a ZIP, skipping preamble rows."""
|
||||
import xlrd
|
||||
|
||||
data = zf.read(name)
|
||||
wb = xlrd.open_workbook(file_contents=data)
|
||||
sh = wb.sheet_by_index(0)
|
||||
hdr_row = -1
|
||||
for r in range(min(30, sh.nrows)):
|
||||
first_cell = str(sh.cell_value(r, 0)).strip().lower()
|
||||
if first_cell == "hcpcs code":
|
||||
hdr_row = r
|
||||
break
|
||||
if hdr_row < 0:
|
||||
return pd.DataFrame()
|
||||
headers = [str(sh.cell_value(hdr_row, c)).strip() for c in range(sh.ncols)]
|
||||
rows = []
|
||||
for r in range(hdr_row + 1, sh.nrows):
|
||||
row = [str(sh.cell_value(r, c)).strip() for c in range(sh.ncols)]
|
||||
if any(row):
|
||||
rows.append(row)
|
||||
df = pd.DataFrame(rows, columns=headers)
|
||||
return normalise_columns(df)
|
||||
|
||||
|
||||
def pick_pricing_file(names: list[str]) -> str | None:
|
||||
"""From a list of files in a ZIP, pick the ASP pricing file.
|
||||
|
||||
Prefer CSV (section 508) over XLS. Skip NOC, crosswalk, NDC,
|
||||
and "not payable" files.
|
||||
"""
|
||||
skip_patterns = ["noc", "crosswalk", "ndc", "not payable", "not_payable"]
|
||||
candidates: list[str] = []
|
||||
for n in names:
|
||||
low = n.lower()
|
||||
if any(s in low for s in skip_patterns):
|
||||
continue
|
||||
if low.endswith((".csv", ".xls", ".xlsx")):
|
||||
candidates.append(n)
|
||||
|
||||
csvs = [c for c in candidates if c.lower().endswith(".csv")]
|
||||
if csvs:
|
||||
for c in csvs:
|
||||
low = c.lower()
|
||||
if "payment limit" in low or "pricing" in low:
|
||||
return c
|
||||
return csvs[0]
|
||||
|
||||
xlss = [c for c in candidates if c.lower().endswith((".xls", ".xlsx"))]
|
||||
if xlss:
|
||||
for c in xlss:
|
||||
low = c.lower()
|
||||
if "payment limit" in low or "pricing" in low or "asp" in low:
|
||||
return c
|
||||
return xlss[0]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Numeric cleaning
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def clean_numeric(series: pd.Series) -> pd.Series:
|
||||
"""Strip $ signs and convert to float, coercing errors to NaN."""
|
||||
return (
|
||||
series.astype(str)
|
||||
.str.replace("$", "", regex=False)
|
||||
.str.replace(",", "", regex=False)
|
||||
.str.strip()
|
||||
.pipe(pd.to_numeric, errors="coerce")
|
||||
def build_skin_df() -> pd.DataFrame:
|
||||
"""Every payment-limit row of a skin-substitute code (strict filter: only
|
||||
codes in skin_substitute_hcpcs.csv), one file per quarter, with
|
||||
``asp_per_unit = payment_limit / 1.06`` (CMS payment limit = ASP + 6% for
|
||||
drugs/biologicals; no longer meaningful for the flat 2026+ rate, kept for
|
||||
consistency)."""
|
||||
skin_codes = set(
|
||||
pd.read_csv(HCPCS_REF, dtype=str)["hcpcs_code"].str.strip().str.upper()
|
||||
)
|
||||
print(f"Loaded {len(skin_codes)} skin-substitute HCPCS codes from reference")
|
||||
source_urls = _load_source_urls()
|
||||
print(f"Loaded {len(source_urls)} source URLs")
|
||||
zips = newest_per_quarter(ASP_DIR.glob("asp_*.zip"))
|
||||
print(f"Found {len(zips)} quarterly ZIP files\n")
|
||||
|
||||
records: list[dict[str, object]] = []
|
||||
seen_q4: set[str] = set()
|
||||
total = 0
|
||||
for quarter, zpath in sorted(zips.items()):
|
||||
try:
|
||||
rows, _ = parse_quarter_zip(
|
||||
zpath, quarter, source_url=source_urls.get(quarter, "")
|
||||
)
|
||||
except Exception as e: # a bad zip must not hide the other quarters
|
||||
print(f" ERR {quarter}: {e}")
|
||||
continue
|
||||
payable = [r for r in rows if r.list_kind == "payment"]
|
||||
if not payable:
|
||||
print(f" SKIP {quarter}: no pricing rows found in ZIP")
|
||||
continue
|
||||
total += len(payable)
|
||||
print(f" OK {quarter}: {len(payable):>5} rows from {payable[0].source_file}")
|
||||
for r in payable:
|
||||
if r.hcpcs in skin_codes:
|
||||
records.append(
|
||||
{
|
||||
"quarter": quarter,
|
||||
"hcpcs_code": r.hcpcs,
|
||||
"short_description": r.short_desc,
|
||||
"payment_limit": r.payment_limit,
|
||||
"dosage_or_unit": r.dosage,
|
||||
"source_url": r.source_url,
|
||||
"source_file": r.source_file,
|
||||
}
|
||||
)
|
||||
elif r.hcpcs.startswith("Q4") and len(r.hcpcs) in (4, 5):
|
||||
seen_q4.add(r.hcpcs)
|
||||
print(f"\nTotal rows across all quarters: {total}")
|
||||
print(f"Skin substitute rows (strict filter): {len(records)}")
|
||||
if seen_q4:
|
||||
print(f" Excluded Q4 codes NOT in skin sub universe: {sorted(seen_q4)}")
|
||||
|
||||
skin_df = pd.DataFrame.from_records(records)
|
||||
if skin_df.empty:
|
||||
return pd.DataFrame(columns=OUT_COLS)
|
||||
skin_df["payment_limit"] = pd.to_numeric(skin_df["payment_limit"], errors="coerce")
|
||||
# ASP per unit = payment_limit / 1.06
|
||||
skin_df["asp_per_unit"] = (skin_df["payment_limit"] / 1.06).round(3)
|
||||
return skin_df[OUT_COLS].sort_values(["quarter", "hcpcs_code"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -225,106 +169,17 @@ def clean_numeric(series: pd.Series) -> pd.Series:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
# Load skin-substitute HCPCS codes (strict filter — no regex fallback)
|
||||
hcpcs_ref = pd.read_csv(HCPCS_REF, dtype=str)
|
||||
skin_codes = set(hcpcs_ref["hcpcs_code"].str.strip().str.upper())
|
||||
print(f"Loaded {len(skin_codes)} skin-substitute HCPCS codes from reference")
|
||||
|
||||
# Load canonical source URLs
|
||||
source_urls = _load_source_urls()
|
||||
print(f"Loaded {len(source_urls)} source URLs")
|
||||
|
||||
# Ingest all quarters
|
||||
zips = sorted(ASP_DIR.glob("asp_*.zip"))
|
||||
print(f"Found {len(zips)} quarterly ZIP files\n")
|
||||
|
||||
frames: list[pd.DataFrame] = []
|
||||
for zpath in zips:
|
||||
m = re.match(r"asp_(\d{4}-Q\d)", zpath.name)
|
||||
if not m:
|
||||
print(f" SKIP {zpath.name} (cannot parse quarter)")
|
||||
continue
|
||||
quarter = m.group(1)
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(zpath) as zf:
|
||||
target = pick_pricing_file(zf.namelist())
|
||||
if target is None:
|
||||
print(f" SKIP {quarter}: no pricing file found in ZIP")
|
||||
continue
|
||||
|
||||
if target.lower().endswith(".csv"):
|
||||
df = read_csv_from_zip(zf, target)
|
||||
else:
|
||||
df = read_xls_from_zip(zf, target)
|
||||
|
||||
if df.empty:
|
||||
print(f" SKIP {quarter}: empty after parsing {target}")
|
||||
continue
|
||||
|
||||
df["quarter"] = quarter
|
||||
df["source_file"] = target
|
||||
df["source_url"] = source_urls.get(quarter, "")
|
||||
frames.append(df)
|
||||
print(f" OK {quarter}: {len(df):>5} rows from {target}")
|
||||
except Exception as e:
|
||||
print(f" ERR {quarter}: {e}")
|
||||
|
||||
if not frames:
|
||||
raise RuntimeError("No data parsed from any file")
|
||||
|
||||
all_df = pd.concat(frames, ignore_index=True)
|
||||
print(f"\nTotal rows across all quarters: {len(all_df)}")
|
||||
|
||||
# Normalise HCPCS codes
|
||||
all_df["hcpcs_code"] = all_df["hcpcs_code"].astype(str).str.strip().str.upper()
|
||||
|
||||
# Filter STRICTLY to skin substitute code universe
|
||||
# NO regex fallback — only codes in skin_substitute_hcpcs.csv
|
||||
skin_df = all_df[all_df["hcpcs_code"].isin(skin_codes)].copy()
|
||||
print(f"Skin substitute rows (strict filter): {len(skin_df)}")
|
||||
|
||||
# Show what we excluded
|
||||
q4_but_not_skin = all_df[
|
||||
all_df["hcpcs_code"].str.match(r"^Q4\d{2,3}$")
|
||||
& ~all_df["hcpcs_code"].isin(skin_codes)
|
||||
]["hcpcs_code"].unique()
|
||||
if len(q4_but_not_skin) > 0:
|
||||
print(
|
||||
f" Excluded Q4 codes NOT in skin sub universe: {sorted(q4_but_not_skin)}"
|
||||
)
|
||||
|
||||
if not skin_df.empty:
|
||||
skin_df["payment_limit"] = clean_numeric(skin_df["payment_limit"])
|
||||
# ASP per unit = payment_limit / 1.06
|
||||
# (CMS payment limit = ASP + 6% for drugs/biologicals)
|
||||
skin_df["asp_per_unit"] = (skin_df["payment_limit"] / 1.06).round(3)
|
||||
|
||||
# Reorder columns
|
||||
out_cols = [
|
||||
"quarter",
|
||||
"hcpcs_code",
|
||||
"short_description",
|
||||
"asp_per_unit",
|
||||
"payment_limit",
|
||||
"dosage_or_unit",
|
||||
"source_url",
|
||||
"source_file",
|
||||
]
|
||||
for c in out_cols:
|
||||
if c not in skin_df.columns:
|
||||
skin_df[c] = None
|
||||
skin_df = skin_df[out_cols].sort_values(["quarter", "hcpcs_code"])
|
||||
skin_df = build_skin_df()
|
||||
require_rows(skin_df)
|
||||
|
||||
# Write CSV
|
||||
skin_df.to_csv(OUTPUT_CSV, index=False)
|
||||
print(f"\nWrote {len(skin_df)} rows to {OUTPUT_CSV}")
|
||||
|
||||
if len(skin_df) > 0:
|
||||
quarters = sorted(skin_df["quarter"].unique())
|
||||
codes = sorted(skin_df["hcpcs_code"].unique())
|
||||
print(f" Quarters: {len(quarters)} ({quarters[0]} to {quarters[-1]})")
|
||||
print(f" Unique HCPCS codes: {len(codes)}")
|
||||
quarters = sorted(skin_df["quarter"].unique())
|
||||
codes = sorted(skin_df["hcpcs_code"].unique())
|
||||
print(f" Quarters: {len(quarters)} ({quarters[0]} to {quarters[-1]})")
|
||||
print(f" Unique HCPCS codes: {len(codes)}")
|
||||
|
||||
# Load into DuckDB
|
||||
print(f"\nLoading into DuckDB at {DUCKDB_PATH} ...")
|
||||
@@ -335,24 +190,26 @@ def main() -> None:
|
||||
with duckdb_batch("aco") as con:
|
||||
con.execute("CREATE SCHEMA IF NOT EXISTS skin_subs")
|
||||
con.execute("DROP TABLE IF EXISTS skin_subs.asp_quarterly")
|
||||
if not skin_df.empty:
|
||||
con.execute(
|
||||
"""
|
||||
CREATE TABLE skin_subs.asp_quarterly AS
|
||||
SELECT * FROM read_csv_auto(?, header=true)
|
||||
""",
|
||||
[str(OUTPUT_CSV)],
|
||||
)
|
||||
count = con.execute("SELECT count(*) FROM skin_subs.asp_quarterly").fetchone()[
|
||||
0
|
||||
]
|
||||
print(f" Loaded {count} rows into skin_subs.asp_quarterly")
|
||||
if HCPCS_REFERENCE.exists():
|
||||
# the scrubbed reference (CPT descriptors blanked; codes and
|
||||
# HCPCS Level II text kept)
|
||||
con.execute("DROP TABLE IF EXISTS skin_subs.hcpcs_reference")
|
||||
con.execute(
|
||||
"""
|
||||
CREATE TABLE skin_subs.asp_quarterly AS
|
||||
SELECT * FROM read_csv_auto(?, header=true)
|
||||
""",
|
||||
[str(OUTPUT_CSV)],
|
||||
"CREATE TABLE skin_subs.hcpcs_reference AS "
|
||||
"SELECT * FROM read_csv_auto(?, header=true)",
|
||||
[str(HCPCS_REFERENCE)],
|
||||
)
|
||||
count = con.execute(
|
||||
"SELECT count(*) FROM skin_subs.asp_quarterly"
|
||||
).fetchone()[0]
|
||||
print(f" Loaded {count} rows into skin_subs.asp_quarterly")
|
||||
else:
|
||||
print(" WARNING: No skin substitute data found in ASP files")
|
||||
print(
|
||||
" This may be expected — skin subs may not appear in main ASP pricing files"
|
||||
)
|
||||
print(" They may be in separate NOC or tissue coding files")
|
||||
|
||||
# Refresh the notebook read replica (#510).
|
||||
print(f"replica → {publish_replica('aco')}")
|
||||
|
||||
167
src/cli/drugs.py
167
src/cli/drugs.py
@@ -2,12 +2,18 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import zipfile
|
||||
from typing import Any
|
||||
|
||||
import typer
|
||||
|
||||
from drugs import asp as A
|
||||
from drugs import http as dh
|
||||
from drugs import tables as T
|
||||
from drugs.tables import table_counts
|
||||
from market.tables import read_gaps, write_gap
|
||||
from market.http import Refused
|
||||
from market.tables import clear_gap, read_gaps, write_gap
|
||||
from pfs.codetables import is_missing_table_error
|
||||
|
||||
app = typer.Typer(no_args_is_help=True)
|
||||
@@ -89,3 +95,162 @@ def status() -> None:
|
||||
typer.echo("no gaps recorded")
|
||||
for g in gaps:
|
||||
typer.echo(f"gap: {g.source}: {g.note} ({g.recorded_on})")
|
||||
|
||||
|
||||
# ── asp ──────────────────────────────────────────────────────────────
|
||||
ASP_GAP = "cms-asp:asp"
|
||||
|
||||
|
||||
def _cite(store: Any, url: str, title: str, table: str) -> str:
|
||||
if not url:
|
||||
return ""
|
||||
return dh.cite_file(store, url=url, title=title, provider="cms-asp", table=table)
|
||||
|
||||
|
||||
def _quarters_option(values: list[str] | None) -> list[str]:
|
||||
for v in values or []:
|
||||
if not re.fullmatch(r"[0-9]{4}-Q[1-4]", v):
|
||||
raise typer.BadParameter(f"{v!r} is not a quarter like 2026-Q4")
|
||||
return list(values or [])
|
||||
|
||||
|
||||
@app.command()
|
||||
def asp(
|
||||
fetch: bool = typer.Option(False, "--fetch", help="Hoard the page and its zips."),
|
||||
refresh: bool = typer.Option(False, "--refresh", help="Re-download every zip."),
|
||||
quarter: list[str] = typer.Option(None, "--quarter", help="Only these quarters."),
|
||||
dry_run: bool = typer.Option(
|
||||
False, "--dry-run", help="List the links; change nothing."
|
||||
),
|
||||
write: bool = typer.Option(
|
||||
False, "--write", help="Write drugs.asp_payment and drugs.asp_noc."
|
||||
),
|
||||
) -> None:
|
||||
"""Quarterly Part B payment limits for every code (payable, not-payable
|
||||
and NOC rows) from the hoarded CMS ASP files."""
|
||||
sel = set(_quarters_option(quarter))
|
||||
write = write and not dry_run
|
||||
cl = None
|
||||
if fetch:
|
||||
cl = dh.client("cms")
|
||||
try:
|
||||
A.fetch_page(cl)
|
||||
except Refused as exc:
|
||||
_gap(ASP_GAP, f"{A.ASP_PAGE} refused: {exc}", write=write)
|
||||
return
|
||||
page = A.hoarded_page()
|
||||
links = [
|
||||
lk
|
||||
for lk in (A.parse_asp_page(page.read_text(errors="replace")) if page else [])
|
||||
if lk.kind in ("pricing", "noc", "crosswalk") and (not sel or lk.quarter in sel)
|
||||
]
|
||||
steps = A.plan_fetch(links, refresh=refresh)
|
||||
if dry_run:
|
||||
if not links:
|
||||
typer.echo("no page hoarded; run with --fetch")
|
||||
for st in steps:
|
||||
typer.echo(f"{st.link.kind} {st.link.quarter} {st.action} {st.link.url}")
|
||||
return
|
||||
refusal = ""
|
||||
if fetch:
|
||||
for st in steps:
|
||||
if st.action == "have":
|
||||
continue
|
||||
try:
|
||||
A.apply_plan(cl, st)
|
||||
except Refused as exc:
|
||||
refusal = f"refused: {exc}"
|
||||
break
|
||||
typer.echo(f"{st.action} {st.link.kind} {st.link.quarter}")
|
||||
|
||||
def pick(kind: str, prefix: str) -> dict[str, Any]:
|
||||
got = A.newest_per_quarter(A.hoard_dir(kind).glob(f"{prefix}_*.zip"))
|
||||
return {q: p for q, p in got.items() if not sel or q in sel}
|
||||
|
||||
pricing, nocs, xwalks = (
|
||||
pick("pricing", "asp"),
|
||||
pick("noc", "noc"),
|
||||
pick("crosswalk", "xwalk"),
|
||||
)
|
||||
url_of = {(lk.quarter, lk.kind): lk.url for lk in links}
|
||||
canon = A.canonical_urls()
|
||||
store = _store() if write else None
|
||||
|
||||
def source(q: str, kind: str, title: str, table: str) -> tuple[str, str]:
|
||||
url = url_of.get((q, kind)) or (canon.get(q, "") if kind == "pricing" else "")
|
||||
return url, (_cite(store, url, title, table) if write else "")
|
||||
|
||||
payments: dict[str, list[T.AspRow]] = {}
|
||||
noc_rows: dict[str, list[T.NocRow]] = {}
|
||||
files: dict[tuple[str, str], Any] = {}
|
||||
unreadable: list[str] = []
|
||||
for q, path in sorted(pricing.items()):
|
||||
url, key = source(
|
||||
q, "pricing", f"CMS ASP pricing file {q}", "drugs.asp_payment"
|
||||
)
|
||||
try:
|
||||
rows, bundled = A.parse_quarter_zip(path, q, source_url=url, item_key=key)
|
||||
except zipfile.BadZipFile:
|
||||
unreadable.append(f"{q} pricing unreadable")
|
||||
continue
|
||||
payments[q] = rows
|
||||
files[(q, "drugs.asp_payment")] = path
|
||||
if bundled and q not in nocs:
|
||||
noc_rows[q] = bundled
|
||||
files[(q, "drugs.asp_noc")] = path
|
||||
for q, path in sorted(nocs.items()):
|
||||
url, key = source(q, "noc", f"CMS ASP NOC pricing file {q}", "drugs.asp_noc")
|
||||
try:
|
||||
_, rows = A.parse_quarter_zip(path, q, source_url=url, item_key=key)
|
||||
except zipfile.BadZipFile:
|
||||
unreadable.append(f"{q} noc unreadable")
|
||||
continue
|
||||
noc_rows[q] = rows
|
||||
files[(q, "drugs.asp_noc")] = path
|
||||
|
||||
for q in sorted(payments):
|
||||
rows = payments[q]
|
||||
kinds = [r.list_kind for r in rows]
|
||||
typer.echo(
|
||||
f"{q}: {kinds.count('payment')} payment, {kinds.count('not-payable')} not-payable,"
|
||||
f" {len(noc_rows.get(q, []))} noc"
|
||||
)
|
||||
for q in sorted(set(noc_rows) - set(payments)):
|
||||
typer.echo(f"{q}: {len(noc_rows[q])} noc")
|
||||
expected = sorted(sel) if sel else A.expected_quarters(A.today())
|
||||
typer.echo(
|
||||
f"pricing {len(payments)}/{len(expected)} quarters,"
|
||||
f" {sum(len(r) for r in payments.values())} rows,"
|
||||
f" {sum(r.list_kind == 'not-payable' for v in payments.values() for r in v)} not-payable;"
|
||||
f" noc {len(noc_rows)} quarters, {sum(len(r) for r in noc_rows.values())} rows"
|
||||
)
|
||||
have = {
|
||||
"pricing": set(pricing),
|
||||
"noc": set(nocs) | set(noc_rows),
|
||||
"crosswalk": set(xwalks),
|
||||
}
|
||||
listed = {(lk.quarter, lk.kind) for lk in links}
|
||||
pairs = unreadable + A.gap_pairs(expected, have, listed)
|
||||
note = "; ".join(x for x in (refusal, A.gap_note(pairs)) if x)
|
||||
if note:
|
||||
typer.echo(f"gap: {ASP_GAP}: {note}")
|
||||
if not write:
|
||||
return
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
with _batch() as con:
|
||||
T.ensure_tables(con)
|
||||
for q, rows in payments.items():
|
||||
T.write_asp(con, q, rows)
|
||||
for q, nrows in noc_rows.items():
|
||||
T.write_noc(con, q, nrows)
|
||||
for (q, table), path in sorted(files.items()):
|
||||
n = len(payments[q]) if table == "drugs.asp_payment" else len(noc_rows[q])
|
||||
log_ingest(
|
||||
con, module="drugs.asp", table_name=table, rows=n, source_file=str(path)
|
||||
)
|
||||
if note:
|
||||
write_gap(con, ASP_GAP, note)
|
||||
else:
|
||||
clear_gap(con, ASP_GAP)
|
||||
_publish()
|
||||
|
||||
594
src/drugs/asp.py
Normal file
594
src/drugs/asp.py
Normal file
@@ -0,0 +1,594 @@
|
||||
"""CMS Medicare Part B drug payment-limit ("ASP") files, every quarter.
|
||||
|
||||
One parser serves two readers: ``stack drugs asp`` (all codes, plus the
|
||||
"drugs not payable" lists and the not-otherwise-classified drugs) and the
|
||||
skin-substitute ingest (``dev/scripts/ingest_asp.py``).
|
||||
|
||||
Rules held here:
|
||||
|
||||
* CMS's short descriptions of CPT (Level I) codes are AMA text and are never
|
||||
kept: a licensed code's description is ``""``. HCPCS Level II text is CMS's
|
||||
own and is kept.
|
||||
* A zip holds one member per kind (pricing, not-payable, NOC); when CMS ships
|
||||
both a section-508 ``.csv`` and its ``.xls`` twin, the ``.csv`` is read and
|
||||
the twin is not, so no row is counted twice.
|
||||
* Files are hoarded under ``data/cms/asp`` and never overwritten; an updated
|
||||
posting lands beside the old file and is the one read.
|
||||
* A quarter with no file is a gap, never a zero.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import zipfile
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urljoin, urlparse
|
||||
|
||||
from drugs import http as dh
|
||||
from drugs.codes import level, licensed
|
||||
from drugs.tables import ( # noqa: F401 (re-exported)
|
||||
AspRow,
|
||||
NocRow,
|
||||
read_asp,
|
||||
read_noc,
|
||||
write_asp,
|
||||
write_noc,
|
||||
)
|
||||
|
||||
ASP_PAGE = "https://www.cms.gov/medicare/payment/part-b-drugs/asp-pricing-files"
|
||||
CMS = "https://www.cms.gov"
|
||||
FIRST_QUARTER = "2005-Q1"
|
||||
KINDS = ("pricing", "noc", "crosswalk", "vaccine", "other")
|
||||
|
||||
|
||||
def today() -> date:
|
||||
return date.today()
|
||||
|
||||
|
||||
# ── the page ─────────────────────────────────────────────────────────
|
||||
@dataclass(frozen=True)
|
||||
class AspLink:
|
||||
quarter: str
|
||||
kind: str
|
||||
url: str
|
||||
text: str
|
||||
|
||||
|
||||
class _Anchors(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.found: list[tuple[str, str]] = []
|
||||
self._href: str | None = None
|
||||
self._text: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag == "a":
|
||||
self._href = dict(attrs).get("href")
|
||||
self._text = []
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._href is not None:
|
||||
self._text.append(data)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag == "a" and self._href is not None:
|
||||
text = re.sub(r"\s+", " ", "".join(self._text).replace("\xa0", " "))
|
||||
self.found.append((self._href, text.strip()))
|
||||
self._href = None
|
||||
|
||||
|
||||
_MONTH_Q = {"jan": 1, "apr": 2, "jul": 3, "oct": 4}
|
||||
_TEXT_Q = re.compile(r"\b(jan|apr|jul|oct)[a-z]*\.?\s+(\d{4})\b")
|
||||
_HREF_Q = re.compile(r"(january|april|july|october)[-_ ]?(\d{4})")
|
||||
_HREF_YQ = re.compile(r"(\d{4})[-_ ](january|april|july|october)")
|
||||
|
||||
|
||||
def _quarter_of(text: str, href: str) -> str:
|
||||
m = _TEXT_Q.search(text.lower())
|
||||
if m:
|
||||
return f"{m.group(2)}-Q{_MONTH_Q[m.group(1)]}"
|
||||
low = href.lower()
|
||||
m = _HREF_Q.search(low)
|
||||
if m:
|
||||
return f"{m.group(2)}-Q{_MONTH_Q[m.group(1)[:3]]}"
|
||||
m = _HREF_YQ.search(low)
|
||||
if m:
|
||||
return f"{m.group(1)}-Q{_MONTH_Q[m.group(2)[:3]]}"
|
||||
return ""
|
||||
|
||||
|
||||
_KIND_MARKS = (
|
||||
("crosswalk", ("crosswalk", "ndc")),
|
||||
("noc", ("noc",)),
|
||||
("pricing", ("asp pricing", "payment limit")),
|
||||
)
|
||||
|
||||
|
||||
def _kind_of(text: str) -> str:
|
||||
"""The first kind the link text names: a bundle such as "ASP Pricing
|
||||
File/ASP NOC Pricing File/ASP NDC-HCPCS Crosswalk" is its pricing zip."""
|
||||
low = text.lower()
|
||||
if "vaccine" in low:
|
||||
return "vaccine"
|
||||
best, where = "other", len(low) + 1
|
||||
for kind, marks in _KIND_MARKS:
|
||||
for mark in marks:
|
||||
i = low.find(mark)
|
||||
if 0 <= i < where:
|
||||
best, where = kind, i
|
||||
return best
|
||||
|
||||
|
||||
def parse_asp_page(html: str) -> list[AspLink]:
|
||||
"""Every zip link of the ASP pricing-files page, with its quarter and
|
||||
kind. Vaccine and unrecognised links have quarter ``""``."""
|
||||
p = _Anchors()
|
||||
p.feed(html)
|
||||
out: list[AspLink] = []
|
||||
for href, text in p.found:
|
||||
url = direct_url(href)
|
||||
if not urlparse(url).path.lower().endswith(".zip"):
|
||||
continue
|
||||
kind = _kind_of(text)
|
||||
quarter = _quarter_of(text, url)
|
||||
if kind in ("vaccine", "other") or not quarter:
|
||||
kind, quarter = ("vaccine" if kind == "vaccine" else "other"), ""
|
||||
out.append(AspLink(quarter, kind, url, text))
|
||||
return out
|
||||
|
||||
|
||||
def direct_url(href: str) -> str:
|
||||
"""The file URL behind a CMS click-through link
|
||||
(``.../license.asp?file=<p>`` or ``.../license/ama?file=<p>`` ->
|
||||
``https://www.cms.gov<p>``); a relative href is made absolute."""
|
||||
parts = urlparse(href)
|
||||
if parts.path.lower().endswith(("/license.asp", "/license/ama")):
|
||||
target = parse_qs(parts.query).get("file", [""])[0]
|
||||
if target:
|
||||
return urljoin(CMS, target)
|
||||
return urljoin(CMS, href)
|
||||
|
||||
|
||||
def page_path() -> Path:
|
||||
return dh.path("drugs", "cms", "asp-pricing-files.html")
|
||||
|
||||
|
||||
def fetch_page(cl: Any) -> Path:
|
||||
"""Hoard the page (a changed page lands beside the old one)."""
|
||||
return dh.fetch(
|
||||
cl, ASP_PAGE, page_path(), provider="cms", source="cms-asp", refresh=True
|
||||
)
|
||||
|
||||
|
||||
def hoarded_page() -> Path | None:
|
||||
p = dh.newest(page_path())
|
||||
return p if p.is_file() else None
|
||||
|
||||
|
||||
# ── hoard layout ─────────────────────────────────────────────────────
|
||||
_DIRS = {
|
||||
"pricing": ("asp", "asp"),
|
||||
"noc": ("asp/noc", "noc"),
|
||||
"crosswalk": ("asp/xwalk", "xwalk"),
|
||||
}
|
||||
|
||||
|
||||
def hoard_dir(kind: str) -> Path:
|
||||
if kind not in _DIRS:
|
||||
raise ValueError(f"no hoard for kind {kind!r}")
|
||||
return dh.path("cms", *_DIRS[kind][0].split("/"))
|
||||
|
||||
|
||||
def hoard_path(link: AspLink) -> Path:
|
||||
return hoard_dir(link.kind) / f"{_DIRS[link.kind][1]}_{link.quarter}.zip"
|
||||
|
||||
|
||||
_ZIP_Q = re.compile(r"^(?:asp|noc|xwalk)_(\d{4}-Q[1-4])(?:\.[0-9a-f]{8})?\.zip$")
|
||||
|
||||
|
||||
def newest_per_quarter(paths: Iterable[Path]) -> dict[str, Path]:
|
||||
"""One path per quarter: the newest copy (``market.http.newest``'s order:
|
||||
write time, then name), so an updated posting beside the original wins and
|
||||
a quarter is never read twice."""
|
||||
best: dict[str, Path] = {}
|
||||
for p in paths:
|
||||
m = _ZIP_Q.match(p.name)
|
||||
if not m:
|
||||
continue
|
||||
q = m.group(1)
|
||||
key = (p.stat().st_mtime_ns, p.name)
|
||||
if q not in best or key > (best[q].stat().st_mtime_ns, best[q].name):
|
||||
best[q] = p
|
||||
return best
|
||||
|
||||
|
||||
def expected_quarters(today: date) -> list[str]:
|
||||
"""Every quarter from 2005-Q1 to the one containing *today*."""
|
||||
out: list[str] = []
|
||||
y, q = 2005, 1
|
||||
while (y, q) <= (today.year, (today.month - 1) // 3 + 1):
|
||||
out.append(f"{y}-Q{q}")
|
||||
y, q = (y + 1, 1) if q == 4 else (y, q + 1)
|
||||
return out
|
||||
|
||||
|
||||
# ── the manifest and what to fetch ──────────────────────────────────
|
||||
def manifest_urls() -> set[str]:
|
||||
mf = dh.data_root() / "drugs" / dh.MANIFEST
|
||||
if not mf.exists():
|
||||
return set()
|
||||
return {json.loads(x)["url"] for x in mf.read_text().splitlines() if x.strip()}
|
||||
|
||||
|
||||
def canonical_urls() -> dict[str, str]:
|
||||
"""``data/cms/asp_source_urls.csv``: the URL each of the first 86 pricing
|
||||
zips was downloaded from."""
|
||||
p = dh.path("cms", "asp_source_urls.csv")
|
||||
if not p.exists():
|
||||
return {}
|
||||
with p.open(newline="") as fh:
|
||||
return {r["quarter"]: r["url"] for r in csv.DictReader(fh)}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Step:
|
||||
link: AspLink
|
||||
dest: Path
|
||||
action: str # have | adopt | fetch | fetch-beside
|
||||
|
||||
|
||||
def plan_fetch(links: Sequence[AspLink], *, refresh: bool) -> list[Step]:
|
||||
"""What to do with each pricing, NOC and crosswalk link: ``have`` (its
|
||||
URL is in the manifest), ``adopt`` (the file on disk was downloaded from
|
||||
this URL, per ``asp_source_urls.csv``; record it), ``fetch`` (nothing on
|
||||
disk) or ``fetch-beside`` (a different posting of a hoarded quarter, or
|
||||
*refresh*; it lands beside the old file)."""
|
||||
known = manifest_urls()
|
||||
canon = canonical_urls()
|
||||
steps: list[Step] = []
|
||||
for link in links:
|
||||
if link.kind not in _DIRS or not link.quarter:
|
||||
continue
|
||||
dest = hoard_path(link)
|
||||
if not dest.exists():
|
||||
action = "fetch"
|
||||
elif refresh:
|
||||
action = "fetch-beside"
|
||||
elif link.url in known:
|
||||
action = "have"
|
||||
elif link.kind == "pricing" and canon.get(link.quarter) == link.url:
|
||||
action = "adopt"
|
||||
else:
|
||||
action = "fetch-beside"
|
||||
steps.append(Step(link, dest, action))
|
||||
return steps
|
||||
|
||||
|
||||
def apply_plan(cl: Any, step: Step) -> Path:
|
||||
"""Carry out one step; returns the hoarded path (the newest copy)."""
|
||||
if step.action == "have":
|
||||
return dh.newest(step.dest)
|
||||
if step.action == "adopt":
|
||||
dh.manifest(
|
||||
step.dest, step.link.url, "adopted: downloaded from asp_source_urls.csv"
|
||||
)
|
||||
return step.dest
|
||||
return dh.fetch(
|
||||
cl,
|
||||
step.link.url,
|
||||
step.dest,
|
||||
provider="cms",
|
||||
source="cms-asp",
|
||||
refresh=True,
|
||||
note=f"{step.link.kind} {step.link.quarter}",
|
||||
)
|
||||
|
||||
|
||||
def gap_pairs(
|
||||
expected: Sequence[str],
|
||||
have: dict[str, set[str]],
|
||||
listed: set[tuple[str, str]],
|
||||
) -> list[str]:
|
||||
"""``"<quarter> <kind>"`` for every expected quarter with no pricing file
|
||||
and every NOC or crosswalk file the page lists but the hoard lacks."""
|
||||
pairs = [f"{q} pricing" for q in expected if q not in have["pricing"]]
|
||||
for q, kind in sorted(listed):
|
||||
if kind != "pricing" and q not in have.get(kind, set()):
|
||||
pairs.append(f"{q} {kind}")
|
||||
return pairs
|
||||
|
||||
|
||||
def gap_note(pairs: Sequence[str], show: int = 12) -> str:
|
||||
if not pairs:
|
||||
return ""
|
||||
note = "no hoarded file for: " + ", ".join(pairs[:show])
|
||||
if len(pairs) > show:
|
||||
note += f" (+{len(pairs) - show} more)"
|
||||
return note
|
||||
|
||||
|
||||
# ── reading a file ───────────────────────────────────────────────────
|
||||
#: observed CMS column names (lower-cased) -> ours
|
||||
COL_MAP = {
|
||||
"hcpcs code": "hcpcs_code",
|
||||
"hcpcs_code": "hcpcs_code",
|
||||
"short description": "short_description",
|
||||
"short_description": "short_description",
|
||||
"long description": "short_description", # the not-payable list
|
||||
"hcpcs code dosage": "dosage_or_unit",
|
||||
"hcpcs_code_dosage": "dosage_or_unit",
|
||||
"dosage": "dosage_or_unit",
|
||||
"dosage or unit": "dosage_or_unit",
|
||||
"payment limit": "payment_limit",
|
||||
"payment_limit": "payment_limit",
|
||||
"co-insurance percentage": "coinsurance_pct",
|
||||
"notes": "notes",
|
||||
"manufacturer": "manufacturer", # the not-payable list
|
||||
"drug generic name (trade name)": "drug_name", # NOC files
|
||||
"drug generic name": "drug_name",
|
||||
"drug name (generic)": "drug_name", # the 2005 NOC sheet
|
||||
}
|
||||
|
||||
_HEADER_STARTS = (
|
||||
"hcpcs code,",
|
||||
'"hcpcs code",',
|
||||
"drug generic name",
|
||||
'"drug generic name',
|
||||
"drug name (generic)",
|
||||
)
|
||||
|
||||
|
||||
def find_header_row(lines: Sequence[str]) -> int:
|
||||
"""0-based index of the column-header row, or -1. The header starts with
|
||||
``HCPCS Code,`` (or ``Drug Generic Name`` in NOC files); a note line that
|
||||
merely mentions HCPCS codes does not match."""
|
||||
for i, line in enumerate(lines):
|
||||
if line.strip().lower().startswith(_HEADER_STARTS):
|
||||
return i
|
||||
return -1
|
||||
|
||||
|
||||
def _skip_member(low: str) -> bool:
|
||||
return any(
|
||||
s in low for s in ("noc", "crosswalk", "ndc", "not payable", "not_payable")
|
||||
)
|
||||
|
||||
|
||||
def pick_pricing_file(names: Sequence[str]) -> str | None:
|
||||
"""The ASP pricing member of a zip: ``.csv`` before ``.xls``; NOC,
|
||||
crosswalk, NDC and "not payable" members are skipped."""
|
||||
candidates = [
|
||||
n
|
||||
for n in names
|
||||
if not _skip_member(n.lower()) and n.lower().endswith((".csv", ".xls", ".xlsx"))
|
||||
]
|
||||
csvs = [c for c in candidates if c.lower().endswith(".csv")]
|
||||
if csvs:
|
||||
for c in csvs:
|
||||
low = c.lower()
|
||||
if "payment limit" in low or "pricing" in low:
|
||||
return c
|
||||
return csvs[0]
|
||||
xlss = [c for c in candidates if c.lower().endswith((".xls", ".xlsx"))]
|
||||
if xlss:
|
||||
for c in xlss:
|
||||
low = c.lower()
|
||||
if "payment limit" in low or "pricing" in low or "asp" in low:
|
||||
return c
|
||||
return xlss[0]
|
||||
return None
|
||||
|
||||
|
||||
_MEMBER_MARKS = {
|
||||
"not-payable": ("not payable", "not_payable", "notpayable"),
|
||||
"noc": ("noc",),
|
||||
}
|
||||
|
||||
|
||||
def pick_member(names: Sequence[str], kind: str) -> str | None:
|
||||
"""The one member of kind ``"not-payable"`` or ``"noc"``: its ``.csv``
|
||||
when CMS ships a twin, else its ``.xls``."""
|
||||
if kind not in _MEMBER_MARKS:
|
||||
raise ValueError(f"no member kind {kind!r}")
|
||||
marks = _MEMBER_MARKS[kind]
|
||||
found = [
|
||||
n
|
||||
for n in names
|
||||
if n.lower().endswith((".csv", ".xls", ".xlsx"))
|
||||
and any(m in n.lower() for m in marks)
|
||||
and not ("crosswalk" in n.lower() or "ndc" in n.lower())
|
||||
]
|
||||
csvs = [n for n in found if n.lower().endswith(".csv")]
|
||||
return (csvs or found or [None])[0]
|
||||
|
||||
|
||||
def _cell(v: Any) -> str:
|
||||
if isinstance(v, float) and v == int(v):
|
||||
return str(int(v))
|
||||
return str(v).strip()
|
||||
|
||||
|
||||
def read_table(
|
||||
zf: zipfile.ZipFile,
|
||||
name: str,
|
||||
*,
|
||||
open_workbook: Callable[..., Any] | None = None,
|
||||
sheet: str = "",
|
||||
) -> list[dict[str, str]]:
|
||||
"""The rows of a zip member as ``{our column: text}`` (only the columns of
|
||||
:data:`COL_MAP`); preamble rows are skipped and blank rows dropped. A CSV
|
||||
is latin-1 with its header row found; an ``.xls`` goes through
|
||||
``xlrd.open_workbook`` (injectable for tests): its first sheet, or with
|
||||
*sheet* the first sheet whose name contains that text (the 2005 files keep
|
||||
the NOC table on a second sheet)."""
|
||||
if name.lower().endswith((".xls", ".xlsx")):
|
||||
return _read_xls(zf, name, open_workbook, sheet)
|
||||
lines = zf.read(name).decode("latin-1").splitlines()
|
||||
hdr = find_header_row(lines)
|
||||
if hdr < 0:
|
||||
return []
|
||||
reader = csv.reader(io.StringIO("\n".join(lines[hdr:]), newline=""))
|
||||
header = [COL_MAP.get(c.strip().lower()) for c in next(reader)]
|
||||
rows: list[dict[str, str]] = []
|
||||
for raw in reader:
|
||||
if len(raw) > len(header):
|
||||
if any(c.strip() for c in raw[len(header) :]):
|
||||
continue # more fields than columns: a malformed line
|
||||
raw = raw[: len(header)]
|
||||
row: dict[str, str] = {}
|
||||
for col, val in zip(header, raw, strict=False):
|
||||
if col and col not in row:
|
||||
row[col] = val
|
||||
if any(v.strip() for v in row.values()):
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _read_xls(
|
||||
zf: zipfile.ZipFile,
|
||||
name: str,
|
||||
open_workbook: Callable[..., Any] | None,
|
||||
sheet: str,
|
||||
) -> list[dict[str, str]]:
|
||||
if open_workbook is None:
|
||||
import xlrd
|
||||
|
||||
open_workbook = xlrd.open_workbook
|
||||
wb = open_workbook(file_contents=zf.read(name))
|
||||
index = 0
|
||||
if sheet:
|
||||
found = [i for i, n in enumerate(wb.sheet_names()) if sheet in n.lower()]
|
||||
if not found:
|
||||
return []
|
||||
index = found[0]
|
||||
sh = wb.sheet_by_index(index)
|
||||
hdr = -1
|
||||
for r in range(min(30, sh.nrows)):
|
||||
first = str(sh.cell_value(r, 0)).strip().lower()
|
||||
if first == "hcpcs code" or first.startswith(
|
||||
("drug generic name", "drug name (generic)")
|
||||
):
|
||||
hdr = r
|
||||
break
|
||||
if hdr < 0:
|
||||
return []
|
||||
header = [
|
||||
COL_MAP.get(_cell(sh.cell_value(hdr, c)).lower()) for c in range(sh.ncols)
|
||||
]
|
||||
rows: list[dict[str, str]] = []
|
||||
for r in range(hdr + 1, sh.nrows):
|
||||
vals = [_cell(sh.cell_value(r, c)) for c in range(sh.ncols)]
|
||||
if not any(vals):
|
||||
continue
|
||||
row: dict[str, str] = {}
|
||||
for col, val in zip(header, vals, strict=True):
|
||||
if col and col not in row:
|
||||
row[col] = val
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _number(raw: str | None) -> float | None:
|
||||
s = (raw or "").replace("$", "").replace(",", "").strip()
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _asp_rows(
|
||||
rows: Sequence[dict[str, str]],
|
||||
*,
|
||||
quarter: str,
|
||||
list_kind: str,
|
||||
member: str,
|
||||
source_url: str,
|
||||
item_key: str,
|
||||
) -> list[AspRow]:
|
||||
out: list[AspRow] = []
|
||||
for r in rows:
|
||||
code = r.get("hcpcs_code", "").strip().upper()
|
||||
if not code:
|
||||
continue
|
||||
payable = list_kind == "payment"
|
||||
out.append(
|
||||
AspRow(
|
||||
quarter=quarter,
|
||||
hcpcs=code,
|
||||
level=level(code),
|
||||
list_kind=list_kind,
|
||||
dosage=r.get("dosage_or_unit", ""),
|
||||
payment_limit=_number(r.get("payment_limit")) if payable else None,
|
||||
coinsurance_pct=_number(r.get("coinsurance_pct")) if payable else None,
|
||||
short_desc="" if licensed(code) else r.get("short_description", ""),
|
||||
notes=r.get("notes", ""),
|
||||
source_file=member,
|
||||
source_url=source_url,
|
||||
item_key=item_key,
|
||||
manufacturer=r.get("manufacturer", "").strip() if not payable else "",
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def parse_quarter_zip(
|
||||
path: Path,
|
||||
quarter: str,
|
||||
*,
|
||||
source_url: str,
|
||||
item_key: str = "",
|
||||
open_workbook: Callable[..., Any] | None = None,
|
||||
) -> tuple[list[AspRow], list[NocRow]]:
|
||||
"""The payment, not-payable and NOC rows of one quarter's zip: one member
|
||||
per kind. A licensed (CPT/CDT) code gets ``short_desc=""``."""
|
||||
asp: list[AspRow] = []
|
||||
noc: list[NocRow] = []
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
names = zf.namelist()
|
||||
pricing = pick_pricing_file(names)
|
||||
for kind, member in (
|
||||
("payment", pricing),
|
||||
("not-payable", pick_member(names, "not-payable")),
|
||||
):
|
||||
if member:
|
||||
asp += _asp_rows(
|
||||
read_table(zf, member, open_workbook=open_workbook),
|
||||
quarter=quarter,
|
||||
list_kind=kind,
|
||||
member=member,
|
||||
source_url=source_url,
|
||||
item_key=item_key,
|
||||
)
|
||||
member = pick_member(names, "noc")
|
||||
table: list[dict[str, str]] = []
|
||||
if member:
|
||||
table = read_table(zf, member, open_workbook=open_workbook)
|
||||
elif pricing and pricing.lower().endswith((".xls", ".xlsx")):
|
||||
table = read_table(zf, pricing, open_workbook=open_workbook, sheet="noc")
|
||||
member = f"{pricing}#NOC"
|
||||
if member:
|
||||
for r in table:
|
||||
name = r.get("drug_name", "").strip()
|
||||
if name:
|
||||
noc.append(
|
||||
NocRow(
|
||||
quarter=quarter,
|
||||
drug_name=name,
|
||||
dosage=r.get("dosage_or_unit", ""),
|
||||
payment_limit=_number(r.get("payment_limit")),
|
||||
source_file=member,
|
||||
source_url=source_url,
|
||||
item_key=item_key,
|
||||
)
|
||||
)
|
||||
return asp, noc
|
||||
@@ -5,18 +5,18 @@ from __future__ import annotations
|
||||
import re
|
||||
from datetime import date, timedelta
|
||||
|
||||
_L1 = re.compile(r"^\d{4}[0-9A-Z]$")
|
||||
_L2 = re.compile(r"^[ABCEGHJKLMPQRSTUV]\d{4}$")
|
||||
_CDT = re.compile(r"^D\d{4}$")
|
||||
_L1 = re.compile(r"[0-9]{4}[0-9A-Z]")
|
||||
_L2 = re.compile(r"[ABCEGHJKLMPQRSTUV][0-9]{4}")
|
||||
_CDT = re.compile(r"D[0-9]{4}")
|
||||
|
||||
|
||||
def level(code: str) -> str:
|
||||
"""``"I"`` (CPT), ``"II"`` (HCPCS Level II), ``"CDT"`` or ``""``."""
|
||||
if _L1.match(code):
|
||||
if _L1.fullmatch(code):
|
||||
return "I"
|
||||
if _L2.match(code):
|
||||
if _L2.fullmatch(code):
|
||||
return "II"
|
||||
if _CDT.match(code):
|
||||
if _CDT.fullmatch(code):
|
||||
return "CDT"
|
||||
return ""
|
||||
|
||||
@@ -30,9 +30,9 @@ def ndc11(raw: str) -> str | None:
|
||||
"""The 11-digit NDC of a 4-4-2, 5-3-2, 5-4-1 or 5-4-2 hyphenated code, or
|
||||
of 11 bare digits; anything else is ``None``."""
|
||||
s = raw.strip()
|
||||
if re.fullmatch(r"\d{11}", s):
|
||||
if re.fullmatch(r"[0-9]{11}", s):
|
||||
return s
|
||||
m = re.fullmatch(r"(\d+)-(\d+)-(\d+)", s)
|
||||
m = re.fullmatch(r"([0-9]+)-([0-9]+)-([0-9]+)", s)
|
||||
if not m:
|
||||
return None
|
||||
a, b, c = m.groups()
|
||||
@@ -48,27 +48,36 @@ def ndc11(raw: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
_APPL = {"N": "NDA", "A": "ANDA", "BLA": "BLA"}
|
||||
_APPL = {"N": "NDA", "NDA": "NDA", "A": "ANDA", "ANDA": "ANDA", "BLA": "BLA"}
|
||||
|
||||
|
||||
def appl_id(kind: str, number: str) -> str:
|
||||
"""``("N","21436")`` -> ``NDA021436``; an id already spelled out passes."""
|
||||
if re.fullmatch(r"[A-Z]+\d{6}", kind):
|
||||
if re.fullmatch(r"[A-Z]+[0-9]{6}", kind):
|
||||
return kind
|
||||
return f"{_APPL.get(kind, kind)}{int(number):06d}"
|
||||
if kind not in _APPL:
|
||||
raise ValueError(f"unknown application kind {kind!r}")
|
||||
return f"{_APPL[kind]}{int(number):06d}"
|
||||
|
||||
|
||||
#: the longest range :func:`expand` will list; a wider one is a data error
|
||||
MAX_RANGE = 500
|
||||
|
||||
|
||||
def expand(lo: str, hi: str) -> list[str]:
|
||||
"""Every code from *lo* to *hi* when they share a letter prefix and a
|
||||
numeric tail; otherwise just *lo*."""
|
||||
a = re.fullmatch(r"([A-Z]+)(\d+)", lo)
|
||||
b = re.fullmatch(r"([A-Z]+)(\d+)", hi)
|
||||
numeric tail; otherwise just *lo*. A range wider than :data:`MAX_RANGE`
|
||||
codes raises ``ValueError``."""
|
||||
a = re.fullmatch(r"([A-Z]+)([0-9]+)", lo)
|
||||
b = re.fullmatch(r"([A-Z]+)([0-9]+)", hi)
|
||||
if not a or not b or a.group(1) != b.group(1):
|
||||
return [lo]
|
||||
width = len(a.group(2))
|
||||
start, end = int(a.group(2)), int(b.group(2))
|
||||
if end < start or len(b.group(2)) != width:
|
||||
return [lo]
|
||||
if end - start + 1 > MAX_RANGE:
|
||||
raise ValueError(f"range too large: {lo}-{hi} is {end - start + 1} codes")
|
||||
return [f"{a.group(1)}{n:0{width}d}" for n in range(start, end + 1)]
|
||||
|
||||
|
||||
|
||||
@@ -7,11 +7,35 @@ tables. Nothing opens a database at import.
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import astuple, fields
|
||||
from dataclasses import astuple, dataclass, fields
|
||||
from typing import Any
|
||||
|
||||
_DDL = """
|
||||
CREATE SCHEMA IF NOT EXISTS drugs
|
||||
CREATE SCHEMA IF NOT EXISTS drugs;
|
||||
CREATE TABLE IF NOT EXISTS drugs.asp_payment (
|
||||
quarter VARCHAR,
|
||||
hcpcs VARCHAR,
|
||||
level VARCHAR,
|
||||
list_kind VARCHAR,
|
||||
dosage VARCHAR,
|
||||
payment_limit DOUBLE,
|
||||
coinsurance_pct DOUBLE,
|
||||
short_desc VARCHAR,
|
||||
notes VARCHAR,
|
||||
source_file VARCHAR,
|
||||
source_url VARCHAR,
|
||||
item_key VARCHAR,
|
||||
manufacturer VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS drugs.asp_noc (
|
||||
quarter VARCHAR,
|
||||
drug_name VARCHAR,
|
||||
dosage VARCHAR,
|
||||
payment_limit DOUBLE,
|
||||
source_file VARCHAR,
|
||||
source_url VARCHAR,
|
||||
item_key VARCHAR
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
@@ -61,3 +85,79 @@ def _select(con: Any, cls: type, table: str, where: str = "", params: Sequence =
|
||||
def _chunks(items: Sequence[Any], n: int = 500) -> Iterable[Sequence[Any]]:
|
||||
for i in range(0, len(items), n):
|
||||
yield items[i : i + n]
|
||||
|
||||
|
||||
# ── asp_payment / asp_noc ────────────────────────────────────────────
|
||||
@dataclass(frozen=True)
|
||||
class AspRow:
|
||||
"""One line of a quarterly Part B payment-limit file. ``list_kind`` is
|
||||
``payment`` or ``not-payable`` (no Medicaid rebate agreement; carries the
|
||||
file's manufacturer). ``short_desc`` is ``""`` for a licensed (CPT/CDT)
|
||||
code. ``payment_limit`` is the CMS payment allowance (ASP + 6%), never
|
||||
the ASP itself."""
|
||||
|
||||
quarter: str
|
||||
hcpcs: str
|
||||
level: str
|
||||
list_kind: str
|
||||
dosage: str
|
||||
payment_limit: float | None
|
||||
coinsurance_pct: float | None
|
||||
short_desc: str
|
||||
notes: str
|
||||
source_file: str
|
||||
source_url: str
|
||||
item_key: str
|
||||
manufacturer: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NocRow:
|
||||
"""One line of a quarterly not-otherwise-classified drug pricing file."""
|
||||
|
||||
quarter: str
|
||||
drug_name: str
|
||||
dosage: str
|
||||
payment_limit: float | None
|
||||
source_file: str
|
||||
source_url: str
|
||||
item_key: str
|
||||
|
||||
|
||||
def write_asp(con: Any, quarter: str, rows: Sequence[AspRow]) -> int:
|
||||
"""Replace *quarter* of ``drugs.asp_payment`` with *rows*."""
|
||||
ensure_tables(con)
|
||||
con.execute("DELETE FROM drugs.asp_payment WHERE quarter = ?", [quarter])
|
||||
return _insert(con, "drugs.asp_payment", rows)
|
||||
|
||||
|
||||
def read_asp(
|
||||
con: Any, *, hcpcs: str | None = None, quarter: str | None = None
|
||||
) -> list[AspRow]:
|
||||
where, params = [], []
|
||||
if hcpcs is not None:
|
||||
where.append("hcpcs = ?")
|
||||
params.append(hcpcs)
|
||||
if quarter is not None:
|
||||
where.append("quarter = ?")
|
||||
params.append(quarter)
|
||||
rows = _select(con, AspRow, "drugs.asp_payment", " AND ".join(where), params)
|
||||
return sorted(
|
||||
(AspRow(*r) for r in rows),
|
||||
key=lambda r: (r.quarter, r.hcpcs, r.list_kind, r.dosage),
|
||||
)
|
||||
|
||||
|
||||
def write_noc(con: Any, quarter: str, rows: Sequence[NocRow]) -> int:
|
||||
"""Replace *quarter* of ``drugs.asp_noc`` with *rows*."""
|
||||
ensure_tables(con)
|
||||
con.execute("DELETE FROM drugs.asp_noc WHERE quarter = ?", [quarter])
|
||||
return _insert(con, "drugs.asp_noc", rows)
|
||||
|
||||
|
||||
def read_noc(con: Any, quarter: str | None = None) -> list[NocRow]:
|
||||
where, params = ("quarter = ?", [quarter]) if quarter is not None else ("", [])
|
||||
rows = _select(con, NocRow, "drugs.asp_noc", where, params)
|
||||
return sorted(
|
||||
(NocRow(*r) for r in rows), key=lambda r: (r.quarter, r.drug_name, r.dosage)
|
||||
)
|
||||
|
||||
@@ -2,12 +2,20 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
|
||||
import duckdb
|
||||
import httpx
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
import cli.drugs as dg
|
||||
from bib.store import Store
|
||||
from cli import app
|
||||
from drugs import asp as A
|
||||
from drugs import http as dh
|
||||
from drugs import tables as T
|
||||
from market import tables as MT
|
||||
|
||||
@@ -140,3 +148,376 @@ def test_status_gap_read_error_propagates(env, monkeypatch):
|
||||
|
||||
monkeypatch.setattr(dg, "_read", lambda: _Con())
|
||||
assert isinstance(runner.invoke(app, ["drugs", "status"]).exception, RuntimeError)
|
||||
|
||||
|
||||
# ── stack drugs asp ──────────────────────────────────────────────────
|
||||
PRICING = (
|
||||
"Payment Allowance Limits,,,,,,,,,,,\r\n"
|
||||
"HCPCS Code,Short Description,HCPCS Code Dosage,Payment Limit,"
|
||||
"Co-insurance Percentage,Vaccine AWP%,Vaccine Limit,Blood AWP%,Blood limit,"
|
||||
"Clotting Factor,Notes\r\n"
|
||||
"90371,x,1 ML,140.209,20.000,,,,,,\r\n"
|
||||
"J9035,Bevacizumab injection,10 MG,66.123,20.000,,,,,,\r\n"
|
||||
)
|
||||
NP = (
|
||||
"HCPCS Code,Long Description,Manufacturer\r\nJ7330,Chondrocytes implant,Vericel\r\n"
|
||||
)
|
||||
NOC = "Drug Generic Name (Trade Name),Dosage,Payment Limit,Notes\r\nExamplumab,10 MG,12.5,\r\n"
|
||||
|
||||
|
||||
def _zip(members):
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
for n, b in members.items():
|
||||
zf.writestr(n, b)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
PAGE = """<html><body>
|
||||
<a href="/files/zip/july-2026-medicare-part-b-payment-limit-files.zip">July 2026 Medicare Part B Payment Limit Files</a>
|
||||
<a href="/files/zip/july-2026-ndc-hcpcs-crosswalk.zip">July 2026 NDC-HCPCS Crosswalk</a>
|
||||
<a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/x/downloads/2018-April-ASP-Pricing-File.zip">April 2018 ASP Pricing File</a>
|
||||
<a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip">April 2018 NOC Pricing File</a>
|
||||
<a href="/files/zip/2026-2027-seasonal-vaccine-price-file.zip">2026-2027 Seasonal Vaccine Price File</a>
|
||||
</body></html>"""
|
||||
|
||||
ROUTES = {
|
||||
"/medicare/payment/part-b-drugs/asp-pricing-files": PAGE.encode(),
|
||||
"/files/zip/july-2026-medicare-part-b-payment-limit-files.zip": _zip(
|
||||
{
|
||||
"section 508 version of July 2026 Payment Limit File.csv": PRICING,
|
||||
"section 508 version of July 2026 Drugs Not Payable.csv": NP,
|
||||
}
|
||||
),
|
||||
"/files/zip/july-2026-ndc-hcpcs-crosswalk.zip": _zip({"x.csv": "a"}),
|
||||
"/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip": _zip({"NOC.csv": NOC}),
|
||||
"/Medicare/x/downloads/2018-April-ASP-Pricing-File.zip": _zip({"ASP.csv": PRICING}),
|
||||
}
|
||||
|
||||
|
||||
class _NoWait:
|
||||
def wait(self):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def asp_env(env, monkeypatch, tmp_path):
|
||||
c, published = env
|
||||
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
|
||||
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
|
||||
dh._MANIFESTS.clear()
|
||||
store = Store(":memory:", storage_dir=tmp_path / "st")
|
||||
monkeypatch.setattr(dg, "_store", lambda: store)
|
||||
seen: list[httpx.Request] = []
|
||||
|
||||
def handler(req):
|
||||
seen.append(req)
|
||||
body = ROUTES.get(req.url.path)
|
||||
return (
|
||||
httpx.Response(200, content=body)
|
||||
if body is not None
|
||||
else httpx.Response(404)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
dh,
|
||||
"client",
|
||||
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
monkeypatch.setattr(A, "today", lambda: __import__("datetime").date(2026, 8, 1))
|
||||
return c, published, seen, tmp_path, store
|
||||
|
||||
|
||||
def test_asp_cli_fetch_write(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
MT.write_gap(c, "cms-asp:asp", "stale")
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"drugs",
|
||||
"asp",
|
||||
"--fetch",
|
||||
"--write",
|
||||
"--quarter",
|
||||
"2026-Q3",
|
||||
"--quarter",
|
||||
"2018-Q2",
|
||||
],
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
pay = [r for r in T.read_asp(c) if r.quarter == "2026-Q3"]
|
||||
assert {r.quarter for r in T.read_asp(c)} == {"2018-Q2", "2026-Q3"}
|
||||
assert [(r.quarter, r.hcpcs, r.list_kind) for r in pay] == [
|
||||
("2026-Q3", "90371", "payment"),
|
||||
("2026-Q3", "J7330", "not-payable"),
|
||||
("2026-Q3", "J9035", "payment"),
|
||||
]
|
||||
assert pay[0].short_desc == "" and pay[2].short_desc == "Bevacizumab injection"
|
||||
assert pay[1].manufacturer == "Vericel"
|
||||
assert all(
|
||||
r.item_key
|
||||
and r.source_url.startswith("https://www.cms.gov/files/zip/july-2026")
|
||||
for r in pay
|
||||
)
|
||||
noc = T.read_noc(c)
|
||||
assert [(n.quarter, n.drug_name, n.payment_limit) for n in noc] == [
|
||||
("2018-Q2", "Examplumab", 12.5)
|
||||
]
|
||||
assert (
|
||||
noc[0].source_url
|
||||
== "https://www.cms.gov/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip"
|
||||
)
|
||||
assert {
|
||||
k
|
||||
for (k,) in c.execute(
|
||||
"SELECT DISTINCT table_name FROM cms.ingest_log"
|
||||
).fetchall()
|
||||
} == {
|
||||
"drugs.asp_payment",
|
||||
"drugs.asp_noc",
|
||||
}
|
||||
assert published == [True]
|
||||
assert [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"] == []
|
||||
assert "stale" not in " ".join(g.note for g in MT.read_gaps(c))
|
||||
# every request carried the honest user agent through the real client factory
|
||||
hoard = root / "cms" / "asp"
|
||||
assert (hoard / "asp_2026-Q3.zip").exists()
|
||||
assert (hoard / "noc" / "noc_2018-Q2.zip").exists()
|
||||
assert (hoard / "xwalk" / "xwalk_2026-Q3.zip").exists()
|
||||
assert (root / "drugs" / "cms" / "asp-pricing-files.html").exists()
|
||||
manifest = [
|
||||
json.loads(x) for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
|
||||
]
|
||||
assert len(manifest) == 5
|
||||
# vaccine zip is never fetched
|
||||
assert not any("vaccine" in str(r.url) for r in seen)
|
||||
assert "2026-Q3" in res.output and "not-payable" in res.output
|
||||
cites = [i for i in store.list_items() if "ASP" in i.title]
|
||||
assert len(cites) == 3
|
||||
# a second run fetches nothing but the page
|
||||
n = len(seen)
|
||||
again = runner.invoke(
|
||||
app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3", "--quarter", "2018-Q2"]
|
||||
)
|
||||
assert again.exit_code == 0, again.output
|
||||
assert len(seen) == n + 1
|
||||
|
||||
|
||||
def test_asp_cli_gap_note_written_for_missing(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"drugs",
|
||||
"asp",
|
||||
"--fetch",
|
||||
"--write",
|
||||
"--quarter",
|
||||
"2026-Q3",
|
||||
"--quarter",
|
||||
"2026-Q2",
|
||||
],
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"]
|
||||
assert len(gaps) == 1 and "2026-Q2 pricing" in gaps[0].note
|
||||
assert "gap: cms-asp:asp: no hoarded file for: 2026-Q2 pricing" in res.output
|
||||
|
||||
|
||||
def test_asp_cli_dry_run_lists_links(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
first = runner.invoke(app, ["drugs", "asp", "--fetch", "--dry-run"])
|
||||
assert first.exit_code == 0, first.output
|
||||
assert [r.url.path for r in seen] == [
|
||||
"/medicare/payment/part-b-drugs/asp-pricing-files"
|
||||
]
|
||||
out = first.output
|
||||
assert (
|
||||
"pricing 2026-Q3 fetch https://www.cms.gov/files/zip/july-2026-medicare-part-b-payment-limit-files.zip"
|
||||
in out
|
||||
)
|
||||
assert "noc 2018-Q2 fetch" in out and "crosswalk 2026-Q3 fetch" in out
|
||||
assert "vaccine" not in out
|
||||
assert not (root / "cms").exists()
|
||||
assert T.table_counts(c) == {}
|
||||
assert published == []
|
||||
# --write is ignored by a dry run
|
||||
res = runner.invoke(
|
||||
app, ["drugs", "asp", "--dry-run", "--write", "--quarter", "2018-Q2"]
|
||||
)
|
||||
assert res.exit_code == 0 and published == [] and "noc 2018-Q2 fetch" in res.output
|
||||
assert "crosswalk 2026-Q3" not in res.output
|
||||
|
||||
|
||||
def test_asp_cli_refusal_prints_gap(asp_env, monkeypatch):
|
||||
c, published, seen, root, store = asp_env
|
||||
monkeypatch.setattr(
|
||||
dh,
|
||||
"client",
|
||||
lambda source, key=None: httpx.Client(
|
||||
transport=httpx.MockTransport(lambda r: httpx.Response(403))
|
||||
),
|
||||
)
|
||||
res = runner.invoke(app, ["drugs", "asp", "--fetch"])
|
||||
assert res.exit_code == 0, res.output
|
||||
lines = [x for x in res.output.splitlines() if x.strip()]
|
||||
assert len(lines) == 1 and lines[0].startswith("gap: cms-asp:asp:")
|
||||
assert published == []
|
||||
assert T.table_counts(c) == {} and not (root / "cms").exists()
|
||||
res = runner.invoke(app, ["drugs", "asp", "--fetch", "--write"])
|
||||
assert [g.source for g in MT.read_gaps(c)] == ["cms-asp:asp"]
|
||||
assert published == [True]
|
||||
assert T.table_counts(c) == {}
|
||||
|
||||
|
||||
def test_asp_cli_refusal_on_a_zip_stops_fetching(asp_env, monkeypatch):
|
||||
c, published, seen, root, store = asp_env
|
||||
|
||||
def handler(req):
|
||||
if req.url.path.endswith("asp-pricing-files"):
|
||||
return httpx.Response(200, content=PAGE.encode())
|
||||
return httpx.Response(403)
|
||||
|
||||
monkeypatch.setattr(
|
||||
dh,
|
||||
"client",
|
||||
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
res = runner.invoke(
|
||||
app, ["drugs", "asp", "--fetch", "--write", "--quarter", "2026-Q3"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: cms-asp:asp: refused" in res.output
|
||||
assert "2026-Q3 pricing" in res.output
|
||||
assert T.read_asp(c) == []
|
||||
|
||||
|
||||
def test_asp_cli_parse_only_reads_hoard_without_cites(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
assert (
|
||||
runner.invoke(
|
||||
app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3"]
|
||||
).exit_code
|
||||
== 0
|
||||
)
|
||||
assert T.table_counts(c) == {}
|
||||
seen.clear()
|
||||
res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q3"])
|
||||
assert res.exit_code == 0 and seen == []
|
||||
assert "2026-Q3" in res.output and "3 payment" not in res.output
|
||||
assert "2 payment" in res.output and "1 not-payable" in res.output
|
||||
assert store.list_items() == [] and published == []
|
||||
|
||||
|
||||
def test_asp_cli_unreadable_zip_is_a_gap(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
d = root / "cms" / "asp"
|
||||
d.mkdir(parents=True)
|
||||
(d / "asp_2026-Q3.zip").write_bytes(b"not a zip")
|
||||
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
|
||||
assert res.exit_code == 0, res.output
|
||||
gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"]
|
||||
assert gaps and "2026-Q3 pricing unreadable" in gaps[0].note
|
||||
assert T.read_asp(c) == []
|
||||
|
||||
|
||||
def test_asp_cli_source_url_falls_back_to_csv(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
d = root / "cms" / "asp"
|
||||
d.mkdir(parents=True)
|
||||
(d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING}))
|
||||
(root / "cms" / "asp_source_urls.csv").write_text(
|
||||
"quarter,url\n2026-Q3,https://cms.test/canon.zip\n"
|
||||
)
|
||||
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert {r.source_url for r in T.read_asp(c)} == {"https://cms.test/canon.zip"}
|
||||
assert all(r.item_key for r in T.read_asp(c))
|
||||
|
||||
|
||||
def test_asp_cli_no_url_no_cite(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
d = root / "cms" / "asp"
|
||||
d.mkdir(parents=True)
|
||||
(d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING}))
|
||||
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert {(r.source_url, r.item_key) for r in T.read_asp(c)} == {("", "")}
|
||||
|
||||
|
||||
def test_asp_cli_refresh_refetches(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
assert (
|
||||
runner.invoke(
|
||||
app, ["drugs", "asp", "--fetch", "--quarter", "2018-Q2"]
|
||||
).exit_code
|
||||
== 0
|
||||
)
|
||||
n = len(seen)
|
||||
ROUTES_BEFORE = dict(ROUTES)
|
||||
res = runner.invoke(
|
||||
app, ["drugs", "asp", "--fetch", "--refresh", "--quarter", "2018-Q2"]
|
||||
)
|
||||
assert (
|
||||
res.exit_code == 0 and len(seen) == n + 3
|
||||
) # page, the pricing zip and the NOC zip
|
||||
assert ROUTES == ROUTES_BEFORE
|
||||
|
||||
|
||||
def test_asp_cli_unfiltered_uses_expected_quarters(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
res = runner.invoke(app, ["drugs", "asp"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: cms-asp:asp: no hoarded file for: 2005-Q1 pricing" in res.output
|
||||
assert "(+" in res.output and published == []
|
||||
assert seen == []
|
||||
|
||||
|
||||
def test_asp_today_is_real_date():
|
||||
import datetime
|
||||
|
||||
assert A.today() == datetime.date.today()
|
||||
|
||||
|
||||
def test_asp_cli_rejects_bad_quarter(asp_env):
|
||||
res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q5"])
|
||||
assert res.exit_code != 0 and "not a quarter" in res.output
|
||||
|
||||
|
||||
def test_asp_cli_unreadable_noc_zip_is_a_gap(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
d = root / "cms" / "asp" / "noc"
|
||||
d.mkdir(parents=True)
|
||||
(d / "noc_2018-Q2.zip").write_bytes(b"not a zip")
|
||||
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2018-Q2"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "2018-Q2 noc unreadable" in res.output
|
||||
assert T.read_noc(c) == []
|
||||
|
||||
|
||||
def test_asp_cli_bundled_noc_from_pricing_zip(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
d = root / "cms" / "asp"
|
||||
d.mkdir(parents=True)
|
||||
(d / "asp_2006-Q2.zip").write_bytes(
|
||||
_zip({"ASPbyHCPCS.csv": PRICING, "ASPNOC.csv": NOC})
|
||||
)
|
||||
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2006-Q2"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert [n.drug_name for n in T.read_noc(c)] == ["Examplumab"]
|
||||
assert "1 noc" in res.output
|
||||
|
||||
|
||||
def test_asp_cli_dry_run_without_page(asp_env):
|
||||
res = runner.invoke(app, ["drugs", "asp", "--dry-run"])
|
||||
assert res.exit_code == 0 and "no page hoarded; run with --fetch" in res.output
|
||||
|
||||
|
||||
def test_asp_cli_noc_only_quarter_is_listed(asp_env):
|
||||
c, published, seen, root, store = asp_env
|
||||
d = root / "cms" / "asp" / "noc"
|
||||
d.mkdir(parents=True)
|
||||
(d / "noc_2018-Q2.zip").write_bytes(_zip({"NOC.csv": NOC}))
|
||||
res = runner.invoke(app, ["drugs", "asp", "--quarter", "2018-Q2"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "2018-Q2: 1 noc" in res.output
|
||||
|
||||
134
tests/dev/test_ingest_asp_parity.py
Normal file
134
tests/dev/test_ingest_asp_parity.py
Normal file
@@ -0,0 +1,134 @@
|
||||
"""The skin-substitute ingest reads the same ASP parser as the drug tier, and
|
||||
its output is unchanged (1,974 rows, 2009-Q1..2025-Q4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import importlib.util
|
||||
import math
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
ASP_DIR = ROOT / "data" / "cms" / "asp"
|
||||
SKIN_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv"
|
||||
REF_CSV = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv"
|
||||
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"_ingest_asp", ROOT / "dev" / "scripts" / "ingest_asp.py"
|
||||
)
|
||||
ingest = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(ingest)
|
||||
|
||||
needs_data = pytest.mark.skipif(
|
||||
not (SKIN_CSV.exists() and any(ASP_DIR.glob("asp_*.zip"))),
|
||||
reason="needs the hoarded data/cms/asp files",
|
||||
)
|
||||
|
||||
|
||||
def _legacy_pick(names: list[str]) -> str | None:
|
||||
"""The selection the dev script made before the shared parser (frozen)."""
|
||||
skip = ["noc", "crosswalk", "ndc", "not payable", "not_payable"]
|
||||
cand = [
|
||||
n
|
||||
for n in names
|
||||
if not any(s in n.lower() for s in skip)
|
||||
and n.lower().endswith((".csv", ".xls", ".xlsx"))
|
||||
]
|
||||
csvs = [c for c in cand if c.lower().endswith(".csv")]
|
||||
if csvs:
|
||||
for c in csvs:
|
||||
if "payment limit" in c.lower() or "pricing" in c.lower():
|
||||
return c
|
||||
return csvs[0]
|
||||
xlss = [c for c in cand if c.lower().endswith((".xls", ".xlsx"))]
|
||||
if xlss:
|
||||
for c in xlss:
|
||||
low = c.lower()
|
||||
if "payment limit" in low or "pricing" in low or "asp" in low:
|
||||
return c
|
||||
return xlss[0]
|
||||
return None
|
||||
|
||||
|
||||
@needs_data
|
||||
def test_pick_pricing_file_over_every_hoarded_zip():
|
||||
from drugs import asp
|
||||
|
||||
n = 0
|
||||
for z in sorted(ASP_DIR.glob("asp_*.zip")):
|
||||
names = zipfile.ZipFile(z).namelist()
|
||||
want = _legacy_pick(names)
|
||||
assert ingest.pick_pricing_file(names) == want, z.name
|
||||
assert asp.pick_pricing_file(names) == want, z.name
|
||||
n += 1
|
||||
assert n >= 86
|
||||
|
||||
|
||||
@needs_data
|
||||
def test_skin_rows_match_hoarded_csv():
|
||||
df = ingest.build_skin_df()
|
||||
got = sorted(
|
||||
df.to_dict("records"),
|
||||
key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
|
||||
)
|
||||
with SKIN_CSV.open(newline="") as fh:
|
||||
want = sorted(
|
||||
csv.DictReader(fh),
|
||||
key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
|
||||
)
|
||||
assert len(want) == 1974
|
||||
assert len(got) == len(want), "an extra or missing skin row appeared"
|
||||
for g, w in zip(got, want, strict=True):
|
||||
for col in (
|
||||
"quarter",
|
||||
"hcpcs_code",
|
||||
"short_description",
|
||||
"dosage_or_unit",
|
||||
"source_file",
|
||||
"source_url",
|
||||
):
|
||||
gv = g[col]
|
||||
gv = "" if gv is None or (isinstance(gv, float) and math.isnan(gv)) else gv
|
||||
assert gv == w[col], (g["quarter"], g["hcpcs_code"], col)
|
||||
gp = g["payment_limit"]
|
||||
if w["payment_limit"] == "":
|
||||
assert gp is None or math.isnan(gp)
|
||||
else:
|
||||
assert abs(gp - float(w["payment_limit"])) < 1e-9
|
||||
assert abs(g["asp_per_unit"] - float(w["asp_per_unit"])) < 1e-9
|
||||
|
||||
|
||||
def test_reference_csv_carries_no_cpt_descriptor_text():
|
||||
with REF_CSV.open(newline="") as fh:
|
||||
rows = list(csv.DictReader(fh))
|
||||
assert len(rows) == 34
|
||||
assert list(rows[0]) == [
|
||||
"hcpcs_code",
|
||||
"short_descriptor",
|
||||
"code_type",
|
||||
"body_site",
|
||||
"wound_size",
|
||||
"effective_date",
|
||||
"termination_date",
|
||||
"status",
|
||||
"notes",
|
||||
]
|
||||
cpt = [r for r in rows if "CPT" in r["code_type"]]
|
||||
assert {r["hcpcs_code"] for r in cpt} >= {"15271", "15278", "15330", "15430"}
|
||||
assert all(r["short_descriptor"] == "" for r in cpt)
|
||||
# CMS's own HCPCS Level II text stays
|
||||
level2 = {r["hcpcs_code"]: r["short_descriptor"] for r in rows if r not in cpt}
|
||||
assert level2["C5271"].startswith("Low cost skin substitute graft")
|
||||
assert level2["G0440"] and level2["C9363"]
|
||||
assert all(r["short_descriptor"] for r in rows if r not in cpt)
|
||||
|
||||
|
||||
def test_refuses_to_replace_table_with_nothing():
|
||||
import pandas as pd
|
||||
|
||||
with pytest.raises(RuntimeError, match="no skin-substitute rows"):
|
||||
ingest.require_rows(pd.DataFrame())
|
||||
ingest.require_rows(pd.DataFrame({"a": [1]}))
|
||||
17
tests/drugs/fixtures/asp_page_head.html
Normal file
17
tests/drugs/fixtures/asp_page_head.html
Normal file
@@ -0,0 +1,17 @@
|
||||
<!doctype html>
|
||||
<html><head><title>ASP Pricing Files</title></head>
|
||||
<body>
|
||||
<nav><a href="/medicare/payment/part-b-drugs/asp-reporting">ASP Reporting</a></nav>
|
||||
<main>
|
||||
<ul>
|
||||
<li><a href="/files/zip/october-2026-medicare-part-b-payment-limit-files-final.zip">October 2026 Medicare Part B Payment Limit Files</a></li>
|
||||
<li><a href="/files/zip/october-2026-ndc-hcpcs-crosswalk-final.zip">October 2026 NDC-HCPCS Crosswalk</a></li>
|
||||
<li><a href="/files/zip/2026-2027-seasonal-vaccine-price-file.zip">2026-2027 Seasonal Vaccine Price File</a></li>
|
||||
<li><a href="/files/zip/2026-2027-ndc-hcpcs-crosswalk-seasonal-vaccines.zip">2026-2027 NDC-HCPCS Crosswalk for Seasonal Vaccines</a></li>
|
||||
<li><a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/McrPartBDrugAvgSalesPrice/Downloads/2018-January-ASP-Pricing-File.zip">January 2018 ASP Pricing File - Updated 03/02/2018</a></li>
|
||||
<li><a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/McrPartBDrugAvgSalesPrice/Downloads/2018-April-NOC-Pricing-File.zip">April 2018 NOC Pricing File - Updated 05/02/2018</a></li>
|
||||
<li><a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/McrPartBDrugAvgSalesPrice/downloads/jan05aspbyhcpcsv6_033106.zip">Jan 2005 ASP Pricing File and ASP NOC Pricing File - Updated 03/31/2006</a></li>
|
||||
<li><a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/McrPartBDrugAvgSalesPrice/downloads/HCPCSCrosswalk0105.zip">Jan 2005 ASP NDC-HCPCS Crosswalk</a></li>
|
||||
</ul>
|
||||
</main>
|
||||
</body></html>
|
||||
694
tests/drugs/test_asp.py
Normal file
694
tests/drugs/test_asp.py
Normal file
@@ -0,0 +1,694 @@
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from drugs import asp
|
||||
from drugs import http as dh
|
||||
from drugs import tables as T
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "asp_page_head.html"
|
||||
|
||||
|
||||
class _NoWait:
|
||||
def wait(self):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def root(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
|
||||
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
|
||||
dh._MANIFESTS.clear()
|
||||
return tmp_path
|
||||
|
||||
|
||||
# ── the page ─────────────────────────────────────────────────────────
|
||||
def test_parse_asp_page_kinds_and_quarters():
|
||||
links = asp.parse_asp_page(FIXTURE.read_text())
|
||||
got = {(lk.quarter, lk.kind): lk.url for lk in links}
|
||||
assert got[("2026-Q4", "pricing")] == (
|
||||
"https://www.cms.gov/files/zip/"
|
||||
"october-2026-medicare-part-b-payment-limit-files-final.zip"
|
||||
)
|
||||
assert got[("2026-Q4", "crosswalk")] == (
|
||||
"https://www.cms.gov/files/zip/october-2026-ndc-hcpcs-crosswalk-final.zip"
|
||||
)
|
||||
vaccine = [lk for lk in links if lk.kind == "vaccine"]
|
||||
assert len(vaccine) == 2 and {lk.quarter for lk in vaccine} == {""}
|
||||
assert ("2018-Q2", "noc") in got and ("2018-Q1", "pricing") in got
|
||||
assert got[("2005-Q1", "pricing")] == (
|
||||
"https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/"
|
||||
"McrPartBDrugAvgSalesPrice/downloads/jan05aspbyhcpcsv6_033106.zip"
|
||||
)
|
||||
assert ("2005-Q1", "crosswalk") in got
|
||||
assert len(links) == 8
|
||||
assert links[0].text == "October 2026 Medicare Part B Payment Limit Files"
|
||||
|
||||
|
||||
def test_parse_asp_page_edge_cases():
|
||||
html = """
|
||||
<a href="/files/zip/january-2023-asp-pricing-file.zip">2023 ASP Pricing File (ZIP)</a>
|
||||
<a href="/files/zip/april-2023-noc-pricing-file.zip">2023 NOC Pricing File</a>
|
||||
<a href="/x/2018-July-ASP-Pricing-File.zip">ASP Pricing File</a>
|
||||
<a href="/files/zip/jan06.zip">January 2006 ASP Pricing File/ASP NOC Pricing File/ASP NDC-HCPCS Crosswalk</a>
|
||||
<a href="/files/zip/jul06.zip">July 2006 ASP NDC-HCPCS Crosswalk - Updated</a>
|
||||
<a href="/files/zip/oct09.zip">October 2009 ASP NOC Pricing File</a>
|
||||
<a href="/files/zip/mystery.zip">Something else entirely</a>
|
||||
<a href="/files/zip/no-month.zip">ASP Pricing File</a>
|
||||
<a href="/files/pdf/guide.pdf">Guide to the files (PDF)</a>
|
||||
<a>no href at all</a>
|
||||
<a href="/files/zip/oct05.zip">Oct. 2005 ASP Pricing File and ASP NOC Pricing File</a>
|
||||
"""
|
||||
links = asp.parse_asp_page(html)
|
||||
got = [(lk.quarter, lk.kind) for lk in links]
|
||||
assert got == [
|
||||
("2023-Q1", "pricing"),
|
||||
("2023-Q2", "noc"),
|
||||
("2018-Q3", "pricing"),
|
||||
("2006-Q1", "pricing"),
|
||||
("2006-Q3", "crosswalk"),
|
||||
("2009-Q4", "noc"),
|
||||
("", "other"),
|
||||
("", "other"),
|
||||
("2005-Q4", "pricing"),
|
||||
]
|
||||
assert links[1].text == "2023 NOC Pricing File"
|
||||
|
||||
|
||||
def test_direct_url_unwraps_license():
|
||||
p = "/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/x/downloads/a.zip"
|
||||
want = "https://www.cms.gov" + p
|
||||
assert asp.direct_url(f"https://www.cms.gov/apps/ama/license.asp?file={p}") == want
|
||||
assert asp.direct_url(f"/apps/ama/license.asp?file={p}") == want
|
||||
assert asp.direct_url(f"https://www.cms.gov/license/ama?file={p}") == want
|
||||
assert asp.direct_url("/files/zip/a.zip") == "https://www.cms.gov/files/zip/a.zip"
|
||||
assert asp.direct_url("https://x.test/a.zip") == "https://x.test/a.zip"
|
||||
|
||||
|
||||
def test_hoard_path_and_newest_per_quarter(root):
|
||||
def lk(kind):
|
||||
return asp.AspLink("2026-Q2", kind, "u", "t")
|
||||
|
||||
assert asp.hoard_path(lk("pricing")) == root / "cms/asp/asp_2026-Q2.zip"
|
||||
assert asp.hoard_path(lk("noc")) == root / "cms/asp/noc/noc_2026-Q2.zip"
|
||||
assert asp.hoard_path(lk("crosswalk")) == root / "cms/asp/xwalk/xwalk_2026-Q2.zip"
|
||||
with pytest.raises(ValueError):
|
||||
asp.hoard_path(lk("vaccine"))
|
||||
|
||||
|
||||
def test_one_zip_per_quarter_newest_wins(tmp_path):
|
||||
a = tmp_path / "asp_2026-Q2.zip"
|
||||
b = tmp_path / "asp_2026-Q2.ab12cd34.zip"
|
||||
c = tmp_path / "asp_2026-Q1.zip"
|
||||
junk = tmp_path / "asp_notes.zip"
|
||||
for p in (a, b, c, junk):
|
||||
p.write_bytes(b"x")
|
||||
os.utime(a, (1_000, 1_000))
|
||||
os.utime(b, (2_000, 2_000))
|
||||
got = asp.newest_per_quarter([a, b, c, junk])
|
||||
assert got == {"2026-Q1": c, "2026-Q2": b}
|
||||
# the older copy still wins when it is the newer file
|
||||
os.utime(a, (3_000, 3_000))
|
||||
assert asp.newest_per_quarter([a, b])["2026-Q2"] == a
|
||||
assert asp.newest_per_quarter([]) == {}
|
||||
|
||||
|
||||
def test_expected_quarters():
|
||||
q = asp.expected_quarters(date(2026, 11, 13))
|
||||
assert q[0] == "2005-Q1" and q[-1] == "2026-Q4" and len(q) == 88
|
||||
assert asp.expected_quarters(date(2005, 1, 1)) == ["2005-Q1"]
|
||||
assert asp.expected_quarters(date(2005, 3, 31)) == ["2005-Q1"]
|
||||
assert asp.expected_quarters(date(2006, 2, 1)) == [
|
||||
"2005-Q1",
|
||||
"2005-Q2",
|
||||
"2005-Q3",
|
||||
"2005-Q4",
|
||||
"2006-Q1",
|
||||
]
|
||||
|
||||
|
||||
# ── the member logic ─────────────────────────────────────────────────
|
||||
def test_col_map_covers_dev_script_and_new_columns():
|
||||
dev = {
|
||||
"hcpcs code": "hcpcs_code",
|
||||
"hcpcs_code": "hcpcs_code",
|
||||
"short description": "short_description",
|
||||
"short_description": "short_description",
|
||||
"hcpcs code dosage": "dosage_or_unit",
|
||||
"hcpcs_code_dosage": "dosage_or_unit",
|
||||
"dosage": "dosage_or_unit",
|
||||
"dosage or unit": "dosage_or_unit",
|
||||
"payment limit": "payment_limit",
|
||||
"payment_limit": "payment_limit",
|
||||
}
|
||||
for k, v in dev.items():
|
||||
assert asp.COL_MAP[k] == v
|
||||
assert asp.COL_MAP["co-insurance percentage"] == "coinsurance_pct"
|
||||
assert asp.COL_MAP["notes"] == "notes"
|
||||
assert asp.COL_MAP["long description"] == "short_description"
|
||||
assert asp.COL_MAP["manufacturer"] == "manufacturer"
|
||||
assert asp.COL_MAP["drug generic name (trade name)"] == "drug_name"
|
||||
|
||||
|
||||
def test_find_header_row():
|
||||
assert asp.find_header_row(["a", "Note: HCPCS code x,y", "HCPCS Code,Foo"]) == 2
|
||||
assert asp.find_header_row(['"HCPCS Code","Foo"']) == 0
|
||||
assert asp.find_header_row(["", "Drug Generic Name (Trade Name),Dosage"]) == 1
|
||||
assert asp.find_header_row(["Drug Name (Generic),Dosage"]) == 0
|
||||
assert asp.find_header_row(["nothing here"]) == -1
|
||||
assert asp.find_header_row([]) == -1
|
||||
|
||||
|
||||
NAMES_2015Q3 = [
|
||||
"Jul 15 ASP Pricing File - updated 090115.xls",
|
||||
"section 508 version of Jul 15 ASP Pricing File - updated 090115.csv",
|
||||
]
|
||||
NAMES_2026Q2 = [
|
||||
"April 2026 Drugs Not Payable Under Part B 030926.xls",
|
||||
"April 2026 Medicare Part B Payment Limit File 032426.xls",
|
||||
"section 508 version of April 2026 Drugs Not Payable Under Part B 030926.csv",
|
||||
"section 508 version of April 2026 Medicare Part B Payment Limit File 032426.csv",
|
||||
]
|
||||
|
||||
|
||||
def test_pick_pricing_file_matches_dev_script():
|
||||
assert asp.pick_pricing_file(["Jan05ASPbyHCPCSv6_033106.xls"]) == (
|
||||
"Jan05ASPbyHCPCSv6_033106.xls"
|
||||
)
|
||||
assert asp.pick_pricing_file(NAMES_2015Q3) == NAMES_2015Q3[1]
|
||||
assert asp.pick_pricing_file(
|
||||
[
|
||||
"January 2026 Medicare Part B Payment Limit File updated 022526.xls",
|
||||
"section 508 version of January 2026 Medicare Part B Payment Limit "
|
||||
"File updated 022526.csv",
|
||||
]
|
||||
).endswith("022526.csv")
|
||||
assert asp.pick_pricing_file(NAMES_2026Q2) == NAMES_2026Q2[3]
|
||||
assert asp.pick_pricing_file(["a.xls", "b.pdf"]) == "a.xls"
|
||||
assert asp.pick_pricing_file(["ASP.xlsx", "other.xls"]) == "ASP.xlsx"
|
||||
assert (
|
||||
asp.pick_pricing_file(["other.csv", "payment limit.csv"]) == "payment limit.csv"
|
||||
)
|
||||
assert asp.pick_pricing_file(["x.csv"]) == "x.csv"
|
||||
assert (
|
||||
asp.pick_pricing_file(["Oct 2025 NOC Pricing File.csv", "readme.txt"]) is None
|
||||
)
|
||||
assert asp.pick_pricing_file(["Crosswalk.csv", "ndc.xls"]) is None
|
||||
|
||||
|
||||
def test_pick_member_prefers_csv_over_its_xls_twin():
|
||||
# P7: one member per kind per zip
|
||||
assert asp.pick_member(NAMES_2026Q2, "not-payable") == NAMES_2026Q2[2]
|
||||
assert asp.pick_member(NAMES_2026Q2, "noc") is None
|
||||
names = [
|
||||
"1 - Apr06_ASPbyHCPCS_122107.xls",
|
||||
"1 - Apr06_ASPbyHCPCS_122107.csv",
|
||||
"1 - Apr06_ASPNOC_20Sep06.xls",
|
||||
"1 - Apr06_ASPNOC_20Sep06.csv",
|
||||
"April06ASPNDC-HCPCSCrosswalk_20Mar06.xls",
|
||||
]
|
||||
assert asp.pick_member(names, "noc") == "1 - Apr06_ASPNOC_20Sep06.csv"
|
||||
assert asp.pick_member(names[2:3] + names[4:], "noc") == names[2]
|
||||
assert asp.pick_member(["Drugs_Not_Payable.xlsx"], "not-payable") == (
|
||||
"Drugs_Not_Payable.xlsx"
|
||||
)
|
||||
assert asp.pick_member(["x.csv"], "noc") is None
|
||||
with pytest.raises(ValueError):
|
||||
asp.pick_member(names, "crosswalk")
|
||||
|
||||
|
||||
# ── reading tables ───────────────────────────────────────────────────
|
||||
PRICING_HEAD = (
|
||||
"Payment Allowance Limits for Medicare Part B Drugs,,,,,,,,,,,\r\n"
|
||||
",,,,,,,,,,,\r\n"
|
||||
"Note 1: HCPCS code lines mention HCPCS code here,,,,,,,,,,,\r\n"
|
||||
"HCPCS Code,Short Description,HCPCS Code Dosage,Payment Limit,"
|
||||
"Co-insurance Percentage,Vaccine AWP%,Vaccine Limit,Blood AWP%,Blood limit,"
|
||||
"Clotting Factor,Notes\r\n"
|
||||
)
|
||||
PRICING_ROWS = (
|
||||
"90371,x,1 ML,140.209,20.000,,,,,,\r\n"
|
||||
'J9035,Bevacizumab injection,10 MG,"$66.123",20.000,,,,,,\r\n'
|
||||
)
|
||||
NP_CSV = (
|
||||
"Drugs Not Payable Under Part B,,,\r\n"
|
||||
",,,\r\n"
|
||||
"HCPCS Code,Long Description,Manufacturer,,,,\r\n"
|
||||
'J7330," Autologous cultured chondrocytes, implant ",Vericel Corporation,,,,\r\n'
|
||||
"90999,x,Acme,,,,\r\n"
|
||||
",,,,,,\r\n"
|
||||
)
|
||||
NOC_CSV = (
|
||||
"Payment Allowance Limits for NOC Drugs,,,\r\n"
|
||||
",,,\r\n"
|
||||
"Drug Generic Name (Trade Name),Dosage,Payment Limit,Notes\r\n"
|
||||
"Examplumab,10 MG,12.5,\r\n"
|
||||
"Vasopressin (Long Grove),1 UNIT,$2.462,\r\n"
|
||||
",,,\r\n"
|
||||
)
|
||||
|
||||
|
||||
def make_zip(path: Path, members: dict[str, str | bytes]) -> Path:
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
for name, body in members.items():
|
||||
zf.writestr(
|
||||
name, body if isinstance(body, bytes) else body.encode("latin-1")
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def test_cpt_short_description_never_stored(tmp_path):
|
||||
z = make_zip(
|
||||
tmp_path / "asp_2026-Q2.zip",
|
||||
{
|
||||
"section 508 version of April 2026 Payment Limit File.csv": PRICING_HEAD
|
||||
+ PRICING_ROWS
|
||||
},
|
||||
)
|
||||
rows, noc = asp.parse_quarter_zip(
|
||||
z, "2026-Q2", source_url="https://cms.test/a.zip", item_key="K1"
|
||||
)
|
||||
assert noc == []
|
||||
by = {r.hcpcs: r for r in rows}
|
||||
assert by["90371"].level == "I" and by["90371"].short_desc == ""
|
||||
assert by["90371"].dosage == "1 ML" and by["90371"].payment_limit == 140.209
|
||||
assert by["90371"].coinsurance_pct == 20.0
|
||||
j = by["J9035"]
|
||||
assert j.level == "II" and j.short_desc == "Bevacizumab injection"
|
||||
assert j.payment_limit == 66.123 and j.list_kind == "payment"
|
||||
assert j.quarter == "2026-Q2" and j.item_key == "K1"
|
||||
assert j.source_url == "https://cms.test/a.zip"
|
||||
assert j.source_file == "section 508 version of April 2026 Payment Limit File.csv"
|
||||
assert "Bevacizumab" not in " ".join(r.short_desc for r in rows if r.level == "I")
|
||||
|
||||
|
||||
def test_numbers_codes_and_blank_rows(tmp_path):
|
||||
body = (
|
||||
PRICING_HEAD
|
||||
+ ' j9035 ,Bevacizumab,10 MG,"$1,066.50",,,,,,,see note\r\n'
|
||||
+ "J0000,Unparseable,1 MG,n/a,abc,,,,,,\r\n"
|
||||
+ "J0001,Short row\r\n"
|
||||
+ ",,,,,,,,,,\r\n"
|
||||
+ " ,blank code,,,,,,,,,\r\n"
|
||||
+ "J0002,Too many,1 MG,1.0,20,,,,,,,extra,fields\r\n"
|
||||
+ "D0120,dental,1,2.0,,,,,,,\r\n"
|
||||
)
|
||||
z = make_zip(tmp_path / "asp_2020-Q1.zip", {"ASP Pricing File.csv": body})
|
||||
rows, _ = asp.parse_quarter_zip(z, "2020-Q1", source_url="")
|
||||
by = {r.hcpcs: r for r in rows}
|
||||
# a line with more fields than columns is malformed and skipped
|
||||
assert set(by) == {"J9035", "J0000", "J0001", "D0120"}
|
||||
assert by["J9035"].payment_limit == 1066.5 and by["J9035"].notes == "see note"
|
||||
assert by["J0000"].payment_limit is None and by["J0000"].coinsurance_pct is None
|
||||
assert by["J0001"].dosage == "" and by["J0001"].short_desc == "Short row"
|
||||
assert by["D0120"].level == "CDT" and by["D0120"].short_desc == ""
|
||||
assert by["J9035"].item_key == "" and by["J9035"].source_url == ""
|
||||
|
||||
|
||||
def test_not_payable_rows_kept(tmp_path):
|
||||
z = make_zip(
|
||||
tmp_path / "asp_2026-Q2.zip",
|
||||
{
|
||||
"April 2026 Drugs Not Payable Under Part B 030926.xls": b"not read",
|
||||
"section 508 version of April 2026 Drugs Not Payable Under Part B 030926.csv": NP_CSV,
|
||||
"section 508 version of April 2026 Payment Limit File.csv": PRICING_HEAD
|
||||
+ PRICING_ROWS,
|
||||
},
|
||||
)
|
||||
rows, _ = asp.parse_quarter_zip(z, "2026-Q2", source_url="u")
|
||||
np_rows = [r for r in rows if r.list_kind == "not-payable"]
|
||||
assert len(np_rows) == 2 and len(rows) == 4
|
||||
j = next(r for r in np_rows if r.hcpcs == "J7330")
|
||||
assert j.payment_limit is None and j.coinsurance_pct is None
|
||||
assert j.manufacturer == "Vericel Corporation"
|
||||
assert j.short_desc == " Autologous cultured chondrocytes, implant "
|
||||
assert j.source_file.endswith("Under Part B 030926.csv")
|
||||
lvl1 = next(r for r in np_rows if r.hcpcs == "90999")
|
||||
assert lvl1.level == "I" and lvl1.short_desc == "" and lvl1.manufacturer == "Acme"
|
||||
assert all(r.manufacturer == "" for r in rows if r.list_kind == "payment")
|
||||
|
||||
|
||||
def test_noc_member_parsed(tmp_path):
|
||||
z = make_zip(
|
||||
tmp_path / "noc_2025-Q4.zip",
|
||||
{
|
||||
"Oct 2025 NOC Pricing File 090425.xls": b"not read",
|
||||
"section 508 version of Oct 2025 NOC Pricing File 090425.csv": NOC_CSV,
|
||||
},
|
||||
)
|
||||
rows, noc = asp.parse_quarter_zip(
|
||||
z, "2025-Q4", source_url="https://cms.test/n.zip", item_key="K2"
|
||||
)
|
||||
assert rows == []
|
||||
assert [(n.drug_name, n.dosage, n.payment_limit) for n in noc] == [
|
||||
("Examplumab", "10 MG", 12.5),
|
||||
("Vasopressin (Long Grove)", "1 UNIT", 2.462),
|
||||
]
|
||||
assert noc[0].quarter == "2025-Q4" and noc[0].item_key == "K2"
|
||||
assert noc[0].source_url == "https://cms.test/n.zip"
|
||||
assert noc[0].source_file.endswith("090425.csv")
|
||||
|
||||
|
||||
def test_bundled_zip_gives_both_and_one_member_per_kind(tmp_path):
|
||||
# 2006-style zip: pricing, NOC and crosswalk, each as .xls and .csv twins
|
||||
z = make_zip(
|
||||
tmp_path / "asp_2006-Q2.zip",
|
||||
{
|
||||
"1 - Apr06_ASPbyHCPCS.xls": b"x",
|
||||
"1 - Apr06_ASPbyHCPCS.csv": PRICING_HEAD + PRICING_ROWS,
|
||||
"1 - Apr06_ASPNOC.xls": b"x",
|
||||
"1 - Apr06_ASPNOC.csv": NOC_CSV,
|
||||
"April06ASPNDC-HCPCSCrosswalk.csv": "HCPCS Code,NDC\r\nJ9035,1\r\n",
|
||||
},
|
||||
)
|
||||
rows, noc = asp.parse_quarter_zip(z, "2006-Q2", source_url="")
|
||||
assert [r.hcpcs for r in rows] == ["90371", "J9035"]
|
||||
assert len(noc) == 2
|
||||
|
||||
|
||||
def test_header_missing_gives_no_rows(tmp_path):
|
||||
z = make_zip(
|
||||
tmp_path / "asp_2020-Q1.zip", {"ASP Pricing File.csv": "a,b\r\n1,2\r\n"}
|
||||
)
|
||||
assert asp.parse_quarter_zip(z, "2020-Q1", source_url="") == ([], [])
|
||||
z2 = make_zip(tmp_path / "asp_2020-Q2.zip", {"readme.txt": "hi"})
|
||||
assert asp.parse_quarter_zip(z2, "2020-Q2", source_url="") == ([], [])
|
||||
|
||||
|
||||
class _Sheet:
|
||||
def __init__(self, grid):
|
||||
self.grid = grid
|
||||
self.nrows = len(grid)
|
||||
self.ncols = max(len(r) for r in grid)
|
||||
|
||||
def cell_value(self, r, c):
|
||||
row = self.grid[r]
|
||||
return row[c] if c < len(row) else ""
|
||||
|
||||
|
||||
class _Book:
|
||||
def __init__(self, grid, sheets=None):
|
||||
self.sheets = [("Sheet1", _Sheet(grid))] + [
|
||||
(name, _Sheet(g)) for name, g in (sheets or [])
|
||||
]
|
||||
|
||||
def sheet_names(self):
|
||||
return [n for n, _ in self.sheets]
|
||||
|
||||
def sheet_by_index(self, i):
|
||||
return self.sheets[i][1]
|
||||
|
||||
|
||||
XLS_GRID = [
|
||||
["Payment Allowance Limits"],
|
||||
[""],
|
||||
["HCPCS Code", "Short Description", "HCPCS Code Dosage", "Payment Limit"],
|
||||
[90371.0, "x", "1 ML", 140.209],
|
||||
["J9035 ", "Bevacizumab injection", "10 MG", 66.123],
|
||||
["", "", "", ""],
|
||||
["J0001", "Whole", "1 MG", 5.0],
|
||||
]
|
||||
|
||||
|
||||
def test_xls_member_through_fake_workbook(tmp_path):
|
||||
z = make_zip(
|
||||
tmp_path / "asp_2005-Q1.zip", {"Jan05ASPbyHCPCSv6_033106.xls": b"BIFF"}
|
||||
)
|
||||
seen = {}
|
||||
|
||||
def open_wb(**kw):
|
||||
seen.update(kw)
|
||||
return _Book(XLS_GRID)
|
||||
|
||||
rows, noc = asp.parse_quarter_zip(
|
||||
z, "2005-Q1", source_url="u", open_workbook=open_wb
|
||||
)
|
||||
assert seen == {"file_contents": b"BIFF"}
|
||||
assert noc == []
|
||||
assert [
|
||||
(r.hcpcs, r.level, r.short_desc, r.dosage, r.payment_limit) for r in rows
|
||||
] == [
|
||||
("90371", "I", "", "1 ML", 140.209),
|
||||
("J9035", "II", "Bevacizumab injection", "10 MG", 66.123),
|
||||
("J0001", "II", "Whole", "1 MG", 5.0),
|
||||
]
|
||||
csv_z = make_zip(
|
||||
tmp_path / "asp_2005-Q2.zip",
|
||||
{"ASP.csv": PRICING_HEAD + PRICING_ROWS},
|
||||
)
|
||||
csv_rows, _ = asp.parse_quarter_zip(csv_z, "2005-Q2", source_url="u")
|
||||
assert [(r.hcpcs, r.dosage, r.payment_limit) for r in csv_rows] == [
|
||||
(r.hcpcs, r.dosage, r.payment_limit) for r in rows[:2]
|
||||
]
|
||||
|
||||
|
||||
def test_xls_without_header_and_default_workbook(tmp_path, monkeypatch):
|
||||
z = make_zip(tmp_path / "asp_2005-Q1.zip", {"ASP.xls": b"BIFF"})
|
||||
assert asp.parse_quarter_zip(
|
||||
z, "2005-Q1", source_url="", open_workbook=lambda **kw: _Book([["a"], ["b"]])
|
||||
) == ([], [])
|
||||
import sys
|
||||
import types
|
||||
|
||||
fake = types.SimpleNamespace(open_workbook=lambda **kw: _Book(XLS_GRID))
|
||||
monkeypatch.setitem(sys.modules, "xlrd", fake)
|
||||
rows, _ = asp.parse_quarter_zip(z, "2005-Q1", source_url="")
|
||||
assert len(rows) == 3
|
||||
|
||||
|
||||
def test_noc_xls_member(tmp_path):
|
||||
grid = [
|
||||
["Drug Generic Name (Trade Name)", "Dosage", "Payment Limit"],
|
||||
["Abatacept", "250 MG", 477.0],
|
||||
["", "", ""],
|
||||
]
|
||||
z = make_zip(tmp_path / "noc_2008-Q1.zip", {"Jan08 NOC.xls": b"BIFF"})
|
||||
_, noc = asp.parse_quarter_zip(
|
||||
z, "2008-Q1", source_url="", open_workbook=lambda **kw: _Book(grid)
|
||||
)
|
||||
assert [(n.drug_name, n.dosage, n.payment_limit) for n in noc] == [
|
||||
("Abatacept", "250 MG", 477.0)
|
||||
]
|
||||
|
||||
|
||||
# ── manifest and gaps ────────────────────────────────────────────────
|
||||
def test_manifest_urls_and_known_urls(root):
|
||||
assert asp.manifest_urls() == set()
|
||||
d = root / "cms" / "asp"
|
||||
d.mkdir(parents=True)
|
||||
f = d / "asp_2026-Q1.zip"
|
||||
f.write_bytes(b"z")
|
||||
dh.manifest(f, "https://cms.test/q1.zip")
|
||||
assert asp.manifest_urls() == {"https://cms.test/q1.zip"}
|
||||
(root / "cms" / "asp_source_urls.csv").write_text(
|
||||
"quarter,url\n2026-Q2,https://cms.test/q2.zip\n"
|
||||
)
|
||||
assert asp.canonical_urls() == {"2026-Q2": "https://cms.test/q2.zip"}
|
||||
|
||||
|
||||
def test_plan_fetch_actions(root):
|
||||
d = root / "cms" / "asp"
|
||||
d.mkdir(parents=True)
|
||||
(d / "asp_2026-Q1.zip").write_bytes(b"q1")
|
||||
(d / "asp_2026-Q2.zip").write_bytes(b"q2")
|
||||
(d / "asp_2025-Q4.zip").write_bytes(b"q4")
|
||||
dh.manifest(d / "asp_2025-Q4.zip", "https://cms.test/2025q4.zip")
|
||||
(root / "cms" / "asp_source_urls.csv").write_text(
|
||||
"quarter,url\n2026-Q2,https://cms.test/q2-a.zip\n2026-Q1,https://cms.test/q1.zip\n"
|
||||
)
|
||||
|
||||
def lk(q, kind, url):
|
||||
return asp.AspLink(q, kind, url, "t")
|
||||
|
||||
links = [
|
||||
lk("2025-Q4", "pricing", "https://cms.test/2025q4.zip"), # in manifest
|
||||
lk("2026-Q1", "pricing", "https://cms.test/q1.zip"), # canonical, adopt
|
||||
lk("2026-Q2", "pricing", "https://cms.test/q2-b.zip"), # updated posting
|
||||
lk("2026-Q3", "pricing", "https://cms.test/q3.zip"), # new
|
||||
lk("2026-Q3", "noc", "https://cms.test/n3.zip"),
|
||||
lk("", "vaccine", "https://cms.test/v.zip"),
|
||||
]
|
||||
plan = asp.plan_fetch(links, refresh=False)
|
||||
assert [(p.link.quarter, p.link.kind, p.action) for p in plan] == [
|
||||
("2025-Q4", "pricing", "have"),
|
||||
("2026-Q1", "pricing", "adopt"),
|
||||
("2026-Q2", "pricing", "fetch-beside"),
|
||||
("2026-Q3", "pricing", "fetch"),
|
||||
("2026-Q3", "noc", "fetch"),
|
||||
]
|
||||
assert plan[3].dest == d / "asp_2026-Q3.zip"
|
||||
forced = asp.plan_fetch(links, refresh=True)
|
||||
assert {p.action for p in forced[:3]} == {"fetch-beside"}
|
||||
assert forced[3].action == "fetch"
|
||||
|
||||
|
||||
def test_gap_pairs_and_note():
|
||||
have = {"pricing": {"2026-Q1", "2026-Q3"}, "noc": set(), "crosswalk": set()}
|
||||
listed = {("2026-Q3", "noc"), ("2026-Q3", "crosswalk")}
|
||||
exp = ["2026-Q1", "2026-Q2", "2026-Q3"]
|
||||
pairs = asp.gap_pairs(exp, have, listed)
|
||||
assert pairs == ["2026-Q2 pricing", "2026-Q3 crosswalk", "2026-Q3 noc"]
|
||||
assert asp.gap_note(pairs).startswith("no hoarded file for: 2026-Q2 pricing")
|
||||
assert asp.gap_note([]) == ""
|
||||
many = [f"{2000 + i}-Q1 pricing" for i in range(40)]
|
||||
note = asp.gap_note(many)
|
||||
assert note.endswith("(+28 more)") and "2011-Q1 pricing" in note
|
||||
|
||||
|
||||
def test_missing_quarter_is_gap_not_zero(tmp_path):
|
||||
z1 = make_zip(
|
||||
tmp_path / "asp_2026-Q1.zip", {"ASP.csv": PRICING_HEAD + PRICING_ROWS}
|
||||
)
|
||||
z3 = make_zip(
|
||||
tmp_path / "asp_2026-Q3.zip", {"ASP.csv": PRICING_HEAD + PRICING_ROWS}
|
||||
)
|
||||
got = asp.newest_per_quarter([z1, z3])
|
||||
have = {"pricing": set(got), "noc": set(), "crosswalk": set()}
|
||||
pairs = asp.gap_pairs(["2026-Q1", "2026-Q2", "2026-Q3"], have, set())
|
||||
assert "2026-Q2 pricing" in asp.gap_note(pairs)
|
||||
con = duckdb.connect(":memory:")
|
||||
for q, z in (("2026-Q1", z1), ("2026-Q3", z3)):
|
||||
rows, _ = asp.parse_quarter_zip(z, q, source_url="")
|
||||
asp.write_asp(con, q, rows)
|
||||
assert asp.read_asp(con, quarter="2026-Q2") == []
|
||||
assert len(asp.read_asp(con, quarter="2026-Q1")) == 2
|
||||
|
||||
|
||||
# ── a transport-level updated posting lands beside ──────────────────
|
||||
def test_updated_posting_lands_beside(root):
|
||||
d = root / "cms" / "asp"
|
||||
d.mkdir(parents=True)
|
||||
old = d / "asp_2026-Q2.zip"
|
||||
old.write_bytes(b"old posting")
|
||||
dh.manifest(old, "https://cms.test/A.zip")
|
||||
link = asp.AspLink("2026-Q2", "pricing", "https://cms.test/B.zip", "t")
|
||||
(plan,) = asp.plan_fetch([link], refresh=False)
|
||||
assert plan.action == "fetch-beside"
|
||||
seen = []
|
||||
|
||||
def handler(req):
|
||||
seen.append(str(req.url))
|
||||
return httpx.Response(200, content=b"new posting")
|
||||
|
||||
cl = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
out = asp.apply_plan(cl, plan)
|
||||
assert seen == ["https://cms.test/B.zip"]
|
||||
assert out != old and out.read_bytes() == b"new posting"
|
||||
assert old.read_bytes() == b"old posting"
|
||||
urls = [
|
||||
json.loads(x)["url"]
|
||||
for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
|
||||
]
|
||||
assert urls == ["https://cms.test/A.zip", "https://cms.test/B.zip"]
|
||||
# it is now the newest copy of the quarter
|
||||
assert asp.newest_per_quarter(d.glob("asp_*.zip")) == {"2026-Q2": out}
|
||||
# a second plan sees B in the manifest
|
||||
(again,) = asp.plan_fetch([link], refresh=False)
|
||||
assert again.action == "have"
|
||||
|
||||
|
||||
def test_apply_plan_adopt_and_have(root):
|
||||
d = root / "cms" / "asp"
|
||||
d.mkdir(parents=True)
|
||||
f = d / "asp_2026-Q1.zip"
|
||||
f.write_bytes(b"q1")
|
||||
(root / "cms" / "asp_source_urls.csv").write_text(
|
||||
"quarter,url\n2026-Q1,https://cms.test/q1.zip\n"
|
||||
)
|
||||
link = asp.AspLink("2026-Q1", "pricing", "https://cms.test/q1.zip", "t")
|
||||
(plan,) = asp.plan_fetch([link], refresh=False)
|
||||
assert plan.action == "adopt"
|
||||
boom = httpx.Client(transport=httpx.MockTransport(lambda r: pytest.fail("fetched")))
|
||||
assert asp.apply_plan(boom, plan) == f
|
||||
rec = json.loads((root / "drugs/MANIFEST.jsonl").read_text())
|
||||
assert (
|
||||
rec["url"] == "https://cms.test/q1.zip" and "asp_source_urls.csv" in rec["note"]
|
||||
)
|
||||
(plan2,) = asp.plan_fetch([link], refresh=False)
|
||||
assert plan2.action == "have" and asp.apply_plan(boom, plan2) == f
|
||||
|
||||
|
||||
def test_page_hoard(root):
|
||||
seen = []
|
||||
|
||||
def handler(req):
|
||||
seen.append(req.headers["user-agent"])
|
||||
return httpx.Response(200, content=b"<html>one</html>")
|
||||
|
||||
cl = httpx.Client(
|
||||
headers={"User-Agent": "ua"}, transport=httpx.MockTransport(handler)
|
||||
)
|
||||
p = asp.fetch_page(cl)
|
||||
assert (
|
||||
p == root / "drugs/cms/asp-pricing-files.html"
|
||||
and p.read_bytes() == b"<html>one</html>"
|
||||
)
|
||||
assert asp.hoarded_page() == p
|
||||
cl2 = httpx.Client(
|
||||
transport=httpx.MockTransport(
|
||||
lambda r: httpx.Response(200, content=b"<html>two</html>")
|
||||
)
|
||||
)
|
||||
p2 = asp.fetch_page(cl2)
|
||||
assert p2 != p and p.read_bytes() == b"<html>one</html>"
|
||||
assert asp.hoarded_page() == p2
|
||||
assert (
|
||||
asp.ASP_PAGE
|
||||
== "https://www.cms.gov/medicare/payment/part-b-drugs/asp-pricing-files"
|
||||
)
|
||||
|
||||
|
||||
def test_hoarded_page_absent(root):
|
||||
assert asp.hoarded_page() is None
|
||||
|
||||
|
||||
# ── tables ───────────────────────────────────────────────────────────
|
||||
def test_write_noc_replaces_quarter():
|
||||
con = duckdb.connect(":memory:")
|
||||
a = T.NocRow("2025-Q4", "A", "1 MG", 1.0, "f", "u", "k")
|
||||
b = T.NocRow("2025-Q3", "B", "1 MG", None, "f", "u", "k")
|
||||
asp.write_noc(con, "2025-Q4", [a])
|
||||
asp.write_noc(con, "2025-Q3", [b])
|
||||
asp.write_noc(con, "2025-Q4", [T.NocRow("2025-Q4", "C", "", 2.0, "f", "u", "")])
|
||||
assert [r.drug_name for r in asp.read_noc(con)] == ["B", "C"]
|
||||
assert [r.drug_name for r in asp.read_noc(con, "2025-Q3")] == ["B"]
|
||||
|
||||
|
||||
def test_2005_style_noc_sheet_inside_the_pricing_workbook(tmp_path):
|
||||
noc_grid = [
|
||||
["Note"],
|
||||
["Drug Name (Generic)", "Dosage", "Payment Limit", "Notes"],
|
||||
["Allopurinol Sodium", "500 MG", "455.714", "Revised"],
|
||||
["Amikacin Sulfate", "50 MG", "$0.492", ""],
|
||||
]
|
||||
z = make_zip(tmp_path / "asp_2005-Q1.zip", {"Jan05ASPbyHCPCSv6.xls": b"BIFF"})
|
||||
rows, noc = asp.parse_quarter_zip(
|
||||
z,
|
||||
"2005-Q1",
|
||||
source_url="u",
|
||||
open_workbook=lambda **kw: _Book(XLS_GRID, [("Jan05_NOC_rev", noc_grid)]),
|
||||
)
|
||||
assert len(rows) == 3
|
||||
assert [(n.drug_name, n.payment_limit) for n in noc] == [
|
||||
("Allopurinol Sodium", 455.714),
|
||||
("Amikacin Sulfate", 0.492),
|
||||
]
|
||||
assert noc[0].source_file == "Jan05ASPbyHCPCSv6.xls#NOC"
|
||||
# no NOC sheet: no NOC rows
|
||||
_, none = asp.parse_quarter_zip(
|
||||
z, "2005-Q1", source_url="u", open_workbook=lambda **kw: _Book(XLS_GRID)
|
||||
)
|
||||
assert none == []
|
||||
|
||||
|
||||
def test_manufacturer_is_stripped(tmp_path):
|
||||
body = "HCPCS Code,Long Description,Manufacturer\r\nJ0209,Some drug,\tHope Pharmaceuticals \r\n"
|
||||
z = make_zip(tmp_path / "asp_2026-Q4.zip", {"Drugs Not Payable.csv": body})
|
||||
(row,), _ = asp.parse_quarter_zip(z, "2026-Q4", source_url="")
|
||||
assert row.manufacturer == "Hope Pharmaceuticals" and row.list_kind == "not-payable"
|
||||
@@ -1,5 +1,7 @@
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from drugs import codes
|
||||
|
||||
|
||||
@@ -34,7 +36,8 @@ def test_appl_id():
|
||||
assert codes.appl_id("A", "90123") == "ANDA090123"
|
||||
assert codes.appl_id("BLA", "125057") == "BLA125057"
|
||||
assert codes.appl_id("NDA021436", "") == "NDA021436"
|
||||
assert codes.appl_id("X", "5") == "X000005"
|
||||
assert codes.appl_id("NDA", "21436") == "NDA021436"
|
||||
assert codes.appl_id("ANDA", "90123") == "ANDA090123"
|
||||
|
||||
|
||||
def test_expand():
|
||||
@@ -54,3 +57,31 @@ def test_quarters_and_fy():
|
||||
assert codes.quarter_end("2026-Q1") == date(2026, 3, 31)
|
||||
assert codes.fy_effective("2026") == date(2025, 10, 1)
|
||||
assert codes.fy_effective("2026-04") == date(2026, 4, 1)
|
||||
|
||||
|
||||
def test_level_and_ndc_reject_trailing_newline_and_fullwidth_digits():
|
||||
assert codes.level("J9035\n") == ""
|
||||
assert codes.level("90371") == ""
|
||||
assert codes.level("90371") == ""
|
||||
assert codes.licensed("90371\n") is False
|
||||
assert codes.ndc11("000214338011") is None
|
||||
assert codes.ndc11("00021433801") is None
|
||||
assert codes.ndc11("0002-1433-80") is None
|
||||
assert codes.ndc11("0002-1433-80") == "00002143380"
|
||||
|
||||
|
||||
def test_appl_id_rejects_unknown_kinds():
|
||||
with pytest.raises(ValueError, match="unknown application kind"):
|
||||
codes.appl_id("X", "5")
|
||||
with pytest.raises(ValueError):
|
||||
codes.appl_id("", "5")
|
||||
|
||||
|
||||
def test_expand_caps_range_size():
|
||||
assert len(codes.expand("J0001", "J0500")) == 500
|
||||
assert codes.MAX_RANGE == 500
|
||||
with pytest.raises(ValueError, match="range too large"):
|
||||
codes.expand("J0001", "J9999")
|
||||
with pytest.raises(ValueError):
|
||||
codes.expand("J0001", "J0501")
|
||||
assert codes.expand("J9001", "J9002") == ["J9001"]
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
MODULES = ["drugs.http", "drugs.codes", "drugs.tables"]
|
||||
MODULES = ["drugs.http", "drugs.codes", "drugs.tables", "drugs.asp"]
|
||||
|
||||
_PROBE = """
|
||||
import builtins, importlib, sys
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import duckdb
|
||||
import pytest
|
||||
|
||||
from drugs import tables as T
|
||||
|
||||
@@ -16,7 +17,7 @@ def test_table_names_empty():
|
||||
con = duckdb.connect(":memory:")
|
||||
assert T.table_counts(con) == {}
|
||||
T.ensure_tables(con)
|
||||
assert T.table_counts(con) == {}
|
||||
assert T.table_counts(con) == {"drugs.asp_noc": 0, "drugs.asp_payment": 0}
|
||||
|
||||
|
||||
def test_helpers():
|
||||
@@ -37,3 +38,92 @@ def test_helpers():
|
||||
1,
|
||||
]
|
||||
con.execute("CREATE TABLE drugs.x (a INTEGER)") if False else None
|
||||
|
||||
|
||||
ROUNDTRIP = {
|
||||
"asp_payment": (
|
||||
T.write_asp,
|
||||
T.read_asp,
|
||||
[
|
||||
T.AspRow(
|
||||
"2026-Q2",
|
||||
"J9035",
|
||||
"II",
|
||||
"payment",
|
||||
"10 MG",
|
||||
66.123,
|
||||
20.0,
|
||||
"Bevacizumab injection",
|
||||
"n",
|
||||
"f.csv",
|
||||
"https://x.test/a.zip",
|
||||
"K1",
|
||||
),
|
||||
T.AspRow(
|
||||
"2026-Q2",
|
||||
"J7330",
|
||||
"II",
|
||||
"not-payable",
|
||||
"",
|
||||
None,
|
||||
None,
|
||||
"Autologous cultured chondrocytes, implant",
|
||||
"",
|
||||
"np.csv",
|
||||
"https://x.test/a.zip",
|
||||
"K1",
|
||||
"Vericel Corporation",
|
||||
),
|
||||
],
|
||||
),
|
||||
"asp_noc": (
|
||||
T.write_noc,
|
||||
T.read_noc,
|
||||
[
|
||||
T.NocRow(
|
||||
"2026-Q2",
|
||||
"Examplumab",
|
||||
"10 MG",
|
||||
12.5,
|
||||
"n.csv",
|
||||
"https://x.test/n.zip",
|
||||
"K2",
|
||||
),
|
||||
T.NocRow("2026-Q2", "Other", "", None, "n.csv", "https://x.test/n.zip", ""),
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(ROUNDTRIP))
|
||||
def test_roundtrip(name):
|
||||
write, read, rows = ROUNDTRIP[name]
|
||||
con = duckdb.connect(":memory:")
|
||||
assert write(con, "2026-Q2", rows) == len(rows)
|
||||
assert sorted(read(con), key=repr) == sorted(rows, key=repr)
|
||||
# writing a quarter again replaces it, other quarters are untouched
|
||||
assert write(con, "2026-Q2", rows[:1]) == 1
|
||||
assert read(con) == rows[:1]
|
||||
assert T.table_counts(con)[f"drugs.{name}"] == 1
|
||||
|
||||
|
||||
def test_write_empty_quarter_clears_it():
|
||||
con = duckdb.connect(":memory:")
|
||||
rows = ROUNDTRIP["asp_payment"][2]
|
||||
T.write_asp(con, "2026-Q2", rows)
|
||||
assert T.write_asp(con, "2026-Q2", []) == 0
|
||||
assert T.read_asp(con) == []
|
||||
|
||||
|
||||
def test_read_asp_filters():
|
||||
con = duckdb.connect(":memory:")
|
||||
a, b = ROUNDTRIP["asp_payment"][2]
|
||||
c = T.AspRow(
|
||||
"2026-Q1", "J9035", "II", "payment", "10 MG", 60.0, 20.0, "x", "", "f", "u", ""
|
||||
)
|
||||
T.write_asp(con, "2026-Q2", [a, b])
|
||||
T.write_asp(con, "2026-Q1", [c])
|
||||
assert [r.quarter for r in T.read_asp(con, hcpcs="J9035")] == ["2026-Q1", "2026-Q2"]
|
||||
assert T.read_asp(con, quarter="2026-Q1") == [c]
|
||||
assert T.read_asp(con, hcpcs="J7330", quarter="2026-Q2") == [b]
|
||||
assert T.read_asp(con, hcpcs="ZZZZZ") == []
|
||||
|
||||
Reference in New Issue
Block a user