diff --git a/data/cms/skin_substitute_hcpcs_reference.csv b/data/cms/skin_substitute_hcpcs_reference.csv
index 8735b9e..23cbc64 100644
--- a/data/cms/skin_substitute_hcpcs_reference.csv
+++ b/data/cms/skin_substitute_hcpcs_reference.csv
@@ -1,12 +1,12 @@
"hcpcs_code","short_descriptor","code_type","body_site","wound_size","effective_date","termination_date","status","notes"
-"15271","Skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Replaced CPT 15330"
-"15272","Skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Add-on to 15271"
-"15273","Skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15331"
-"15274","Skin substitute graft, trunk/arms/legs, >=100 sq cm; each addl 100 sq cm","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Add-on to 15273"
-"15275","Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; first 25 sq cm or less","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Replaced CPT 15335"
-"15276","Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Add-on to 15275"
-"15277","Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15336"
-"15278","Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Add-on to 15277"
+"15271","","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Replaced CPT 15330"
+"15272","","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Add-on to 15271"
+"15273","","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15331"
+"15274","","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Add-on to 15273"
+"15275","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Replaced CPT 15335"
+"15276","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Add-on to 15275"
+"15277","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15336"
+"15278","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Add-on to 15277"
"C5271","Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less","Application (OPPS Low-Cost)","Trunk/arms/legs","<100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026 per CY 2026 OPPS final rule"
"C5272","Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm","Application (OPPS Low-Cost)","Trunk/arms/legs","<100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026"
"C5273","Low cost skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm","Application (OPPS Low-Cost)","Trunk/arms/legs",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026"
@@ -15,15 +15,15 @@
"C5276","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026"
"C5277","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026"
"C5278","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026"
-"15330","Acellular dermal allograft, trunk/arms/legs; first 100 sq cm or less","Application (CPT - Historical)","Trunk/arms/legs","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15271/15272 in 2012"
-"15331","Acellular dermal allograft, trunk/arms/legs; each addl 100 sq cm","Application (CPT - Historical)","Trunk/arms/legs",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15273/15274 in 2012"
-"15335","Acellular dermal allograft, face/scalp/hands/feet; first 100 sq cm or less","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15275/15276 in 2012"
-"15336","Acellular dermal allograft, face/scalp/hands/feet; each addl 100 sq cm","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15277/15278 in 2012"
-"15340","Tissue cultured allogeneic skin substitute; first 25 sq cm or less","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 tissue-cultured allogeneic graft code"
-"15360","Tissue cultured autograft; first 25 sq cm or less","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 autologous cultured skin code"
-"15400","Xenograft; first 100 sq cm or less","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft application code"
-"15401","Xenograft; each addl 100 sq cm","Application (CPT - Historical)","",">100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft add-on code"
-"15430","Acellular xenograft implant; first 100 sq cm or less","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 acellular xenograft code"
+"15330","","Application (CPT - Historical)","Trunk/arms/legs","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15271/15272 in 2012"
+"15331","","Application (CPT - Historical)","Trunk/arms/legs",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15273/15274 in 2012"
+"15335","","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15275/15276 in 2012"
+"15336","","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15277/15278 in 2012"
+"15340","","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 tissue-cultured allogeneic graft code"
+"15360","","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 autologous cultured skin code"
+"15400","","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft application code"
+"15401","","Application (CPT - Historical)","",">100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft add-on code"
+"15430","","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 acellular xenograft code"
"G0440","Skin substitute, tissue cultured; first 25 sq cm (OPPS)","Application (OPPS - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Transitional OPPS code for pass-through products"
"C9363","Integra Meshed Bilayer Wound Matrix, per sq cm","Product (C-code Historical)","","","","","Deleted","Transitional pass-through C-code; replaced by Q-code"
"C9358","SurgiMend Collagen Matrix (fetal), per 0.5 sq cm","Product (C-code Historical)","","","","","Active","Bovine collagen - fetal origin"
diff --git a/dev/scripts/ingest_asp.py b/dev/scripts/ingest_asp.py
index 970ed32..aaba410 100644
--- a/dev/scripts/ingest_asp.py
+++ b/dev/scripts/ingest_asp.py
@@ -1,7 +1,8 @@
"""Ingest CMS Average Sales Price (ASP) Drug Pricing Files.
-Reads all quarterly ASP ZIP files from data/cms/asp/, normalises column
-names across 20+ years of CMS format drift, filters **strictly** to
+Reads all quarterly ASP ZIP files from data/cms/asp/ through the drug tier's
+parser (``drugs.asp.parse_quarter_zip``: column-name drift, section-508 CSV
+before its XLS twin, one file per quarter), filters **strictly** to
skin substitute HCPCS codes defined in data/cms/skin_substitute_hcpcs.csv,
and produces:
@@ -15,8 +16,8 @@ quarterly ZIP and the file within it that was parsed.
Source URLs are read from data/cms/asp_source_urls.csv (canonical URLs
provided from CMS.gov, one per quarter 2005-Q1 through 2026-Q2).
-Column mapping rationale
-------------------------
+Column mapping rationale (the map itself now lives in ``drugs.asp.COL_MAP``)
+---------------------------------------------------------------------------
CMS has changed column names across 20+ years of ASP files:
2005–2006: "HCPCS Code", "Short Description", "HCPCS Code Dosage",
@@ -53,23 +54,34 @@ Usage:
from __future__ import annotations
import csv
-import io
-import re
-import zipfile
from pathlib import Path
import pandas as pd
+from drugs.asp import newest_per_quarter, parse_quarter_zip, pick_pricing_file # noqa: F401
+
# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
ROOT = Path(__file__).resolve().parents[2]
ASP_DIR = ROOT / "data" / "cms" / "asp"
HCPCS_REF = ROOT / "data" / "cms" / "skin_substitute_hcpcs.csv"
+HCPCS_REFERENCE = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv"
URL_REF = ROOT / "data" / "cms" / "asp_source_urls.csv"
DUCKDB_PATH = ROOT / "data" / "aco.duckdb"
OUTPUT_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv"
+OUT_COLS = [
+ "quarter",
+ "hcpcs_code",
+ "short_description",
+ "asp_per_unit",
+ "payment_limit",
+ "dosage_or_unit",
+ "source_url",
+ "source_file",
+]
+
def _load_source_urls() -> dict[str, str]:
"""Load canonical source URLs: {quarter: url}."""
@@ -81,142 +93,74 @@ def _load_source_urls() -> dict[str, str]:
return urls
-# ---------------------------------------------------------------------------
-# Column normalisation
-# ---------------------------------------------------------------------------
-# Mapping from observed CMS column names (lowercased) to our standard names.
-# Each entry documents which CMS file years use that variant.
-COL_MAP = {
- # HCPCS code — consistent across all years
- "hcpcs code": "hcpcs_code",
- "hcpcs_code": "hcpcs_code",
- # Short description — consistent
- "short description": "short_description",
- "short_description": "short_description",
- # Dosage/unit — varies by year
- "hcpcs code dosage": "dosage_or_unit", # 2005-2006, 2020-2025
- "hcpcs_code_dosage": "dosage_or_unit",
- "dosage": "dosage_or_unit", # 2007-2019
- "dosage or unit": "dosage_or_unit", # 2026+
- # Payment limit — consistent (ASP + 6%)
- "payment limit": "payment_limit",
- "payment_limit": "payment_limit",
-}
-
-KEEP_COLS = ["hcpcs_code", "short_description", "dosage_or_unit", "payment_limit"]
+def require_rows(df: pd.DataFrame) -> None:
+ """Refuse to replace ``skin_subs.asp_quarterly`` with nothing: a parser
+ fault must not publish an empty table over the replica."""
+ if df.empty:
+ raise RuntimeError(
+ "no skin-substitute rows parsed; leaving skin_subs.asp_quarterly as it is"
+ )
-def normalise_columns(df: pd.DataFrame) -> pd.DataFrame:
- """Lower-case columns, apply COL_MAP, keep only KEEP_COLS."""
- df.columns = [str(c).strip().lower() for c in df.columns]
- df = df.rename(columns=COL_MAP)
- present = [c for c in KEEP_COLS if c in df.columns]
- return df[present]
-
-
-# ---------------------------------------------------------------------------
-# Readers
-# ---------------------------------------------------------------------------
-
-
-def _find_header_row(lines: list[str]) -> int:
- """Return the 0-based row index of the column-header row.
-
- The header row starts with 'HCPCS Code,' (possibly quoted).
- Must NOT match note lines that incidentally mention 'HCPCS code'.
- """
- for i, line in enumerate(lines):
- stripped = line.strip().lower()
- if stripped.startswith("hcpcs code,") or stripped.startswith('"hcpcs code",'):
- return i
- return -1
-
-
-def read_csv_from_zip(zf: zipfile.ZipFile, name: str) -> pd.DataFrame:
- """Read a section-508 CSV inside a ZIP, skipping preamble rows."""
- raw = zf.read(name).decode("latin-1")
- lines = raw.splitlines()
- hdr = _find_header_row(lines)
- if hdr < 0:
- return pd.DataFrame()
- buf = "\n".join(lines[hdr:])
- df = pd.read_csv(io.StringIO(buf), dtype=str, on_bad_lines="skip")
- return normalise_columns(df)
-
-
-def read_xls_from_zip(zf: zipfile.ZipFile, name: str) -> pd.DataFrame:
- """Read .xls inside a ZIP, skipping preamble rows."""
- import xlrd
-
- data = zf.read(name)
- wb = xlrd.open_workbook(file_contents=data)
- sh = wb.sheet_by_index(0)
- hdr_row = -1
- for r in range(min(30, sh.nrows)):
- first_cell = str(sh.cell_value(r, 0)).strip().lower()
- if first_cell == "hcpcs code":
- hdr_row = r
- break
- if hdr_row < 0:
- return pd.DataFrame()
- headers = [str(sh.cell_value(hdr_row, c)).strip() for c in range(sh.ncols)]
- rows = []
- for r in range(hdr_row + 1, sh.nrows):
- row = [str(sh.cell_value(r, c)).strip() for c in range(sh.ncols)]
- if any(row):
- rows.append(row)
- df = pd.DataFrame(rows, columns=headers)
- return normalise_columns(df)
-
-
-def pick_pricing_file(names: list[str]) -> str | None:
- """From a list of files in a ZIP, pick the ASP pricing file.
-
- Prefer CSV (section 508) over XLS. Skip NOC, crosswalk, NDC,
- and "not payable" files.
- """
- skip_patterns = ["noc", "crosswalk", "ndc", "not payable", "not_payable"]
- candidates: list[str] = []
- for n in names:
- low = n.lower()
- if any(s in low for s in skip_patterns):
- continue
- if low.endswith((".csv", ".xls", ".xlsx")):
- candidates.append(n)
-
- csvs = [c for c in candidates if c.lower().endswith(".csv")]
- if csvs:
- for c in csvs:
- low = c.lower()
- if "payment limit" in low or "pricing" in low:
- return c
- return csvs[0]
-
- xlss = [c for c in candidates if c.lower().endswith((".xls", ".xlsx"))]
- if xlss:
- for c in xlss:
- low = c.lower()
- if "payment limit" in low or "pricing" in low or "asp" in low:
- return c
- return xlss[0]
-
- return None
-
-
-# ---------------------------------------------------------------------------
-# Numeric cleaning
-# ---------------------------------------------------------------------------
-
-
-def clean_numeric(series: pd.Series) -> pd.Series:
- """Strip $ signs and convert to float, coercing errors to NaN."""
- return (
- series.astype(str)
- .str.replace("$", "", regex=False)
- .str.replace(",", "", regex=False)
- .str.strip()
- .pipe(pd.to_numeric, errors="coerce")
+def build_skin_df() -> pd.DataFrame:
+ """Every payment-limit row of a skin-substitute code (strict filter: only
+ codes in skin_substitute_hcpcs.csv), one file per quarter, with
+ ``asp_per_unit = payment_limit / 1.06`` (CMS payment limit = ASP + 6% for
+ drugs/biologicals; no longer meaningful for the flat 2026+ rate, kept for
+ consistency)."""
+ skin_codes = set(
+ pd.read_csv(HCPCS_REF, dtype=str)["hcpcs_code"].str.strip().str.upper()
)
+ print(f"Loaded {len(skin_codes)} skin-substitute HCPCS codes from reference")
+ source_urls = _load_source_urls()
+ print(f"Loaded {len(source_urls)} source URLs")
+ zips = newest_per_quarter(ASP_DIR.glob("asp_*.zip"))
+ print(f"Found {len(zips)} quarterly ZIP files\n")
+
+ records: list[dict[str, object]] = []
+ seen_q4: set[str] = set()
+ total = 0
+ for quarter, zpath in sorted(zips.items()):
+ try:
+ rows, _ = parse_quarter_zip(
+ zpath, quarter, source_url=source_urls.get(quarter, "")
+ )
+ except Exception as e: # a bad zip must not hide the other quarters
+ print(f" ERR {quarter}: {e}")
+ continue
+ payable = [r for r in rows if r.list_kind == "payment"]
+ if not payable:
+ print(f" SKIP {quarter}: no pricing rows found in ZIP")
+ continue
+ total += len(payable)
+ print(f" OK {quarter}: {len(payable):>5} rows from {payable[0].source_file}")
+ for r in payable:
+ if r.hcpcs in skin_codes:
+ records.append(
+ {
+ "quarter": quarter,
+ "hcpcs_code": r.hcpcs,
+ "short_description": r.short_desc,
+ "payment_limit": r.payment_limit,
+ "dosage_or_unit": r.dosage,
+ "source_url": r.source_url,
+ "source_file": r.source_file,
+ }
+ )
+ elif r.hcpcs.startswith("Q4") and len(r.hcpcs) in (4, 5):
+ seen_q4.add(r.hcpcs)
+ print(f"\nTotal rows across all quarters: {total}")
+ print(f"Skin substitute rows (strict filter): {len(records)}")
+ if seen_q4:
+ print(f" Excluded Q4 codes NOT in skin sub universe: {sorted(seen_q4)}")
+
+ skin_df = pd.DataFrame.from_records(records)
+ if skin_df.empty:
+ return pd.DataFrame(columns=OUT_COLS)
+ skin_df["payment_limit"] = pd.to_numeric(skin_df["payment_limit"], errors="coerce")
+ # ASP per unit = payment_limit / 1.06
+ skin_df["asp_per_unit"] = (skin_df["payment_limit"] / 1.06).round(3)
+ return skin_df[OUT_COLS].sort_values(["quarter", "hcpcs_code"])
# ---------------------------------------------------------------------------
@@ -225,106 +169,17 @@ def clean_numeric(series: pd.Series) -> pd.Series:
def main() -> None:
- # Load skin-substitute HCPCS codes (strict filter — no regex fallback)
- hcpcs_ref = pd.read_csv(HCPCS_REF, dtype=str)
- skin_codes = set(hcpcs_ref["hcpcs_code"].str.strip().str.upper())
- print(f"Loaded {len(skin_codes)} skin-substitute HCPCS codes from reference")
-
- # Load canonical source URLs
- source_urls = _load_source_urls()
- print(f"Loaded {len(source_urls)} source URLs")
-
- # Ingest all quarters
- zips = sorted(ASP_DIR.glob("asp_*.zip"))
- print(f"Found {len(zips)} quarterly ZIP files\n")
-
- frames: list[pd.DataFrame] = []
- for zpath in zips:
- m = re.match(r"asp_(\d{4}-Q\d)", zpath.name)
- if not m:
- print(f" SKIP {zpath.name} (cannot parse quarter)")
- continue
- quarter = m.group(1)
-
- try:
- with zipfile.ZipFile(zpath) as zf:
- target = pick_pricing_file(zf.namelist())
- if target is None:
- print(f" SKIP {quarter}: no pricing file found in ZIP")
- continue
-
- if target.lower().endswith(".csv"):
- df = read_csv_from_zip(zf, target)
- else:
- df = read_xls_from_zip(zf, target)
-
- if df.empty:
- print(f" SKIP {quarter}: empty after parsing {target}")
- continue
-
- df["quarter"] = quarter
- df["source_file"] = target
- df["source_url"] = source_urls.get(quarter, "")
- frames.append(df)
- print(f" OK {quarter}: {len(df):>5} rows from {target}")
- except Exception as e:
- print(f" ERR {quarter}: {e}")
-
- if not frames:
- raise RuntimeError("No data parsed from any file")
-
- all_df = pd.concat(frames, ignore_index=True)
- print(f"\nTotal rows across all quarters: {len(all_df)}")
-
- # Normalise HCPCS codes
- all_df["hcpcs_code"] = all_df["hcpcs_code"].astype(str).str.strip().str.upper()
-
- # Filter STRICTLY to skin substitute code universe
- # NO regex fallback — only codes in skin_substitute_hcpcs.csv
- skin_df = all_df[all_df["hcpcs_code"].isin(skin_codes)].copy()
- print(f"Skin substitute rows (strict filter): {len(skin_df)}")
-
- # Show what we excluded
- q4_but_not_skin = all_df[
- all_df["hcpcs_code"].str.match(r"^Q4\d{2,3}$")
- & ~all_df["hcpcs_code"].isin(skin_codes)
- ]["hcpcs_code"].unique()
- if len(q4_but_not_skin) > 0:
- print(
- f" Excluded Q4 codes NOT in skin sub universe: {sorted(q4_but_not_skin)}"
- )
-
- if not skin_df.empty:
- skin_df["payment_limit"] = clean_numeric(skin_df["payment_limit"])
- # ASP per unit = payment_limit / 1.06
- # (CMS payment limit = ASP + 6% for drugs/biologicals)
- skin_df["asp_per_unit"] = (skin_df["payment_limit"] / 1.06).round(3)
-
- # Reorder columns
- out_cols = [
- "quarter",
- "hcpcs_code",
- "short_description",
- "asp_per_unit",
- "payment_limit",
- "dosage_or_unit",
- "source_url",
- "source_file",
- ]
- for c in out_cols:
- if c not in skin_df.columns:
- skin_df[c] = None
- skin_df = skin_df[out_cols].sort_values(["quarter", "hcpcs_code"])
+ skin_df = build_skin_df()
+ require_rows(skin_df)
# Write CSV
skin_df.to_csv(OUTPUT_CSV, index=False)
print(f"\nWrote {len(skin_df)} rows to {OUTPUT_CSV}")
- if len(skin_df) > 0:
- quarters = sorted(skin_df["quarter"].unique())
- codes = sorted(skin_df["hcpcs_code"].unique())
- print(f" Quarters: {len(quarters)} ({quarters[0]} to {quarters[-1]})")
- print(f" Unique HCPCS codes: {len(codes)}")
+ quarters = sorted(skin_df["quarter"].unique())
+ codes = sorted(skin_df["hcpcs_code"].unique())
+ print(f" Quarters: {len(quarters)} ({quarters[0]} to {quarters[-1]})")
+ print(f" Unique HCPCS codes: {len(codes)}")
# Load into DuckDB
print(f"\nLoading into DuckDB at {DUCKDB_PATH} ...")
@@ -335,24 +190,26 @@ def main() -> None:
with duckdb_batch("aco") as con:
con.execute("CREATE SCHEMA IF NOT EXISTS skin_subs")
con.execute("DROP TABLE IF EXISTS skin_subs.asp_quarterly")
- if not skin_df.empty:
+ con.execute(
+ """
+ CREATE TABLE skin_subs.asp_quarterly AS
+ SELECT * FROM read_csv_auto(?, header=true)
+ """,
+ [str(OUTPUT_CSV)],
+ )
+ count = con.execute("SELECT count(*) FROM skin_subs.asp_quarterly").fetchone()[
+ 0
+ ]
+ print(f" Loaded {count} rows into skin_subs.asp_quarterly")
+ if HCPCS_REFERENCE.exists():
+ # the scrubbed reference (CPT descriptors blanked; codes and
+ # HCPCS Level II text kept)
+ con.execute("DROP TABLE IF EXISTS skin_subs.hcpcs_reference")
con.execute(
- """
- CREATE TABLE skin_subs.asp_quarterly AS
- SELECT * FROM read_csv_auto(?, header=true)
- """,
- [str(OUTPUT_CSV)],
+ "CREATE TABLE skin_subs.hcpcs_reference AS "
+ "SELECT * FROM read_csv_auto(?, header=true)",
+ [str(HCPCS_REFERENCE)],
)
- count = con.execute(
- "SELECT count(*) FROM skin_subs.asp_quarterly"
- ).fetchone()[0]
- print(f" Loaded {count} rows into skin_subs.asp_quarterly")
- else:
- print(" WARNING: No skin substitute data found in ASP files")
- print(
- " This may be expected — skin subs may not appear in main ASP pricing files"
- )
- print(" They may be in separate NOC or tissue coding files")
# Refresh the notebook read replica (#510).
print(f"replica → {publish_replica('aco')}")
diff --git a/src/cli/drugs.py b/src/cli/drugs.py
index f0bd902..2c5b086 100644
--- a/src/cli/drugs.py
+++ b/src/cli/drugs.py
@@ -2,12 +2,18 @@
from __future__ import annotations
+import re
+import zipfile
from typing import Any
import typer
+from drugs import asp as A
+from drugs import http as dh
+from drugs import tables as T
from drugs.tables import table_counts
-from market.tables import read_gaps, write_gap
+from market.http import Refused
+from market.tables import clear_gap, read_gaps, write_gap
from pfs.codetables import is_missing_table_error
app = typer.Typer(no_args_is_help=True)
@@ -89,3 +95,162 @@ def status() -> None:
typer.echo("no gaps recorded")
for g in gaps:
typer.echo(f"gap: {g.source}: {g.note} ({g.recorded_on})")
+
+
+# ── asp ──────────────────────────────────────────────────────────────
+ASP_GAP = "cms-asp:asp"
+
+
+def _cite(store: Any, url: str, title: str, table: str) -> str:
+ if not url:
+ return ""
+ return dh.cite_file(store, url=url, title=title, provider="cms-asp", table=table)
+
+
+def _quarters_option(values: list[str] | None) -> list[str]:
+ for v in values or []:
+ if not re.fullmatch(r"[0-9]{4}-Q[1-4]", v):
+ raise typer.BadParameter(f"{v!r} is not a quarter like 2026-Q4")
+ return list(values or [])
+
+
+@app.command()
+def asp(
+ fetch: bool = typer.Option(False, "--fetch", help="Hoard the page and its zips."),
+ refresh: bool = typer.Option(False, "--refresh", help="Re-download every zip."),
+ quarter: list[str] = typer.Option(None, "--quarter", help="Only these quarters."),
+ dry_run: bool = typer.Option(
+ False, "--dry-run", help="List the links; change nothing."
+ ),
+ write: bool = typer.Option(
+ False, "--write", help="Write drugs.asp_payment and drugs.asp_noc."
+ ),
+) -> None:
+ """Quarterly Part B payment limits for every code (payable, not-payable
+ and NOC rows) from the hoarded CMS ASP files."""
+ sel = set(_quarters_option(quarter))
+ write = write and not dry_run
+ cl = None
+ if fetch:
+ cl = dh.client("cms")
+ try:
+ A.fetch_page(cl)
+ except Refused as exc:
+ _gap(ASP_GAP, f"{A.ASP_PAGE} refused: {exc}", write=write)
+ return
+ page = A.hoarded_page()
+ links = [
+ lk
+ for lk in (A.parse_asp_page(page.read_text(errors="replace")) if page else [])
+ if lk.kind in ("pricing", "noc", "crosswalk") and (not sel or lk.quarter in sel)
+ ]
+ steps = A.plan_fetch(links, refresh=refresh)
+ if dry_run:
+ if not links:
+ typer.echo("no page hoarded; run with --fetch")
+ for st in steps:
+ typer.echo(f"{st.link.kind} {st.link.quarter} {st.action} {st.link.url}")
+ return
+ refusal = ""
+ if fetch:
+ for st in steps:
+ if st.action == "have":
+ continue
+ try:
+ A.apply_plan(cl, st)
+ except Refused as exc:
+ refusal = f"refused: {exc}"
+ break
+ typer.echo(f"{st.action} {st.link.kind} {st.link.quarter}")
+
+ def pick(kind: str, prefix: str) -> dict[str, Any]:
+ got = A.newest_per_quarter(A.hoard_dir(kind).glob(f"{prefix}_*.zip"))
+ return {q: p for q, p in got.items() if not sel or q in sel}
+
+ pricing, nocs, xwalks = (
+ pick("pricing", "asp"),
+ pick("noc", "noc"),
+ pick("crosswalk", "xwalk"),
+ )
+ url_of = {(lk.quarter, lk.kind): lk.url for lk in links}
+ canon = A.canonical_urls()
+ store = _store() if write else None
+
+ def source(q: str, kind: str, title: str, table: str) -> tuple[str, str]:
+ url = url_of.get((q, kind)) or (canon.get(q, "") if kind == "pricing" else "")
+ return url, (_cite(store, url, title, table) if write else "")
+
+ payments: dict[str, list[T.AspRow]] = {}
+ noc_rows: dict[str, list[T.NocRow]] = {}
+ files: dict[tuple[str, str], Any] = {}
+ unreadable: list[str] = []
+ for q, path in sorted(pricing.items()):
+ url, key = source(
+ q, "pricing", f"CMS ASP pricing file {q}", "drugs.asp_payment"
+ )
+ try:
+ rows, bundled = A.parse_quarter_zip(path, q, source_url=url, item_key=key)
+ except zipfile.BadZipFile:
+ unreadable.append(f"{q} pricing unreadable")
+ continue
+ payments[q] = rows
+ files[(q, "drugs.asp_payment")] = path
+ if bundled and q not in nocs:
+ noc_rows[q] = bundled
+ files[(q, "drugs.asp_noc")] = path
+ for q, path in sorted(nocs.items()):
+ url, key = source(q, "noc", f"CMS ASP NOC pricing file {q}", "drugs.asp_noc")
+ try:
+ _, rows = A.parse_quarter_zip(path, q, source_url=url, item_key=key)
+ except zipfile.BadZipFile:
+ unreadable.append(f"{q} noc unreadable")
+ continue
+ noc_rows[q] = rows
+ files[(q, "drugs.asp_noc")] = path
+
+ for q in sorted(payments):
+ rows = payments[q]
+ kinds = [r.list_kind for r in rows]
+ typer.echo(
+ f"{q}: {kinds.count('payment')} payment, {kinds.count('not-payable')} not-payable,"
+ f" {len(noc_rows.get(q, []))} noc"
+ )
+ for q in sorted(set(noc_rows) - set(payments)):
+ typer.echo(f"{q}: {len(noc_rows[q])} noc")
+ expected = sorted(sel) if sel else A.expected_quarters(A.today())
+ typer.echo(
+ f"pricing {len(payments)}/{len(expected)} quarters,"
+ f" {sum(len(r) for r in payments.values())} rows,"
+ f" {sum(r.list_kind == 'not-payable' for v in payments.values() for r in v)} not-payable;"
+ f" noc {len(noc_rows)} quarters, {sum(len(r) for r in noc_rows.values())} rows"
+ )
+ have = {
+ "pricing": set(pricing),
+ "noc": set(nocs) | set(noc_rows),
+ "crosswalk": set(xwalks),
+ }
+ listed = {(lk.quarter, lk.kind) for lk in links}
+ pairs = unreadable + A.gap_pairs(expected, have, listed)
+ note = "; ".join(x for x in (refusal, A.gap_note(pairs)) if x)
+ if note:
+ typer.echo(f"gap: {ASP_GAP}: {note}")
+ if not write:
+ return
+ from cms.ingest_log import log_ingest
+
+ with _batch() as con:
+ T.ensure_tables(con)
+ for q, rows in payments.items():
+ T.write_asp(con, q, rows)
+ for q, nrows in noc_rows.items():
+ T.write_noc(con, q, nrows)
+ for (q, table), path in sorted(files.items()):
+ n = len(payments[q]) if table == "drugs.asp_payment" else len(noc_rows[q])
+ log_ingest(
+ con, module="drugs.asp", table_name=table, rows=n, source_file=str(path)
+ )
+ if note:
+ write_gap(con, ASP_GAP, note)
+ else:
+ clear_gap(con, ASP_GAP)
+ _publish()
diff --git a/src/drugs/asp.py b/src/drugs/asp.py
new file mode 100644
index 0000000..24fd6ae
--- /dev/null
+++ b/src/drugs/asp.py
@@ -0,0 +1,594 @@
+"""CMS Medicare Part B drug payment-limit ("ASP") files, every quarter.
+
+One parser serves two readers: ``stack drugs asp`` (all codes, plus the
+"drugs not payable" lists and the not-otherwise-classified drugs) and the
+skin-substitute ingest (``dev/scripts/ingest_asp.py``).
+
+Rules held here:
+
+* CMS's short descriptions of CPT (Level I) codes are AMA text and are never
+ kept: a licensed code's description is ``""``. HCPCS Level II text is CMS's
+ own and is kept.
+* A zip holds one member per kind (pricing, not-payable, NOC); when CMS ships
+ both a section-508 ``.csv`` and its ``.xls`` twin, the ``.csv`` is read and
+ the twin is not, so no row is counted twice.
+* Files are hoarded under ``data/cms/asp`` and never overwritten; an updated
+ posting lands beside the old file and is the one read.
+* A quarter with no file is a gap, never a zero.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import re
+import zipfile
+from collections.abc import Callable, Iterable, Sequence
+from dataclasses import dataclass
+from datetime import date
+from html.parser import HTMLParser
+from pathlib import Path
+from typing import Any
+from urllib.parse import parse_qs, urljoin, urlparse
+
+from drugs import http as dh
+from drugs.codes import level, licensed
+from drugs.tables import ( # noqa: F401 (re-exported)
+ AspRow,
+ NocRow,
+ read_asp,
+ read_noc,
+ write_asp,
+ write_noc,
+)
+
+ASP_PAGE = "https://www.cms.gov/medicare/payment/part-b-drugs/asp-pricing-files"
+CMS = "https://www.cms.gov"
+FIRST_QUARTER = "2005-Q1"
+KINDS = ("pricing", "noc", "crosswalk", "vaccine", "other")
+
+
+def today() -> date:
+ return date.today()
+
+
+# ── the page ─────────────────────────────────────────────────────────
+@dataclass(frozen=True)
+class AspLink:
+ quarter: str
+ kind: str
+ url: str
+ text: str
+
+
+class _Anchors(HTMLParser):
+ def __init__(self) -> None:
+ super().__init__(convert_charrefs=True)
+ self.found: list[tuple[str, str]] = []
+ self._href: str | None = None
+ self._text: list[str] = []
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ if tag == "a":
+ self._href = dict(attrs).get("href")
+ self._text = []
+
+ def handle_data(self, data: str) -> None:
+ if self._href is not None:
+ self._text.append(data)
+
+ def handle_endtag(self, tag: str) -> None:
+ if tag == "a" and self._href is not None:
+ text = re.sub(r"\s+", " ", "".join(self._text).replace("\xa0", " "))
+ self.found.append((self._href, text.strip()))
+ self._href = None
+
+
+_MONTH_Q = {"jan": 1, "apr": 2, "jul": 3, "oct": 4}
+_TEXT_Q = re.compile(r"\b(jan|apr|jul|oct)[a-z]*\.?\s+(\d{4})\b")
+_HREF_Q = re.compile(r"(january|april|july|october)[-_ ]?(\d{4})")
+_HREF_YQ = re.compile(r"(\d{4})[-_ ](january|april|july|october)")
+
+
+def _quarter_of(text: str, href: str) -> str:
+ m = _TEXT_Q.search(text.lower())
+ if m:
+ return f"{m.group(2)}-Q{_MONTH_Q[m.group(1)]}"
+ low = href.lower()
+ m = _HREF_Q.search(low)
+ if m:
+ return f"{m.group(2)}-Q{_MONTH_Q[m.group(1)[:3]]}"
+ m = _HREF_YQ.search(low)
+ if m:
+ return f"{m.group(1)}-Q{_MONTH_Q[m.group(2)[:3]]}"
+ return ""
+
+
+_KIND_MARKS = (
+ ("crosswalk", ("crosswalk", "ndc")),
+ ("noc", ("noc",)),
+ ("pricing", ("asp pricing", "payment limit")),
+)
+
+
+def _kind_of(text: str) -> str:
+ """The first kind the link text names: a bundle such as "ASP Pricing
+ File/ASP NOC Pricing File/ASP NDC-HCPCS Crosswalk" is its pricing zip."""
+ low = text.lower()
+ if "vaccine" in low:
+ return "vaccine"
+ best, where = "other", len(low) + 1
+ for kind, marks in _KIND_MARKS:
+ for mark in marks:
+ i = low.find(mark)
+ if 0 <= i < where:
+ best, where = kind, i
+ return best
+
+
+def parse_asp_page(html: str) -> list[AspLink]:
+ """Every zip link of the ASP pricing-files page, with its quarter and
+ kind. Vaccine and unrecognised links have quarter ``""``."""
+ p = _Anchors()
+ p.feed(html)
+ out: list[AspLink] = []
+ for href, text in p.found:
+ url = direct_url(href)
+ if not urlparse(url).path.lower().endswith(".zip"):
+ continue
+ kind = _kind_of(text)
+ quarter = _quarter_of(text, url)
+ if kind in ("vaccine", "other") or not quarter:
+ kind, quarter = ("vaccine" if kind == "vaccine" else "other"), ""
+ out.append(AspLink(quarter, kind, url, text))
+ return out
+
+
+def direct_url(href: str) -> str:
+ """The file URL behind a CMS click-through link
+ (``.../license.asp?file=
`` or ``.../license/ama?file=
`` ->
+ ``https://www.cms.gov
``); a relative href is made absolute."""
+ parts = urlparse(href)
+ if parts.path.lower().endswith(("/license.asp", "/license/ama")):
+ target = parse_qs(parts.query).get("file", [""])[0]
+ if target:
+ return urljoin(CMS, target)
+ return urljoin(CMS, href)
+
+
+def page_path() -> Path:
+ return dh.path("drugs", "cms", "asp-pricing-files.html")
+
+
+def fetch_page(cl: Any) -> Path:
+ """Hoard the page (a changed page lands beside the old one)."""
+ return dh.fetch(
+ cl, ASP_PAGE, page_path(), provider="cms", source="cms-asp", refresh=True
+ )
+
+
+def hoarded_page() -> Path | None:
+ p = dh.newest(page_path())
+ return p if p.is_file() else None
+
+
+# ── hoard layout ─────────────────────────────────────────────────────
+_DIRS = {
+ "pricing": ("asp", "asp"),
+ "noc": ("asp/noc", "noc"),
+ "crosswalk": ("asp/xwalk", "xwalk"),
+}
+
+
+def hoard_dir(kind: str) -> Path:
+ if kind not in _DIRS:
+ raise ValueError(f"no hoard for kind {kind!r}")
+ return dh.path("cms", *_DIRS[kind][0].split("/"))
+
+
+def hoard_path(link: AspLink) -> Path:
+ return hoard_dir(link.kind) / f"{_DIRS[link.kind][1]}_{link.quarter}.zip"
+
+
+_ZIP_Q = re.compile(r"^(?:asp|noc|xwalk)_(\d{4}-Q[1-4])(?:\.[0-9a-f]{8})?\.zip$")
+
+
+def newest_per_quarter(paths: Iterable[Path]) -> dict[str, Path]:
+ """One path per quarter: the newest copy (``market.http.newest``'s order:
+ write time, then name), so an updated posting beside the original wins and
+ a quarter is never read twice."""
+ best: dict[str, Path] = {}
+ for p in paths:
+ m = _ZIP_Q.match(p.name)
+ if not m:
+ continue
+ q = m.group(1)
+ key = (p.stat().st_mtime_ns, p.name)
+ if q not in best or key > (best[q].stat().st_mtime_ns, best[q].name):
+ best[q] = p
+ return best
+
+
+def expected_quarters(today: date) -> list[str]:
+ """Every quarter from 2005-Q1 to the one containing *today*."""
+ out: list[str] = []
+ y, q = 2005, 1
+ while (y, q) <= (today.year, (today.month - 1) // 3 + 1):
+ out.append(f"{y}-Q{q}")
+ y, q = (y + 1, 1) if q == 4 else (y, q + 1)
+ return out
+
+
+# ── the manifest and what to fetch ──────────────────────────────────
+def manifest_urls() -> set[str]:
+ mf = dh.data_root() / "drugs" / dh.MANIFEST
+ if not mf.exists():
+ return set()
+ return {json.loads(x)["url"] for x in mf.read_text().splitlines() if x.strip()}
+
+
+def canonical_urls() -> dict[str, str]:
+ """``data/cms/asp_source_urls.csv``: the URL each of the first 86 pricing
+ zips was downloaded from."""
+ p = dh.path("cms", "asp_source_urls.csv")
+ if not p.exists():
+ return {}
+ with p.open(newline="") as fh:
+ return {r["quarter"]: r["url"] for r in csv.DictReader(fh)}
+
+
+@dataclass(frozen=True)
+class Step:
+ link: AspLink
+ dest: Path
+ action: str # have | adopt | fetch | fetch-beside
+
+
+def plan_fetch(links: Sequence[AspLink], *, refresh: bool) -> list[Step]:
+ """What to do with each pricing, NOC and crosswalk link: ``have`` (its
+ URL is in the manifest), ``adopt`` (the file on disk was downloaded from
+ this URL, per ``asp_source_urls.csv``; record it), ``fetch`` (nothing on
+ disk) or ``fetch-beside`` (a different posting of a hoarded quarter, or
+ *refresh*; it lands beside the old file)."""
+ known = manifest_urls()
+ canon = canonical_urls()
+ steps: list[Step] = []
+ for link in links:
+ if link.kind not in _DIRS or not link.quarter:
+ continue
+ dest = hoard_path(link)
+ if not dest.exists():
+ action = "fetch"
+ elif refresh:
+ action = "fetch-beside"
+ elif link.url in known:
+ action = "have"
+ elif link.kind == "pricing" and canon.get(link.quarter) == link.url:
+ action = "adopt"
+ else:
+ action = "fetch-beside"
+ steps.append(Step(link, dest, action))
+ return steps
+
+
+def apply_plan(cl: Any, step: Step) -> Path:
+ """Carry out one step; returns the hoarded path (the newest copy)."""
+ if step.action == "have":
+ return dh.newest(step.dest)
+ if step.action == "adopt":
+ dh.manifest(
+ step.dest, step.link.url, "adopted: downloaded from asp_source_urls.csv"
+ )
+ return step.dest
+ return dh.fetch(
+ cl,
+ step.link.url,
+ step.dest,
+ provider="cms",
+ source="cms-asp",
+ refresh=True,
+ note=f"{step.link.kind} {step.link.quarter}",
+ )
+
+
+def gap_pairs(
+ expected: Sequence[str],
+ have: dict[str, set[str]],
+ listed: set[tuple[str, str]],
+) -> list[str]:
+ """``" "`` for every expected quarter with no pricing file
+ and every NOC or crosswalk file the page lists but the hoard lacks."""
+ pairs = [f"{q} pricing" for q in expected if q not in have["pricing"]]
+ for q, kind in sorted(listed):
+ if kind != "pricing" and q not in have.get(kind, set()):
+ pairs.append(f"{q} {kind}")
+ return pairs
+
+
+def gap_note(pairs: Sequence[str], show: int = 12) -> str:
+ if not pairs:
+ return ""
+ note = "no hoarded file for: " + ", ".join(pairs[:show])
+ if len(pairs) > show:
+ note += f" (+{len(pairs) - show} more)"
+ return note
+
+
+# ── reading a file ───────────────────────────────────────────────────
+#: observed CMS column names (lower-cased) -> ours
+COL_MAP = {
+ "hcpcs code": "hcpcs_code",
+ "hcpcs_code": "hcpcs_code",
+ "short description": "short_description",
+ "short_description": "short_description",
+ "long description": "short_description", # the not-payable list
+ "hcpcs code dosage": "dosage_or_unit",
+ "hcpcs_code_dosage": "dosage_or_unit",
+ "dosage": "dosage_or_unit",
+ "dosage or unit": "dosage_or_unit",
+ "payment limit": "payment_limit",
+ "payment_limit": "payment_limit",
+ "co-insurance percentage": "coinsurance_pct",
+ "notes": "notes",
+ "manufacturer": "manufacturer", # the not-payable list
+ "drug generic name (trade name)": "drug_name", # NOC files
+ "drug generic name": "drug_name",
+ "drug name (generic)": "drug_name", # the 2005 NOC sheet
+}
+
+_HEADER_STARTS = (
+ "hcpcs code,",
+ '"hcpcs code",',
+ "drug generic name",
+ '"drug generic name',
+ "drug name (generic)",
+)
+
+
+def find_header_row(lines: Sequence[str]) -> int:
+ """0-based index of the column-header row, or -1. The header starts with
+ ``HCPCS Code,`` (or ``Drug Generic Name`` in NOC files); a note line that
+ merely mentions HCPCS codes does not match."""
+ for i, line in enumerate(lines):
+ if line.strip().lower().startswith(_HEADER_STARTS):
+ return i
+ return -1
+
+
+def _skip_member(low: str) -> bool:
+ return any(
+ s in low for s in ("noc", "crosswalk", "ndc", "not payable", "not_payable")
+ )
+
+
+def pick_pricing_file(names: Sequence[str]) -> str | None:
+ """The ASP pricing member of a zip: ``.csv`` before ``.xls``; NOC,
+ crosswalk, NDC and "not payable" members are skipped."""
+ candidates = [
+ n
+ for n in names
+ if not _skip_member(n.lower()) and n.lower().endswith((".csv", ".xls", ".xlsx"))
+ ]
+ csvs = [c for c in candidates if c.lower().endswith(".csv")]
+ if csvs:
+ for c in csvs:
+ low = c.lower()
+ if "payment limit" in low or "pricing" in low:
+ return c
+ return csvs[0]
+ xlss = [c for c in candidates if c.lower().endswith((".xls", ".xlsx"))]
+ if xlss:
+ for c in xlss:
+ low = c.lower()
+ if "payment limit" in low or "pricing" in low or "asp" in low:
+ return c
+ return xlss[0]
+ return None
+
+
+_MEMBER_MARKS = {
+ "not-payable": ("not payable", "not_payable", "notpayable"),
+ "noc": ("noc",),
+}
+
+
+def pick_member(names: Sequence[str], kind: str) -> str | None:
+ """The one member of kind ``"not-payable"`` or ``"noc"``: its ``.csv``
+ when CMS ships a twin, else its ``.xls``."""
+ if kind not in _MEMBER_MARKS:
+ raise ValueError(f"no member kind {kind!r}")
+ marks = _MEMBER_MARKS[kind]
+ found = [
+ n
+ for n in names
+ if n.lower().endswith((".csv", ".xls", ".xlsx"))
+ and any(m in n.lower() for m in marks)
+ and not ("crosswalk" in n.lower() or "ndc" in n.lower())
+ ]
+ csvs = [n for n in found if n.lower().endswith(".csv")]
+ return (csvs or found or [None])[0]
+
+
+def _cell(v: Any) -> str:
+ if isinstance(v, float) and v == int(v):
+ return str(int(v))
+ return str(v).strip()
+
+
+def read_table(
+ zf: zipfile.ZipFile,
+ name: str,
+ *,
+ open_workbook: Callable[..., Any] | None = None,
+ sheet: str = "",
+) -> list[dict[str, str]]:
+ """The rows of a zip member as ``{our column: text}`` (only the columns of
+ :data:`COL_MAP`); preamble rows are skipped and blank rows dropped. A CSV
+ is latin-1 with its header row found; an ``.xls`` goes through
+ ``xlrd.open_workbook`` (injectable for tests): its first sheet, or with
+ *sheet* the first sheet whose name contains that text (the 2005 files keep
+ the NOC table on a second sheet)."""
+ if name.lower().endswith((".xls", ".xlsx")):
+ return _read_xls(zf, name, open_workbook, sheet)
+ lines = zf.read(name).decode("latin-1").splitlines()
+ hdr = find_header_row(lines)
+ if hdr < 0:
+ return []
+ reader = csv.reader(io.StringIO("\n".join(lines[hdr:]), newline=""))
+ header = [COL_MAP.get(c.strip().lower()) for c in next(reader)]
+ rows: list[dict[str, str]] = []
+ for raw in reader:
+ if len(raw) > len(header):
+ if any(c.strip() for c in raw[len(header) :]):
+ continue # more fields than columns: a malformed line
+ raw = raw[: len(header)]
+ row: dict[str, str] = {}
+ for col, val in zip(header, raw, strict=False):
+ if col and col not in row:
+ row[col] = val
+ if any(v.strip() for v in row.values()):
+ rows.append(row)
+ return rows
+
+
+def _read_xls(
+ zf: zipfile.ZipFile,
+ name: str,
+ open_workbook: Callable[..., Any] | None,
+ sheet: str,
+) -> list[dict[str, str]]:
+ if open_workbook is None:
+ import xlrd
+
+ open_workbook = xlrd.open_workbook
+ wb = open_workbook(file_contents=zf.read(name))
+ index = 0
+ if sheet:
+ found = [i for i, n in enumerate(wb.sheet_names()) if sheet in n.lower()]
+ if not found:
+ return []
+ index = found[0]
+ sh = wb.sheet_by_index(index)
+ hdr = -1
+ for r in range(min(30, sh.nrows)):
+ first = str(sh.cell_value(r, 0)).strip().lower()
+ if first == "hcpcs code" or first.startswith(
+ ("drug generic name", "drug name (generic)")
+ ):
+ hdr = r
+ break
+ if hdr < 0:
+ return []
+ header = [
+ COL_MAP.get(_cell(sh.cell_value(hdr, c)).lower()) for c in range(sh.ncols)
+ ]
+ rows: list[dict[str, str]] = []
+ for r in range(hdr + 1, sh.nrows):
+ vals = [_cell(sh.cell_value(r, c)) for c in range(sh.ncols)]
+ if not any(vals):
+ continue
+ row: dict[str, str] = {}
+ for col, val in zip(header, vals, strict=True):
+ if col and col not in row:
+ row[col] = val
+ rows.append(row)
+ return rows
+
+
+def _number(raw: str | None) -> float | None:
+ s = (raw or "").replace("$", "").replace(",", "").strip()
+ if not s:
+ return None
+ try:
+ return float(s)
+ except ValueError:
+ return None
+
+
+def _asp_rows(
+ rows: Sequence[dict[str, str]],
+ *,
+ quarter: str,
+ list_kind: str,
+ member: str,
+ source_url: str,
+ item_key: str,
+) -> list[AspRow]:
+ out: list[AspRow] = []
+ for r in rows:
+ code = r.get("hcpcs_code", "").strip().upper()
+ if not code:
+ continue
+ payable = list_kind == "payment"
+ out.append(
+ AspRow(
+ quarter=quarter,
+ hcpcs=code,
+ level=level(code),
+ list_kind=list_kind,
+ dosage=r.get("dosage_or_unit", ""),
+ payment_limit=_number(r.get("payment_limit")) if payable else None,
+ coinsurance_pct=_number(r.get("coinsurance_pct")) if payable else None,
+ short_desc="" if licensed(code) else r.get("short_description", ""),
+ notes=r.get("notes", ""),
+ source_file=member,
+ source_url=source_url,
+ item_key=item_key,
+ manufacturer=r.get("manufacturer", "").strip() if not payable else "",
+ )
+ )
+ return out
+
+
+def parse_quarter_zip(
+ path: Path,
+ quarter: str,
+ *,
+ source_url: str,
+ item_key: str = "",
+ open_workbook: Callable[..., Any] | None = None,
+) -> tuple[list[AspRow], list[NocRow]]:
+ """The payment, not-payable and NOC rows of one quarter's zip: one member
+ per kind. A licensed (CPT/CDT) code gets ``short_desc=""``."""
+ asp: list[AspRow] = []
+ noc: list[NocRow] = []
+ with zipfile.ZipFile(path) as zf:
+ names = zf.namelist()
+ pricing = pick_pricing_file(names)
+ for kind, member in (
+ ("payment", pricing),
+ ("not-payable", pick_member(names, "not-payable")),
+ ):
+ if member:
+ asp += _asp_rows(
+ read_table(zf, member, open_workbook=open_workbook),
+ quarter=quarter,
+ list_kind=kind,
+ member=member,
+ source_url=source_url,
+ item_key=item_key,
+ )
+ member = pick_member(names, "noc")
+ table: list[dict[str, str]] = []
+ if member:
+ table = read_table(zf, member, open_workbook=open_workbook)
+ elif pricing and pricing.lower().endswith((".xls", ".xlsx")):
+ table = read_table(zf, pricing, open_workbook=open_workbook, sheet="noc")
+ member = f"{pricing}#NOC"
+ if member:
+ for r in table:
+ name = r.get("drug_name", "").strip()
+ if name:
+ noc.append(
+ NocRow(
+ quarter=quarter,
+ drug_name=name,
+ dosage=r.get("dosage_or_unit", ""),
+ payment_limit=_number(r.get("payment_limit")),
+ source_file=member,
+ source_url=source_url,
+ item_key=item_key,
+ )
+ )
+ return asp, noc
diff --git a/src/drugs/codes.py b/src/drugs/codes.py
index 042f028..1aaa0b9 100644
--- a/src/drugs/codes.py
+++ b/src/drugs/codes.py
@@ -5,18 +5,18 @@ from __future__ import annotations
import re
from datetime import date, timedelta
-_L1 = re.compile(r"^\d{4}[0-9A-Z]$")
-_L2 = re.compile(r"^[ABCEGHJKLMPQRSTUV]\d{4}$")
-_CDT = re.compile(r"^D\d{4}$")
+_L1 = re.compile(r"[0-9]{4}[0-9A-Z]")
+_L2 = re.compile(r"[ABCEGHJKLMPQRSTUV][0-9]{4}")
+_CDT = re.compile(r"D[0-9]{4}")
def level(code: str) -> str:
"""``"I"`` (CPT), ``"II"`` (HCPCS Level II), ``"CDT"`` or ``""``."""
- if _L1.match(code):
+ if _L1.fullmatch(code):
return "I"
- if _L2.match(code):
+ if _L2.fullmatch(code):
return "II"
- if _CDT.match(code):
+ if _CDT.fullmatch(code):
return "CDT"
return ""
@@ -30,9 +30,9 @@ def ndc11(raw: str) -> str | None:
"""The 11-digit NDC of a 4-4-2, 5-3-2, 5-4-1 or 5-4-2 hyphenated code, or
of 11 bare digits; anything else is ``None``."""
s = raw.strip()
- if re.fullmatch(r"\d{11}", s):
+ if re.fullmatch(r"[0-9]{11}", s):
return s
- m = re.fullmatch(r"(\d+)-(\d+)-(\d+)", s)
+ m = re.fullmatch(r"([0-9]+)-([0-9]+)-([0-9]+)", s)
if not m:
return None
a, b, c = m.groups()
@@ -48,27 +48,36 @@ def ndc11(raw: str) -> str | None:
return None
-_APPL = {"N": "NDA", "A": "ANDA", "BLA": "BLA"}
+_APPL = {"N": "NDA", "NDA": "NDA", "A": "ANDA", "ANDA": "ANDA", "BLA": "BLA"}
def appl_id(kind: str, number: str) -> str:
"""``("N","21436")`` -> ``NDA021436``; an id already spelled out passes."""
- if re.fullmatch(r"[A-Z]+\d{6}", kind):
+ if re.fullmatch(r"[A-Z]+[0-9]{6}", kind):
return kind
- return f"{_APPL.get(kind, kind)}{int(number):06d}"
+ if kind not in _APPL:
+ raise ValueError(f"unknown application kind {kind!r}")
+ return f"{_APPL[kind]}{int(number):06d}"
+
+
+#: the longest range :func:`expand` will list; a wider one is a data error
+MAX_RANGE = 500
def expand(lo: str, hi: str) -> list[str]:
"""Every code from *lo* to *hi* when they share a letter prefix and a
- numeric tail; otherwise just *lo*."""
- a = re.fullmatch(r"([A-Z]+)(\d+)", lo)
- b = re.fullmatch(r"([A-Z]+)(\d+)", hi)
+ numeric tail; otherwise just *lo*. A range wider than :data:`MAX_RANGE`
+ codes raises ``ValueError``."""
+ a = re.fullmatch(r"([A-Z]+)([0-9]+)", lo)
+ b = re.fullmatch(r"([A-Z]+)([0-9]+)", hi)
if not a or not b or a.group(1) != b.group(1):
return [lo]
width = len(a.group(2))
start, end = int(a.group(2)), int(b.group(2))
if end < start or len(b.group(2)) != width:
return [lo]
+ if end - start + 1 > MAX_RANGE:
+ raise ValueError(f"range too large: {lo}-{hi} is {end - start + 1} codes")
return [f"{a.group(1)}{n:0{width}d}" for n in range(start, end + 1)]
diff --git a/src/drugs/tables.py b/src/drugs/tables.py
index 07fa6b6..051c978 100644
--- a/src/drugs/tables.py
+++ b/src/drugs/tables.py
@@ -7,11 +7,35 @@ tables. Nothing opens a database at import.
from __future__ import annotations
from collections.abc import Iterable, Sequence
-from dataclasses import astuple, fields
+from dataclasses import astuple, dataclass, fields
from typing import Any
_DDL = """
-CREATE SCHEMA IF NOT EXISTS drugs
+CREATE SCHEMA IF NOT EXISTS drugs;
+CREATE TABLE IF NOT EXISTS drugs.asp_payment (
+ quarter VARCHAR,
+ hcpcs VARCHAR,
+ level VARCHAR,
+ list_kind VARCHAR,
+ dosage VARCHAR,
+ payment_limit DOUBLE,
+ coinsurance_pct DOUBLE,
+ short_desc VARCHAR,
+ notes VARCHAR,
+ source_file VARCHAR,
+ source_url VARCHAR,
+ item_key VARCHAR,
+ manufacturer VARCHAR
+);
+CREATE TABLE IF NOT EXISTS drugs.asp_noc (
+ quarter VARCHAR,
+ drug_name VARCHAR,
+ dosage VARCHAR,
+ payment_limit DOUBLE,
+ source_file VARCHAR,
+ source_url VARCHAR,
+ item_key VARCHAR
+)
"""
@@ -61,3 +85,79 @@ def _select(con: Any, cls: type, table: str, where: str = "", params: Sequence =
def _chunks(items: Sequence[Any], n: int = 500) -> Iterable[Sequence[Any]]:
for i in range(0, len(items), n):
yield items[i : i + n]
+
+
+# ── asp_payment / asp_noc ────────────────────────────────────────────
+@dataclass(frozen=True)
+class AspRow:
+ """One line of a quarterly Part B payment-limit file. ``list_kind`` is
+ ``payment`` or ``not-payable`` (no Medicaid rebate agreement; carries the
+ file's manufacturer). ``short_desc`` is ``""`` for a licensed (CPT/CDT)
+ code. ``payment_limit`` is the CMS payment allowance (ASP + 6%), never
+ the ASP itself."""
+
+ quarter: str
+ hcpcs: str
+ level: str
+ list_kind: str
+ dosage: str
+ payment_limit: float | None
+ coinsurance_pct: float | None
+ short_desc: str
+ notes: str
+ source_file: str
+ source_url: str
+ item_key: str
+ manufacturer: str = ""
+
+
+@dataclass(frozen=True)
+class NocRow:
+ """One line of a quarterly not-otherwise-classified drug pricing file."""
+
+ quarter: str
+ drug_name: str
+ dosage: str
+ payment_limit: float | None
+ source_file: str
+ source_url: str
+ item_key: str
+
+
+def write_asp(con: Any, quarter: str, rows: Sequence[AspRow]) -> int:
+ """Replace *quarter* of ``drugs.asp_payment`` with *rows*."""
+ ensure_tables(con)
+ con.execute("DELETE FROM drugs.asp_payment WHERE quarter = ?", [quarter])
+ return _insert(con, "drugs.asp_payment", rows)
+
+
+def read_asp(
+ con: Any, *, hcpcs: str | None = None, quarter: str | None = None
+) -> list[AspRow]:
+ where, params = [], []
+ if hcpcs is not None:
+ where.append("hcpcs = ?")
+ params.append(hcpcs)
+ if quarter is not None:
+ where.append("quarter = ?")
+ params.append(quarter)
+ rows = _select(con, AspRow, "drugs.asp_payment", " AND ".join(where), params)
+ return sorted(
+ (AspRow(*r) for r in rows),
+ key=lambda r: (r.quarter, r.hcpcs, r.list_kind, r.dosage),
+ )
+
+
+def write_noc(con: Any, quarter: str, rows: Sequence[NocRow]) -> int:
+ """Replace *quarter* of ``drugs.asp_noc`` with *rows*."""
+ ensure_tables(con)
+ con.execute("DELETE FROM drugs.asp_noc WHERE quarter = ?", [quarter])
+ return _insert(con, "drugs.asp_noc", rows)
+
+
+def read_noc(con: Any, quarter: str | None = None) -> list[NocRow]:
+ where, params = ("quarter = ?", [quarter]) if quarter is not None else ("", [])
+ rows = _select(con, NocRow, "drugs.asp_noc", where, params)
+ return sorted(
+ (NocRow(*r) for r in rows), key=lambda r: (r.quarter, r.drug_name, r.dosage)
+ )
diff --git a/tests/cli/test_drugs_cli.py b/tests/cli/test_drugs_cli.py
index 4e95e35..f78ac78 100644
--- a/tests/cli/test_drugs_cli.py
+++ b/tests/cli/test_drugs_cli.py
@@ -2,12 +2,20 @@
from __future__ import annotations
+import io
+import json
+import zipfile
+
import duckdb
+import httpx
import pytest
from typer.testing import CliRunner
import cli.drugs as dg
+from bib.store import Store
from cli import app
+from drugs import asp as A
+from drugs import http as dh
from drugs import tables as T
from market import tables as MT
@@ -140,3 +148,376 @@ def test_status_gap_read_error_propagates(env, monkeypatch):
monkeypatch.setattr(dg, "_read", lambda: _Con())
assert isinstance(runner.invoke(app, ["drugs", "status"]).exception, RuntimeError)
+
+
+# ── stack drugs asp ──────────────────────────────────────────────────
+PRICING = (
+ "Payment Allowance Limits,,,,,,,,,,,\r\n"
+ "HCPCS Code,Short Description,HCPCS Code Dosage,Payment Limit,"
+ "Co-insurance Percentage,Vaccine AWP%,Vaccine Limit,Blood AWP%,Blood limit,"
+ "Clotting Factor,Notes\r\n"
+ "90371,x,1 ML,140.209,20.000,,,,,,\r\n"
+ "J9035,Bevacizumab injection,10 MG,66.123,20.000,,,,,,\r\n"
+)
+NP = (
+ "HCPCS Code,Long Description,Manufacturer\r\nJ7330,Chondrocytes implant,Vericel\r\n"
+)
+NOC = "Drug Generic Name (Trade Name),Dosage,Payment Limit,Notes\r\nExamplumab,10 MG,12.5,\r\n"
+
+
+def _zip(members):
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as zf:
+ for n, b in members.items():
+ zf.writestr(n, b)
+ return buf.getvalue()
+
+
+PAGE = """
+July 2026 Medicare Part B Payment Limit Files
+July 2026 NDC-HCPCS Crosswalk
+April 2018 ASP Pricing File
+April 2018 NOC Pricing File
+2026-2027 Seasonal Vaccine Price File
+"""
+
+ROUTES = {
+ "/medicare/payment/part-b-drugs/asp-pricing-files": PAGE.encode(),
+ "/files/zip/july-2026-medicare-part-b-payment-limit-files.zip": _zip(
+ {
+ "section 508 version of July 2026 Payment Limit File.csv": PRICING,
+ "section 508 version of July 2026 Drugs Not Payable.csv": NP,
+ }
+ ),
+ "/files/zip/july-2026-ndc-hcpcs-crosswalk.zip": _zip({"x.csv": "a"}),
+ "/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip": _zip({"NOC.csv": NOC}),
+ "/Medicare/x/downloads/2018-April-ASP-Pricing-File.zip": _zip({"ASP.csv": PRICING}),
+}
+
+
+class _NoWait:
+ def wait(self):
+ pass
+
+
+@pytest.fixture
+def asp_env(env, monkeypatch, tmp_path):
+ c, published = env
+ monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
+ monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
+ dh._MANIFESTS.clear()
+ store = Store(":memory:", storage_dir=tmp_path / "st")
+ monkeypatch.setattr(dg, "_store", lambda: store)
+ seen: list[httpx.Request] = []
+
+ def handler(req):
+ seen.append(req)
+ body = ROUTES.get(req.url.path)
+ return (
+ httpx.Response(200, content=body)
+ if body is not None
+ else httpx.Response(404)
+ )
+
+ monkeypatch.setattr(
+ dh,
+ "client",
+ lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
+ )
+ monkeypatch.setattr(A, "today", lambda: __import__("datetime").date(2026, 8, 1))
+ return c, published, seen, tmp_path, store
+
+
+def test_asp_cli_fetch_write(asp_env):
+ c, published, seen, root, store = asp_env
+ MT.write_gap(c, "cms-asp:asp", "stale")
+ res = runner.invoke(
+ app,
+ [
+ "drugs",
+ "asp",
+ "--fetch",
+ "--write",
+ "--quarter",
+ "2026-Q3",
+ "--quarter",
+ "2018-Q2",
+ ],
+ )
+ assert res.exit_code == 0, res.output
+ pay = [r for r in T.read_asp(c) if r.quarter == "2026-Q3"]
+ assert {r.quarter for r in T.read_asp(c)} == {"2018-Q2", "2026-Q3"}
+ assert [(r.quarter, r.hcpcs, r.list_kind) for r in pay] == [
+ ("2026-Q3", "90371", "payment"),
+ ("2026-Q3", "J7330", "not-payable"),
+ ("2026-Q3", "J9035", "payment"),
+ ]
+ assert pay[0].short_desc == "" and pay[2].short_desc == "Bevacizumab injection"
+ assert pay[1].manufacturer == "Vericel"
+ assert all(
+ r.item_key
+ and r.source_url.startswith("https://www.cms.gov/files/zip/july-2026")
+ for r in pay
+ )
+ noc = T.read_noc(c)
+ assert [(n.quarter, n.drug_name, n.payment_limit) for n in noc] == [
+ ("2018-Q2", "Examplumab", 12.5)
+ ]
+ assert (
+ noc[0].source_url
+ == "https://www.cms.gov/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip"
+ )
+ assert {
+ k
+ for (k,) in c.execute(
+ "SELECT DISTINCT table_name FROM cms.ingest_log"
+ ).fetchall()
+ } == {
+ "drugs.asp_payment",
+ "drugs.asp_noc",
+ }
+ assert published == [True]
+ assert [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"] == []
+ assert "stale" not in " ".join(g.note for g in MT.read_gaps(c))
+ # every request carried the honest user agent through the real client factory
+ hoard = root / "cms" / "asp"
+ assert (hoard / "asp_2026-Q3.zip").exists()
+ assert (hoard / "noc" / "noc_2018-Q2.zip").exists()
+ assert (hoard / "xwalk" / "xwalk_2026-Q3.zip").exists()
+ assert (root / "drugs" / "cms" / "asp-pricing-files.html").exists()
+ manifest = [
+ json.loads(x) for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
+ ]
+ assert len(manifest) == 5
+ # vaccine zip is never fetched
+ assert not any("vaccine" in str(r.url) for r in seen)
+ assert "2026-Q3" in res.output and "not-payable" in res.output
+ cites = [i for i in store.list_items() if "ASP" in i.title]
+ assert len(cites) == 3
+ # a second run fetches nothing but the page
+ n = len(seen)
+ again = runner.invoke(
+ app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3", "--quarter", "2018-Q2"]
+ )
+ assert again.exit_code == 0, again.output
+ assert len(seen) == n + 1
+
+
+def test_asp_cli_gap_note_written_for_missing(asp_env):
+ c, published, seen, root, store = asp_env
+ res = runner.invoke(
+ app,
+ [
+ "drugs",
+ "asp",
+ "--fetch",
+ "--write",
+ "--quarter",
+ "2026-Q3",
+ "--quarter",
+ "2026-Q2",
+ ],
+ )
+ assert res.exit_code == 0, res.output
+ gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"]
+ assert len(gaps) == 1 and "2026-Q2 pricing" in gaps[0].note
+ assert "gap: cms-asp:asp: no hoarded file for: 2026-Q2 pricing" in res.output
+
+
+def test_asp_cli_dry_run_lists_links(asp_env):
+ c, published, seen, root, store = asp_env
+ first = runner.invoke(app, ["drugs", "asp", "--fetch", "--dry-run"])
+ assert first.exit_code == 0, first.output
+ assert [r.url.path for r in seen] == [
+ "/medicare/payment/part-b-drugs/asp-pricing-files"
+ ]
+ out = first.output
+ assert (
+ "pricing 2026-Q3 fetch https://www.cms.gov/files/zip/july-2026-medicare-part-b-payment-limit-files.zip"
+ in out
+ )
+ assert "noc 2018-Q2 fetch" in out and "crosswalk 2026-Q3 fetch" in out
+ assert "vaccine" not in out
+ assert not (root / "cms").exists()
+ assert T.table_counts(c) == {}
+ assert published == []
+ # --write is ignored by a dry run
+ res = runner.invoke(
+ app, ["drugs", "asp", "--dry-run", "--write", "--quarter", "2018-Q2"]
+ )
+ assert res.exit_code == 0 and published == [] and "noc 2018-Q2 fetch" in res.output
+ assert "crosswalk 2026-Q3" not in res.output
+
+
+def test_asp_cli_refusal_prints_gap(asp_env, monkeypatch):
+ c, published, seen, root, store = asp_env
+ monkeypatch.setattr(
+ dh,
+ "client",
+ lambda source, key=None: httpx.Client(
+ transport=httpx.MockTransport(lambda r: httpx.Response(403))
+ ),
+ )
+ res = runner.invoke(app, ["drugs", "asp", "--fetch"])
+ assert res.exit_code == 0, res.output
+ lines = [x for x in res.output.splitlines() if x.strip()]
+ assert len(lines) == 1 and lines[0].startswith("gap: cms-asp:asp:")
+ assert published == []
+ assert T.table_counts(c) == {} and not (root / "cms").exists()
+ res = runner.invoke(app, ["drugs", "asp", "--fetch", "--write"])
+ assert [g.source for g in MT.read_gaps(c)] == ["cms-asp:asp"]
+ assert published == [True]
+ assert T.table_counts(c) == {}
+
+
+def test_asp_cli_refusal_on_a_zip_stops_fetching(asp_env, monkeypatch):
+ c, published, seen, root, store = asp_env
+
+ def handler(req):
+ if req.url.path.endswith("asp-pricing-files"):
+ return httpx.Response(200, content=PAGE.encode())
+ return httpx.Response(403)
+
+ monkeypatch.setattr(
+ dh,
+ "client",
+ lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
+ )
+ res = runner.invoke(
+ app, ["drugs", "asp", "--fetch", "--write", "--quarter", "2026-Q3"]
+ )
+ assert res.exit_code == 0, res.output
+ assert "gap: cms-asp:asp: refused" in res.output
+ assert "2026-Q3 pricing" in res.output
+ assert T.read_asp(c) == []
+
+
+def test_asp_cli_parse_only_reads_hoard_without_cites(asp_env):
+ c, published, seen, root, store = asp_env
+ assert (
+ runner.invoke(
+ app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3"]
+ ).exit_code
+ == 0
+ )
+ assert T.table_counts(c) == {}
+ seen.clear()
+ res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q3"])
+ assert res.exit_code == 0 and seen == []
+ assert "2026-Q3" in res.output and "3 payment" not in res.output
+ assert "2 payment" in res.output and "1 not-payable" in res.output
+ assert store.list_items() == [] and published == []
+
+
+def test_asp_cli_unreadable_zip_is_a_gap(asp_env):
+ c, published, seen, root, store = asp_env
+ d = root / "cms" / "asp"
+ d.mkdir(parents=True)
+ (d / "asp_2026-Q3.zip").write_bytes(b"not a zip")
+ res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
+ assert res.exit_code == 0, res.output
+ gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"]
+ assert gaps and "2026-Q3 pricing unreadable" in gaps[0].note
+ assert T.read_asp(c) == []
+
+
+def test_asp_cli_source_url_falls_back_to_csv(asp_env):
+ c, published, seen, root, store = asp_env
+ d = root / "cms" / "asp"
+ d.mkdir(parents=True)
+ (d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING}))
+ (root / "cms" / "asp_source_urls.csv").write_text(
+ "quarter,url\n2026-Q3,https://cms.test/canon.zip\n"
+ )
+ res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
+ assert res.exit_code == 0, res.output
+ assert {r.source_url for r in T.read_asp(c)} == {"https://cms.test/canon.zip"}
+ assert all(r.item_key for r in T.read_asp(c))
+
+
+def test_asp_cli_no_url_no_cite(asp_env):
+ c, published, seen, root, store = asp_env
+ d = root / "cms" / "asp"
+ d.mkdir(parents=True)
+ (d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING}))
+ res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
+ assert res.exit_code == 0, res.output
+ assert {(r.source_url, r.item_key) for r in T.read_asp(c)} == {("", "")}
+
+
+def test_asp_cli_refresh_refetches(asp_env):
+ c, published, seen, root, store = asp_env
+ assert (
+ runner.invoke(
+ app, ["drugs", "asp", "--fetch", "--quarter", "2018-Q2"]
+ ).exit_code
+ == 0
+ )
+ n = len(seen)
+ ROUTES_BEFORE = dict(ROUTES)
+ res = runner.invoke(
+ app, ["drugs", "asp", "--fetch", "--refresh", "--quarter", "2018-Q2"]
+ )
+ assert (
+ res.exit_code == 0 and len(seen) == n + 3
+ ) # page, the pricing zip and the NOC zip
+ assert ROUTES == ROUTES_BEFORE
+
+
+def test_asp_cli_unfiltered_uses_expected_quarters(asp_env):
+ c, published, seen, root, store = asp_env
+ res = runner.invoke(app, ["drugs", "asp"])
+ assert res.exit_code == 0, res.output
+ assert "gap: cms-asp:asp: no hoarded file for: 2005-Q1 pricing" in res.output
+ assert "(+" in res.output and published == []
+ assert seen == []
+
+
+def test_asp_today_is_real_date():
+ import datetime
+
+ assert A.today() == datetime.date.today()
+
+
+def test_asp_cli_rejects_bad_quarter(asp_env):
+ res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q5"])
+ assert res.exit_code != 0 and "not a quarter" in res.output
+
+
+def test_asp_cli_unreadable_noc_zip_is_a_gap(asp_env):
+ c, published, seen, root, store = asp_env
+ d = root / "cms" / "asp" / "noc"
+ d.mkdir(parents=True)
+ (d / "noc_2018-Q2.zip").write_bytes(b"not a zip")
+ res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2018-Q2"])
+ assert res.exit_code == 0, res.output
+ assert "2018-Q2 noc unreadable" in res.output
+ assert T.read_noc(c) == []
+
+
+def test_asp_cli_bundled_noc_from_pricing_zip(asp_env):
+ c, published, seen, root, store = asp_env
+ d = root / "cms" / "asp"
+ d.mkdir(parents=True)
+ (d / "asp_2006-Q2.zip").write_bytes(
+ _zip({"ASPbyHCPCS.csv": PRICING, "ASPNOC.csv": NOC})
+ )
+ res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2006-Q2"])
+ assert res.exit_code == 0, res.output
+ assert [n.drug_name for n in T.read_noc(c)] == ["Examplumab"]
+ assert "1 noc" in res.output
+
+
+def test_asp_cli_dry_run_without_page(asp_env):
+ res = runner.invoke(app, ["drugs", "asp", "--dry-run"])
+ assert res.exit_code == 0 and "no page hoarded; run with --fetch" in res.output
+
+
+def test_asp_cli_noc_only_quarter_is_listed(asp_env):
+ c, published, seen, root, store = asp_env
+ d = root / "cms" / "asp" / "noc"
+ d.mkdir(parents=True)
+ (d / "noc_2018-Q2.zip").write_bytes(_zip({"NOC.csv": NOC}))
+ res = runner.invoke(app, ["drugs", "asp", "--quarter", "2018-Q2"])
+ assert res.exit_code == 0, res.output
+ assert "2018-Q2: 1 noc" in res.output
diff --git a/tests/dev/test_ingest_asp_parity.py b/tests/dev/test_ingest_asp_parity.py
new file mode 100644
index 0000000..33c72f1
--- /dev/null
+++ b/tests/dev/test_ingest_asp_parity.py
@@ -0,0 +1,134 @@
+"""The skin-substitute ingest reads the same ASP parser as the drug tier, and
+its output is unchanged (1,974 rows, 2009-Q1..2025-Q4)."""
+
+from __future__ import annotations
+
+import csv
+import importlib.util
+import math
+import zipfile
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+ASP_DIR = ROOT / "data" / "cms" / "asp"
+SKIN_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv"
+REF_CSV = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv"
+
+_spec = importlib.util.spec_from_file_location(
+ "_ingest_asp", ROOT / "dev" / "scripts" / "ingest_asp.py"
+)
+ingest = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(ingest)
+
+needs_data = pytest.mark.skipif(
+ not (SKIN_CSV.exists() and any(ASP_DIR.glob("asp_*.zip"))),
+ reason="needs the hoarded data/cms/asp files",
+)
+
+
+def _legacy_pick(names: list[str]) -> str | None:
+ """The selection the dev script made before the shared parser (frozen)."""
+ skip = ["noc", "crosswalk", "ndc", "not payable", "not_payable"]
+ cand = [
+ n
+ for n in names
+ if not any(s in n.lower() for s in skip)
+ and n.lower().endswith((".csv", ".xls", ".xlsx"))
+ ]
+ csvs = [c for c in cand if c.lower().endswith(".csv")]
+ if csvs:
+ for c in csvs:
+ if "payment limit" in c.lower() or "pricing" in c.lower():
+ return c
+ return csvs[0]
+ xlss = [c for c in cand if c.lower().endswith((".xls", ".xlsx"))]
+ if xlss:
+ for c in xlss:
+ low = c.lower()
+ if "payment limit" in low or "pricing" in low or "asp" in low:
+ return c
+ return xlss[0]
+ return None
+
+
+@needs_data
+def test_pick_pricing_file_over_every_hoarded_zip():
+ from drugs import asp
+
+ n = 0
+ for z in sorted(ASP_DIR.glob("asp_*.zip")):
+ names = zipfile.ZipFile(z).namelist()
+ want = _legacy_pick(names)
+ assert ingest.pick_pricing_file(names) == want, z.name
+ assert asp.pick_pricing_file(names) == want, z.name
+ n += 1
+ assert n >= 86
+
+
+@needs_data
+def test_skin_rows_match_hoarded_csv():
+ df = ingest.build_skin_df()
+ got = sorted(
+ df.to_dict("records"),
+ key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
+ )
+ with SKIN_CSV.open(newline="") as fh:
+ want = sorted(
+ csv.DictReader(fh),
+ key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]),
+ )
+ assert len(want) == 1974
+ assert len(got) == len(want), "an extra or missing skin row appeared"
+ for g, w in zip(got, want, strict=True):
+ for col in (
+ "quarter",
+ "hcpcs_code",
+ "short_description",
+ "dosage_or_unit",
+ "source_file",
+ "source_url",
+ ):
+ gv = g[col]
+ gv = "" if gv is None or (isinstance(gv, float) and math.isnan(gv)) else gv
+ assert gv == w[col], (g["quarter"], g["hcpcs_code"], col)
+ gp = g["payment_limit"]
+ if w["payment_limit"] == "":
+ assert gp is None or math.isnan(gp)
+ else:
+ assert abs(gp - float(w["payment_limit"])) < 1e-9
+ assert abs(g["asp_per_unit"] - float(w["asp_per_unit"])) < 1e-9
+
+
+def test_reference_csv_carries_no_cpt_descriptor_text():
+ with REF_CSV.open(newline="") as fh:
+ rows = list(csv.DictReader(fh))
+ assert len(rows) == 34
+ assert list(rows[0]) == [
+ "hcpcs_code",
+ "short_descriptor",
+ "code_type",
+ "body_site",
+ "wound_size",
+ "effective_date",
+ "termination_date",
+ "status",
+ "notes",
+ ]
+ cpt = [r for r in rows if "CPT" in r["code_type"]]
+ assert {r["hcpcs_code"] for r in cpt} >= {"15271", "15278", "15330", "15430"}
+ assert all(r["short_descriptor"] == "" for r in cpt)
+ # CMS's own HCPCS Level II text stays
+ level2 = {r["hcpcs_code"]: r["short_descriptor"] for r in rows if r not in cpt}
+ assert level2["C5271"].startswith("Low cost skin substitute graft")
+ assert level2["G0440"] and level2["C9363"]
+ assert all(r["short_descriptor"] for r in rows if r not in cpt)
+
+
+def test_refuses_to_replace_table_with_nothing():
+ import pandas as pd
+
+ with pytest.raises(RuntimeError, match="no skin-substitute rows"):
+ ingest.require_rows(pd.DataFrame())
+ ingest.require_rows(pd.DataFrame({"a": [1]}))
diff --git a/tests/drugs/fixtures/asp_page_head.html b/tests/drugs/fixtures/asp_page_head.html
new file mode 100644
index 0000000..62908e2
--- /dev/null
+++ b/tests/drugs/fixtures/asp_page_head.html
@@ -0,0 +1,17 @@
+
+ASP Pricing Files
+
+
+
+
+
+
diff --git a/tests/drugs/test_asp.py b/tests/drugs/test_asp.py
new file mode 100644
index 0000000..115900a
--- /dev/null
+++ b/tests/drugs/test_asp.py
@@ -0,0 +1,694 @@
+import json
+import os
+import zipfile
+from datetime import date
+from pathlib import Path
+
+import duckdb
+import httpx
+import pytest
+
+from drugs import asp
+from drugs import http as dh
+from drugs import tables as T
+
+FIXTURE = Path(__file__).parent / "fixtures" / "asp_page_head.html"
+
+
+class _NoWait:
+ def wait(self):
+ pass
+
+
+@pytest.fixture
+def root(monkeypatch, tmp_path):
+ monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
+ monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
+ dh._MANIFESTS.clear()
+ return tmp_path
+
+
+# ── the page ─────────────────────────────────────────────────────────
+def test_parse_asp_page_kinds_and_quarters():
+ links = asp.parse_asp_page(FIXTURE.read_text())
+ got = {(lk.quarter, lk.kind): lk.url for lk in links}
+ assert got[("2026-Q4", "pricing")] == (
+ "https://www.cms.gov/files/zip/"
+ "october-2026-medicare-part-b-payment-limit-files-final.zip"
+ )
+ assert got[("2026-Q4", "crosswalk")] == (
+ "https://www.cms.gov/files/zip/october-2026-ndc-hcpcs-crosswalk-final.zip"
+ )
+ vaccine = [lk for lk in links if lk.kind == "vaccine"]
+ assert len(vaccine) == 2 and {lk.quarter for lk in vaccine} == {""}
+ assert ("2018-Q2", "noc") in got and ("2018-Q1", "pricing") in got
+ assert got[("2005-Q1", "pricing")] == (
+ "https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/"
+ "McrPartBDrugAvgSalesPrice/downloads/jan05aspbyhcpcsv6_033106.zip"
+ )
+ assert ("2005-Q1", "crosswalk") in got
+ assert len(links) == 8
+ assert links[0].text == "October 2026 Medicare Part B Payment Limit Files"
+
+
+def test_parse_asp_page_edge_cases():
+ html = """
+ 2023 ASP Pricing File (ZIP)
+ 2023 NOC Pricing File
+ ASP Pricing File
+ January 2006 ASP Pricing File/ASP NOC Pricing File/ASP NDC-HCPCS Crosswalk
+ July 2006 ASP NDC-HCPCS Crosswalk - Updated
+ October 2009 ASP NOC Pricing File
+ Something else entirely
+ ASP Pricing File
+ Guide to the files (PDF)
+ no href at all
+ Oct. 2005 ASP Pricing File and ASP NOC Pricing File
+ """
+ links = asp.parse_asp_page(html)
+ got = [(lk.quarter, lk.kind) for lk in links]
+ assert got == [
+ ("2023-Q1", "pricing"),
+ ("2023-Q2", "noc"),
+ ("2018-Q3", "pricing"),
+ ("2006-Q1", "pricing"),
+ ("2006-Q3", "crosswalk"),
+ ("2009-Q4", "noc"),
+ ("", "other"),
+ ("", "other"),
+ ("2005-Q4", "pricing"),
+ ]
+ assert links[1].text == "2023 NOC Pricing File"
+
+
+def test_direct_url_unwraps_license():
+ p = "/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/x/downloads/a.zip"
+ want = "https://www.cms.gov" + p
+ assert asp.direct_url(f"https://www.cms.gov/apps/ama/license.asp?file={p}") == want
+ assert asp.direct_url(f"/apps/ama/license.asp?file={p}") == want
+ assert asp.direct_url(f"https://www.cms.gov/license/ama?file={p}") == want
+ assert asp.direct_url("/files/zip/a.zip") == "https://www.cms.gov/files/zip/a.zip"
+ assert asp.direct_url("https://x.test/a.zip") == "https://x.test/a.zip"
+
+
+def test_hoard_path_and_newest_per_quarter(root):
+ def lk(kind):
+ return asp.AspLink("2026-Q2", kind, "u", "t")
+
+ assert asp.hoard_path(lk("pricing")) == root / "cms/asp/asp_2026-Q2.zip"
+ assert asp.hoard_path(lk("noc")) == root / "cms/asp/noc/noc_2026-Q2.zip"
+ assert asp.hoard_path(lk("crosswalk")) == root / "cms/asp/xwalk/xwalk_2026-Q2.zip"
+ with pytest.raises(ValueError):
+ asp.hoard_path(lk("vaccine"))
+
+
+def test_one_zip_per_quarter_newest_wins(tmp_path):
+ a = tmp_path / "asp_2026-Q2.zip"
+ b = tmp_path / "asp_2026-Q2.ab12cd34.zip"
+ c = tmp_path / "asp_2026-Q1.zip"
+ junk = tmp_path / "asp_notes.zip"
+ for p in (a, b, c, junk):
+ p.write_bytes(b"x")
+ os.utime(a, (1_000, 1_000))
+ os.utime(b, (2_000, 2_000))
+ got = asp.newest_per_quarter([a, b, c, junk])
+ assert got == {"2026-Q1": c, "2026-Q2": b}
+ # the older copy still wins when it is the newer file
+ os.utime(a, (3_000, 3_000))
+ assert asp.newest_per_quarter([a, b])["2026-Q2"] == a
+ assert asp.newest_per_quarter([]) == {}
+
+
+def test_expected_quarters():
+ q = asp.expected_quarters(date(2026, 11, 13))
+ assert q[0] == "2005-Q1" and q[-1] == "2026-Q4" and len(q) == 88
+ assert asp.expected_quarters(date(2005, 1, 1)) == ["2005-Q1"]
+ assert asp.expected_quarters(date(2005, 3, 31)) == ["2005-Q1"]
+ assert asp.expected_quarters(date(2006, 2, 1)) == [
+ "2005-Q1",
+ "2005-Q2",
+ "2005-Q3",
+ "2005-Q4",
+ "2006-Q1",
+ ]
+
+
+# ── the member logic ─────────────────────────────────────────────────
+def test_col_map_covers_dev_script_and_new_columns():
+ dev = {
+ "hcpcs code": "hcpcs_code",
+ "hcpcs_code": "hcpcs_code",
+ "short description": "short_description",
+ "short_description": "short_description",
+ "hcpcs code dosage": "dosage_or_unit",
+ "hcpcs_code_dosage": "dosage_or_unit",
+ "dosage": "dosage_or_unit",
+ "dosage or unit": "dosage_or_unit",
+ "payment limit": "payment_limit",
+ "payment_limit": "payment_limit",
+ }
+ for k, v in dev.items():
+ assert asp.COL_MAP[k] == v
+ assert asp.COL_MAP["co-insurance percentage"] == "coinsurance_pct"
+ assert asp.COL_MAP["notes"] == "notes"
+ assert asp.COL_MAP["long description"] == "short_description"
+ assert asp.COL_MAP["manufacturer"] == "manufacturer"
+ assert asp.COL_MAP["drug generic name (trade name)"] == "drug_name"
+
+
+def test_find_header_row():
+ assert asp.find_header_row(["a", "Note: HCPCS code x,y", "HCPCS Code,Foo"]) == 2
+ assert asp.find_header_row(['"HCPCS Code","Foo"']) == 0
+ assert asp.find_header_row(["", "Drug Generic Name (Trade Name),Dosage"]) == 1
+ assert asp.find_header_row(["Drug Name (Generic),Dosage"]) == 0
+ assert asp.find_header_row(["nothing here"]) == -1
+ assert asp.find_header_row([]) == -1
+
+
+NAMES_2015Q3 = [
+ "Jul 15 ASP Pricing File - updated 090115.xls",
+ "section 508 version of Jul 15 ASP Pricing File - updated 090115.csv",
+]
+NAMES_2026Q2 = [
+ "April 2026 Drugs Not Payable Under Part B 030926.xls",
+ "April 2026 Medicare Part B Payment Limit File 032426.xls",
+ "section 508 version of April 2026 Drugs Not Payable Under Part B 030926.csv",
+ "section 508 version of April 2026 Medicare Part B Payment Limit File 032426.csv",
+]
+
+
+def test_pick_pricing_file_matches_dev_script():
+ assert asp.pick_pricing_file(["Jan05ASPbyHCPCSv6_033106.xls"]) == (
+ "Jan05ASPbyHCPCSv6_033106.xls"
+ )
+ assert asp.pick_pricing_file(NAMES_2015Q3) == NAMES_2015Q3[1]
+ assert asp.pick_pricing_file(
+ [
+ "January 2026 Medicare Part B Payment Limit File updated 022526.xls",
+ "section 508 version of January 2026 Medicare Part B Payment Limit "
+ "File updated 022526.csv",
+ ]
+ ).endswith("022526.csv")
+ assert asp.pick_pricing_file(NAMES_2026Q2) == NAMES_2026Q2[3]
+ assert asp.pick_pricing_file(["a.xls", "b.pdf"]) == "a.xls"
+ assert asp.pick_pricing_file(["ASP.xlsx", "other.xls"]) == "ASP.xlsx"
+ assert (
+ asp.pick_pricing_file(["other.csv", "payment limit.csv"]) == "payment limit.csv"
+ )
+ assert asp.pick_pricing_file(["x.csv"]) == "x.csv"
+ assert (
+ asp.pick_pricing_file(["Oct 2025 NOC Pricing File.csv", "readme.txt"]) is None
+ )
+ assert asp.pick_pricing_file(["Crosswalk.csv", "ndc.xls"]) is None
+
+
+def test_pick_member_prefers_csv_over_its_xls_twin():
+ # P7: one member per kind per zip
+ assert asp.pick_member(NAMES_2026Q2, "not-payable") == NAMES_2026Q2[2]
+ assert asp.pick_member(NAMES_2026Q2, "noc") is None
+ names = [
+ "1 - Apr06_ASPbyHCPCS_122107.xls",
+ "1 - Apr06_ASPbyHCPCS_122107.csv",
+ "1 - Apr06_ASPNOC_20Sep06.xls",
+ "1 - Apr06_ASPNOC_20Sep06.csv",
+ "April06ASPNDC-HCPCSCrosswalk_20Mar06.xls",
+ ]
+ assert asp.pick_member(names, "noc") == "1 - Apr06_ASPNOC_20Sep06.csv"
+ assert asp.pick_member(names[2:3] + names[4:], "noc") == names[2]
+ assert asp.pick_member(["Drugs_Not_Payable.xlsx"], "not-payable") == (
+ "Drugs_Not_Payable.xlsx"
+ )
+ assert asp.pick_member(["x.csv"], "noc") is None
+ with pytest.raises(ValueError):
+ asp.pick_member(names, "crosswalk")
+
+
+# ── reading tables ───────────────────────────────────────────────────
+PRICING_HEAD = (
+ "Payment Allowance Limits for Medicare Part B Drugs,,,,,,,,,,,\r\n"
+ ",,,,,,,,,,,\r\n"
+ "Note 1: HCPCS code lines mention HCPCS code here,,,,,,,,,,,\r\n"
+ "HCPCS Code,Short Description,HCPCS Code Dosage,Payment Limit,"
+ "Co-insurance Percentage,Vaccine AWP%,Vaccine Limit,Blood AWP%,Blood limit,"
+ "Clotting Factor,Notes\r\n"
+)
+PRICING_ROWS = (
+ "90371,x,1 ML,140.209,20.000,,,,,,\r\n"
+ 'J9035,Bevacizumab injection,10 MG,"$66.123",20.000,,,,,,\r\n'
+)
+NP_CSV = (
+ "Drugs Not Payable Under Part B,,,\r\n"
+ ",,,\r\n"
+ "HCPCS Code,Long Description,Manufacturer,,,,\r\n"
+ 'J7330," Autologous cultured chondrocytes, implant ",Vericel Corporation,,,,\r\n'
+ "90999,x,Acme,,,,\r\n"
+ ",,,,,,\r\n"
+)
+NOC_CSV = (
+ "Payment Allowance Limits for NOC Drugs,,,\r\n"
+ ",,,\r\n"
+ "Drug Generic Name (Trade Name),Dosage,Payment Limit,Notes\r\n"
+ "Examplumab,10 MG,12.5,\r\n"
+ "Vasopressin (Long Grove),1 UNIT,$2.462,\r\n"
+ ",,,\r\n"
+)
+
+
+def make_zip(path: Path, members: dict[str, str | bytes]) -> Path:
+ with zipfile.ZipFile(path, "w") as zf:
+ for name, body in members.items():
+ zf.writestr(
+ name, body if isinstance(body, bytes) else body.encode("latin-1")
+ )
+ return path
+
+
+def test_cpt_short_description_never_stored(tmp_path):
+ z = make_zip(
+ tmp_path / "asp_2026-Q2.zip",
+ {
+ "section 508 version of April 2026 Payment Limit File.csv": PRICING_HEAD
+ + PRICING_ROWS
+ },
+ )
+ rows, noc = asp.parse_quarter_zip(
+ z, "2026-Q2", source_url="https://cms.test/a.zip", item_key="K1"
+ )
+ assert noc == []
+ by = {r.hcpcs: r for r in rows}
+ assert by["90371"].level == "I" and by["90371"].short_desc == ""
+ assert by["90371"].dosage == "1 ML" and by["90371"].payment_limit == 140.209
+ assert by["90371"].coinsurance_pct == 20.0
+ j = by["J9035"]
+ assert j.level == "II" and j.short_desc == "Bevacizumab injection"
+ assert j.payment_limit == 66.123 and j.list_kind == "payment"
+ assert j.quarter == "2026-Q2" and j.item_key == "K1"
+ assert j.source_url == "https://cms.test/a.zip"
+ assert j.source_file == "section 508 version of April 2026 Payment Limit File.csv"
+ assert "Bevacizumab" not in " ".join(r.short_desc for r in rows if r.level == "I")
+
+
+def test_numbers_codes_and_blank_rows(tmp_path):
+ body = (
+ PRICING_HEAD
+ + ' j9035 ,Bevacizumab,10 MG,"$1,066.50",,,,,,,see note\r\n'
+ + "J0000,Unparseable,1 MG,n/a,abc,,,,,,\r\n"
+ + "J0001,Short row\r\n"
+ + ",,,,,,,,,,\r\n"
+ + " ,blank code,,,,,,,,,\r\n"
+ + "J0002,Too many,1 MG,1.0,20,,,,,,,extra,fields\r\n"
+ + "D0120,dental,1,2.0,,,,,,,\r\n"
+ )
+ z = make_zip(tmp_path / "asp_2020-Q1.zip", {"ASP Pricing File.csv": body})
+ rows, _ = asp.parse_quarter_zip(z, "2020-Q1", source_url="")
+ by = {r.hcpcs: r for r in rows}
+ # a line with more fields than columns is malformed and skipped
+ assert set(by) == {"J9035", "J0000", "J0001", "D0120"}
+ assert by["J9035"].payment_limit == 1066.5 and by["J9035"].notes == "see note"
+ assert by["J0000"].payment_limit is None and by["J0000"].coinsurance_pct is None
+ assert by["J0001"].dosage == "" and by["J0001"].short_desc == "Short row"
+ assert by["D0120"].level == "CDT" and by["D0120"].short_desc == ""
+ assert by["J9035"].item_key == "" and by["J9035"].source_url == ""
+
+
+def test_not_payable_rows_kept(tmp_path):
+ z = make_zip(
+ tmp_path / "asp_2026-Q2.zip",
+ {
+ "April 2026 Drugs Not Payable Under Part B 030926.xls": b"not read",
+ "section 508 version of April 2026 Drugs Not Payable Under Part B 030926.csv": NP_CSV,
+ "section 508 version of April 2026 Payment Limit File.csv": PRICING_HEAD
+ + PRICING_ROWS,
+ },
+ )
+ rows, _ = asp.parse_quarter_zip(z, "2026-Q2", source_url="u")
+ np_rows = [r for r in rows if r.list_kind == "not-payable"]
+ assert len(np_rows) == 2 and len(rows) == 4
+ j = next(r for r in np_rows if r.hcpcs == "J7330")
+ assert j.payment_limit is None and j.coinsurance_pct is None
+ assert j.manufacturer == "Vericel Corporation"
+ assert j.short_desc == " Autologous cultured chondrocytes, implant "
+ assert j.source_file.endswith("Under Part B 030926.csv")
+ lvl1 = next(r for r in np_rows if r.hcpcs == "90999")
+ assert lvl1.level == "I" and lvl1.short_desc == "" and lvl1.manufacturer == "Acme"
+ assert all(r.manufacturer == "" for r in rows if r.list_kind == "payment")
+
+
+def test_noc_member_parsed(tmp_path):
+ z = make_zip(
+ tmp_path / "noc_2025-Q4.zip",
+ {
+ "Oct 2025 NOC Pricing File 090425.xls": b"not read",
+ "section 508 version of Oct 2025 NOC Pricing File 090425.csv": NOC_CSV,
+ },
+ )
+ rows, noc = asp.parse_quarter_zip(
+ z, "2025-Q4", source_url="https://cms.test/n.zip", item_key="K2"
+ )
+ assert rows == []
+ assert [(n.drug_name, n.dosage, n.payment_limit) for n in noc] == [
+ ("Examplumab", "10 MG", 12.5),
+ ("Vasopressin (Long Grove)", "1 UNIT", 2.462),
+ ]
+ assert noc[0].quarter == "2025-Q4" and noc[0].item_key == "K2"
+ assert noc[0].source_url == "https://cms.test/n.zip"
+ assert noc[0].source_file.endswith("090425.csv")
+
+
+def test_bundled_zip_gives_both_and_one_member_per_kind(tmp_path):
+ # 2006-style zip: pricing, NOC and crosswalk, each as .xls and .csv twins
+ z = make_zip(
+ tmp_path / "asp_2006-Q2.zip",
+ {
+ "1 - Apr06_ASPbyHCPCS.xls": b"x",
+ "1 - Apr06_ASPbyHCPCS.csv": PRICING_HEAD + PRICING_ROWS,
+ "1 - Apr06_ASPNOC.xls": b"x",
+ "1 - Apr06_ASPNOC.csv": NOC_CSV,
+ "April06ASPNDC-HCPCSCrosswalk.csv": "HCPCS Code,NDC\r\nJ9035,1\r\n",
+ },
+ )
+ rows, noc = asp.parse_quarter_zip(z, "2006-Q2", source_url="")
+ assert [r.hcpcs for r in rows] == ["90371", "J9035"]
+ assert len(noc) == 2
+
+
+def test_header_missing_gives_no_rows(tmp_path):
+ z = make_zip(
+ tmp_path / "asp_2020-Q1.zip", {"ASP Pricing File.csv": "a,b\r\n1,2\r\n"}
+ )
+ assert asp.parse_quarter_zip(z, "2020-Q1", source_url="") == ([], [])
+ z2 = make_zip(tmp_path / "asp_2020-Q2.zip", {"readme.txt": "hi"})
+ assert asp.parse_quarter_zip(z2, "2020-Q2", source_url="") == ([], [])
+
+
+class _Sheet:
+ def __init__(self, grid):
+ self.grid = grid
+ self.nrows = len(grid)
+ self.ncols = max(len(r) for r in grid)
+
+ def cell_value(self, r, c):
+ row = self.grid[r]
+ return row[c] if c < len(row) else ""
+
+
+class _Book:
+ def __init__(self, grid, sheets=None):
+ self.sheets = [("Sheet1", _Sheet(grid))] + [
+ (name, _Sheet(g)) for name, g in (sheets or [])
+ ]
+
+ def sheet_names(self):
+ return [n for n, _ in self.sheets]
+
+ def sheet_by_index(self, i):
+ return self.sheets[i][1]
+
+
+XLS_GRID = [
+ ["Payment Allowance Limits"],
+ [""],
+ ["HCPCS Code", "Short Description", "HCPCS Code Dosage", "Payment Limit"],
+ [90371.0, "x", "1 ML", 140.209],
+ ["J9035 ", "Bevacizumab injection", "10 MG", 66.123],
+ ["", "", "", ""],
+ ["J0001", "Whole", "1 MG", 5.0],
+]
+
+
+def test_xls_member_through_fake_workbook(tmp_path):
+ z = make_zip(
+ tmp_path / "asp_2005-Q1.zip", {"Jan05ASPbyHCPCSv6_033106.xls": b"BIFF"}
+ )
+ seen = {}
+
+ def open_wb(**kw):
+ seen.update(kw)
+ return _Book(XLS_GRID)
+
+ rows, noc = asp.parse_quarter_zip(
+ z, "2005-Q1", source_url="u", open_workbook=open_wb
+ )
+ assert seen == {"file_contents": b"BIFF"}
+ assert noc == []
+ assert [
+ (r.hcpcs, r.level, r.short_desc, r.dosage, r.payment_limit) for r in rows
+ ] == [
+ ("90371", "I", "", "1 ML", 140.209),
+ ("J9035", "II", "Bevacizumab injection", "10 MG", 66.123),
+ ("J0001", "II", "Whole", "1 MG", 5.0),
+ ]
+ csv_z = make_zip(
+ tmp_path / "asp_2005-Q2.zip",
+ {"ASP.csv": PRICING_HEAD + PRICING_ROWS},
+ )
+ csv_rows, _ = asp.parse_quarter_zip(csv_z, "2005-Q2", source_url="u")
+ assert [(r.hcpcs, r.dosage, r.payment_limit) for r in csv_rows] == [
+ (r.hcpcs, r.dosage, r.payment_limit) for r in rows[:2]
+ ]
+
+
+def test_xls_without_header_and_default_workbook(tmp_path, monkeypatch):
+ z = make_zip(tmp_path / "asp_2005-Q1.zip", {"ASP.xls": b"BIFF"})
+ assert asp.parse_quarter_zip(
+ z, "2005-Q1", source_url="", open_workbook=lambda **kw: _Book([["a"], ["b"]])
+ ) == ([], [])
+ import sys
+ import types
+
+ fake = types.SimpleNamespace(open_workbook=lambda **kw: _Book(XLS_GRID))
+ monkeypatch.setitem(sys.modules, "xlrd", fake)
+ rows, _ = asp.parse_quarter_zip(z, "2005-Q1", source_url="")
+ assert len(rows) == 3
+
+
+def test_noc_xls_member(tmp_path):
+ grid = [
+ ["Drug Generic Name (Trade Name)", "Dosage", "Payment Limit"],
+ ["Abatacept", "250 MG", 477.0],
+ ["", "", ""],
+ ]
+ z = make_zip(tmp_path / "noc_2008-Q1.zip", {"Jan08 NOC.xls": b"BIFF"})
+ _, noc = asp.parse_quarter_zip(
+ z, "2008-Q1", source_url="", open_workbook=lambda **kw: _Book(grid)
+ )
+ assert [(n.drug_name, n.dosage, n.payment_limit) for n in noc] == [
+ ("Abatacept", "250 MG", 477.0)
+ ]
+
+
+# ── manifest and gaps ────────────────────────────────────────────────
+def test_manifest_urls_and_known_urls(root):
+ assert asp.manifest_urls() == set()
+ d = root / "cms" / "asp"
+ d.mkdir(parents=True)
+ f = d / "asp_2026-Q1.zip"
+ f.write_bytes(b"z")
+ dh.manifest(f, "https://cms.test/q1.zip")
+ assert asp.manifest_urls() == {"https://cms.test/q1.zip"}
+ (root / "cms" / "asp_source_urls.csv").write_text(
+ "quarter,url\n2026-Q2,https://cms.test/q2.zip\n"
+ )
+ assert asp.canonical_urls() == {"2026-Q2": "https://cms.test/q2.zip"}
+
+
+def test_plan_fetch_actions(root):
+ d = root / "cms" / "asp"
+ d.mkdir(parents=True)
+ (d / "asp_2026-Q1.zip").write_bytes(b"q1")
+ (d / "asp_2026-Q2.zip").write_bytes(b"q2")
+ (d / "asp_2025-Q4.zip").write_bytes(b"q4")
+ dh.manifest(d / "asp_2025-Q4.zip", "https://cms.test/2025q4.zip")
+ (root / "cms" / "asp_source_urls.csv").write_text(
+ "quarter,url\n2026-Q2,https://cms.test/q2-a.zip\n2026-Q1,https://cms.test/q1.zip\n"
+ )
+
+ def lk(q, kind, url):
+ return asp.AspLink(q, kind, url, "t")
+
+ links = [
+ lk("2025-Q4", "pricing", "https://cms.test/2025q4.zip"), # in manifest
+ lk("2026-Q1", "pricing", "https://cms.test/q1.zip"), # canonical, adopt
+ lk("2026-Q2", "pricing", "https://cms.test/q2-b.zip"), # updated posting
+ lk("2026-Q3", "pricing", "https://cms.test/q3.zip"), # new
+ lk("2026-Q3", "noc", "https://cms.test/n3.zip"),
+ lk("", "vaccine", "https://cms.test/v.zip"),
+ ]
+ plan = asp.plan_fetch(links, refresh=False)
+ assert [(p.link.quarter, p.link.kind, p.action) for p in plan] == [
+ ("2025-Q4", "pricing", "have"),
+ ("2026-Q1", "pricing", "adopt"),
+ ("2026-Q2", "pricing", "fetch-beside"),
+ ("2026-Q3", "pricing", "fetch"),
+ ("2026-Q3", "noc", "fetch"),
+ ]
+ assert plan[3].dest == d / "asp_2026-Q3.zip"
+ forced = asp.plan_fetch(links, refresh=True)
+ assert {p.action for p in forced[:3]} == {"fetch-beside"}
+ assert forced[3].action == "fetch"
+
+
+def test_gap_pairs_and_note():
+ have = {"pricing": {"2026-Q1", "2026-Q3"}, "noc": set(), "crosswalk": set()}
+ listed = {("2026-Q3", "noc"), ("2026-Q3", "crosswalk")}
+ exp = ["2026-Q1", "2026-Q2", "2026-Q3"]
+ pairs = asp.gap_pairs(exp, have, listed)
+ assert pairs == ["2026-Q2 pricing", "2026-Q3 crosswalk", "2026-Q3 noc"]
+ assert asp.gap_note(pairs).startswith("no hoarded file for: 2026-Q2 pricing")
+ assert asp.gap_note([]) == ""
+ many = [f"{2000 + i}-Q1 pricing" for i in range(40)]
+ note = asp.gap_note(many)
+ assert note.endswith("(+28 more)") and "2011-Q1 pricing" in note
+
+
+def test_missing_quarter_is_gap_not_zero(tmp_path):
+ z1 = make_zip(
+ tmp_path / "asp_2026-Q1.zip", {"ASP.csv": PRICING_HEAD + PRICING_ROWS}
+ )
+ z3 = make_zip(
+ tmp_path / "asp_2026-Q3.zip", {"ASP.csv": PRICING_HEAD + PRICING_ROWS}
+ )
+ got = asp.newest_per_quarter([z1, z3])
+ have = {"pricing": set(got), "noc": set(), "crosswalk": set()}
+ pairs = asp.gap_pairs(["2026-Q1", "2026-Q2", "2026-Q3"], have, set())
+ assert "2026-Q2 pricing" in asp.gap_note(pairs)
+ con = duckdb.connect(":memory:")
+ for q, z in (("2026-Q1", z1), ("2026-Q3", z3)):
+ rows, _ = asp.parse_quarter_zip(z, q, source_url="")
+ asp.write_asp(con, q, rows)
+ assert asp.read_asp(con, quarter="2026-Q2") == []
+ assert len(asp.read_asp(con, quarter="2026-Q1")) == 2
+
+
+# ── a transport-level updated posting lands beside ──────────────────
+def test_updated_posting_lands_beside(root):
+ d = root / "cms" / "asp"
+ d.mkdir(parents=True)
+ old = d / "asp_2026-Q2.zip"
+ old.write_bytes(b"old posting")
+ dh.manifest(old, "https://cms.test/A.zip")
+ link = asp.AspLink("2026-Q2", "pricing", "https://cms.test/B.zip", "t")
+ (plan,) = asp.plan_fetch([link], refresh=False)
+ assert plan.action == "fetch-beside"
+ seen = []
+
+ def handler(req):
+ seen.append(str(req.url))
+ return httpx.Response(200, content=b"new posting")
+
+ cl = httpx.Client(transport=httpx.MockTransport(handler))
+ out = asp.apply_plan(cl, plan)
+ assert seen == ["https://cms.test/B.zip"]
+ assert out != old and out.read_bytes() == b"new posting"
+ assert old.read_bytes() == b"old posting"
+ urls = [
+ json.loads(x)["url"]
+ for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
+ ]
+ assert urls == ["https://cms.test/A.zip", "https://cms.test/B.zip"]
+ # it is now the newest copy of the quarter
+ assert asp.newest_per_quarter(d.glob("asp_*.zip")) == {"2026-Q2": out}
+ # a second plan sees B in the manifest
+ (again,) = asp.plan_fetch([link], refresh=False)
+ assert again.action == "have"
+
+
+def test_apply_plan_adopt_and_have(root):
+ d = root / "cms" / "asp"
+ d.mkdir(parents=True)
+ f = d / "asp_2026-Q1.zip"
+ f.write_bytes(b"q1")
+ (root / "cms" / "asp_source_urls.csv").write_text(
+ "quarter,url\n2026-Q1,https://cms.test/q1.zip\n"
+ )
+ link = asp.AspLink("2026-Q1", "pricing", "https://cms.test/q1.zip", "t")
+ (plan,) = asp.plan_fetch([link], refresh=False)
+ assert plan.action == "adopt"
+ boom = httpx.Client(transport=httpx.MockTransport(lambda r: pytest.fail("fetched")))
+ assert asp.apply_plan(boom, plan) == f
+ rec = json.loads((root / "drugs/MANIFEST.jsonl").read_text())
+ assert (
+ rec["url"] == "https://cms.test/q1.zip" and "asp_source_urls.csv" in rec["note"]
+ )
+ (plan2,) = asp.plan_fetch([link], refresh=False)
+ assert plan2.action == "have" and asp.apply_plan(boom, plan2) == f
+
+
+def test_page_hoard(root):
+ seen = []
+
+ def handler(req):
+ seen.append(req.headers["user-agent"])
+ return httpx.Response(200, content=b"one")
+
+ cl = httpx.Client(
+ headers={"User-Agent": "ua"}, transport=httpx.MockTransport(handler)
+ )
+ p = asp.fetch_page(cl)
+ assert (
+ p == root / "drugs/cms/asp-pricing-files.html"
+ and p.read_bytes() == b"one"
+ )
+ assert asp.hoarded_page() == p
+ cl2 = httpx.Client(
+ transport=httpx.MockTransport(
+ lambda r: httpx.Response(200, content=b"two")
+ )
+ )
+ p2 = asp.fetch_page(cl2)
+ assert p2 != p and p.read_bytes() == b"one"
+ assert asp.hoarded_page() == p2
+ assert (
+ asp.ASP_PAGE
+ == "https://www.cms.gov/medicare/payment/part-b-drugs/asp-pricing-files"
+ )
+
+
+def test_hoarded_page_absent(root):
+ assert asp.hoarded_page() is None
+
+
+# ── tables ───────────────────────────────────────────────────────────
+def test_write_noc_replaces_quarter():
+ con = duckdb.connect(":memory:")
+ a = T.NocRow("2025-Q4", "A", "1 MG", 1.0, "f", "u", "k")
+ b = T.NocRow("2025-Q3", "B", "1 MG", None, "f", "u", "k")
+ asp.write_noc(con, "2025-Q4", [a])
+ asp.write_noc(con, "2025-Q3", [b])
+ asp.write_noc(con, "2025-Q4", [T.NocRow("2025-Q4", "C", "", 2.0, "f", "u", "")])
+ assert [r.drug_name for r in asp.read_noc(con)] == ["B", "C"]
+ assert [r.drug_name for r in asp.read_noc(con, "2025-Q3")] == ["B"]
+
+
+def test_2005_style_noc_sheet_inside_the_pricing_workbook(tmp_path):
+ noc_grid = [
+ ["Note"],
+ ["Drug Name (Generic)", "Dosage", "Payment Limit", "Notes"],
+ ["Allopurinol Sodium", "500 MG", "455.714", "Revised"],
+ ["Amikacin Sulfate", "50 MG", "$0.492", ""],
+ ]
+ z = make_zip(tmp_path / "asp_2005-Q1.zip", {"Jan05ASPbyHCPCSv6.xls": b"BIFF"})
+ rows, noc = asp.parse_quarter_zip(
+ z,
+ "2005-Q1",
+ source_url="u",
+ open_workbook=lambda **kw: _Book(XLS_GRID, [("Jan05_NOC_rev", noc_grid)]),
+ )
+ assert len(rows) == 3
+ assert [(n.drug_name, n.payment_limit) for n in noc] == [
+ ("Allopurinol Sodium", 455.714),
+ ("Amikacin Sulfate", 0.492),
+ ]
+ assert noc[0].source_file == "Jan05ASPbyHCPCSv6.xls#NOC"
+ # no NOC sheet: no NOC rows
+ _, none = asp.parse_quarter_zip(
+ z, "2005-Q1", source_url="u", open_workbook=lambda **kw: _Book(XLS_GRID)
+ )
+ assert none == []
+
+
+def test_manufacturer_is_stripped(tmp_path):
+ body = "HCPCS Code,Long Description,Manufacturer\r\nJ0209,Some drug,\tHope Pharmaceuticals \r\n"
+ z = make_zip(tmp_path / "asp_2026-Q4.zip", {"Drugs Not Payable.csv": body})
+ (row,), _ = asp.parse_quarter_zip(z, "2026-Q4", source_url="")
+ assert row.manufacturer == "Hope Pharmaceuticals" and row.list_kind == "not-payable"
diff --git a/tests/drugs/test_codes.py b/tests/drugs/test_codes.py
index c99da51..2af65af 100644
--- a/tests/drugs/test_codes.py
+++ b/tests/drugs/test_codes.py
@@ -1,5 +1,7 @@
from datetime import date
+import pytest
+
from drugs import codes
@@ -34,7 +36,8 @@ def test_appl_id():
assert codes.appl_id("A", "90123") == "ANDA090123"
assert codes.appl_id("BLA", "125057") == "BLA125057"
assert codes.appl_id("NDA021436", "") == "NDA021436"
- assert codes.appl_id("X", "5") == "X000005"
+ assert codes.appl_id("NDA", "21436") == "NDA021436"
+ assert codes.appl_id("ANDA", "90123") == "ANDA090123"
def test_expand():
@@ -54,3 +57,31 @@ def test_quarters_and_fy():
assert codes.quarter_end("2026-Q1") == date(2026, 3, 31)
assert codes.fy_effective("2026") == date(2025, 10, 1)
assert codes.fy_effective("2026-04") == date(2026, 4, 1)
+
+
+def test_level_and_ndc_reject_trailing_newline_and_fullwidth_digits():
+ assert codes.level("J9035\n") == ""
+ assert codes.level("90371") == ""
+ assert codes.level("90371") == ""
+ assert codes.licensed("90371\n") is False
+ assert codes.ndc11("000214338011") is None
+ assert codes.ndc11("00021433801") is None
+ assert codes.ndc11("0002-1433-80") is None
+ assert codes.ndc11("0002-1433-80") == "00002143380"
+
+
+def test_appl_id_rejects_unknown_kinds():
+ with pytest.raises(ValueError, match="unknown application kind"):
+ codes.appl_id("X", "5")
+ with pytest.raises(ValueError):
+ codes.appl_id("", "5")
+
+
+def test_expand_caps_range_size():
+ assert len(codes.expand("J0001", "J0500")) == 500
+ assert codes.MAX_RANGE == 500
+ with pytest.raises(ValueError, match="range too large"):
+ codes.expand("J0001", "J9999")
+ with pytest.raises(ValueError):
+ codes.expand("J0001", "J0501")
+ assert codes.expand("J9001", "J9002") == ["J9001"]
diff --git a/tests/drugs/test_imports.py b/tests/drugs/test_imports.py
index 08c7138..3905e91 100644
--- a/tests/drugs/test_imports.py
+++ b/tests/drugs/test_imports.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import subprocess
import sys
-MODULES = ["drugs.http", "drugs.codes", "drugs.tables"]
+MODULES = ["drugs.http", "drugs.codes", "drugs.tables", "drugs.asp"]
_PROBE = """
import builtins, importlib, sys
diff --git a/tests/drugs/test_tables.py b/tests/drugs/test_tables.py
index 088eecf..2e5e825 100644
--- a/tests/drugs/test_tables.py
+++ b/tests/drugs/test_tables.py
@@ -1,4 +1,5 @@
import duckdb
+import pytest
from drugs import tables as T
@@ -16,7 +17,7 @@ def test_table_names_empty():
con = duckdb.connect(":memory:")
assert T.table_counts(con) == {}
T.ensure_tables(con)
- assert T.table_counts(con) == {}
+ assert T.table_counts(con) == {"drugs.asp_noc": 0, "drugs.asp_payment": 0}
def test_helpers():
@@ -37,3 +38,92 @@ def test_helpers():
1,
]
con.execute("CREATE TABLE drugs.x (a INTEGER)") if False else None
+
+
+ROUNDTRIP = {
+ "asp_payment": (
+ T.write_asp,
+ T.read_asp,
+ [
+ T.AspRow(
+ "2026-Q2",
+ "J9035",
+ "II",
+ "payment",
+ "10 MG",
+ 66.123,
+ 20.0,
+ "Bevacizumab injection",
+ "n",
+ "f.csv",
+ "https://x.test/a.zip",
+ "K1",
+ ),
+ T.AspRow(
+ "2026-Q2",
+ "J7330",
+ "II",
+ "not-payable",
+ "",
+ None,
+ None,
+ "Autologous cultured chondrocytes, implant",
+ "",
+ "np.csv",
+ "https://x.test/a.zip",
+ "K1",
+ "Vericel Corporation",
+ ),
+ ],
+ ),
+ "asp_noc": (
+ T.write_noc,
+ T.read_noc,
+ [
+ T.NocRow(
+ "2026-Q2",
+ "Examplumab",
+ "10 MG",
+ 12.5,
+ "n.csv",
+ "https://x.test/n.zip",
+ "K2",
+ ),
+ T.NocRow("2026-Q2", "Other", "", None, "n.csv", "https://x.test/n.zip", ""),
+ ],
+ ),
+}
+
+
+@pytest.mark.parametrize("name", sorted(ROUNDTRIP))
+def test_roundtrip(name):
+ write, read, rows = ROUNDTRIP[name]
+ con = duckdb.connect(":memory:")
+ assert write(con, "2026-Q2", rows) == len(rows)
+ assert sorted(read(con), key=repr) == sorted(rows, key=repr)
+ # writing a quarter again replaces it, other quarters are untouched
+ assert write(con, "2026-Q2", rows[:1]) == 1
+ assert read(con) == rows[:1]
+ assert T.table_counts(con)[f"drugs.{name}"] == 1
+
+
+def test_write_empty_quarter_clears_it():
+ con = duckdb.connect(":memory:")
+ rows = ROUNDTRIP["asp_payment"][2]
+ T.write_asp(con, "2026-Q2", rows)
+ assert T.write_asp(con, "2026-Q2", []) == 0
+ assert T.read_asp(con) == []
+
+
+def test_read_asp_filters():
+ con = duckdb.connect(":memory:")
+ a, b = ROUNDTRIP["asp_payment"][2]
+ c = T.AspRow(
+ "2026-Q1", "J9035", "II", "payment", "10 MG", 60.0, 20.0, "x", "", "f", "u", ""
+ )
+ T.write_asp(con, "2026-Q2", [a, b])
+ T.write_asp(con, "2026-Q1", [c])
+ assert [r.quarter for r in T.read_asp(con, hcpcs="J9035")] == ["2026-Q1", "2026-Q2"]
+ assert T.read_asp(con, quarter="2026-Q1") == [c]
+ assert T.read_asp(con, hcpcs="J7330", quarter="2026-Q2") == [b]
+ assert T.read_asp(con, hcpcs="ZZZZZ") == []