diff --git a/data/cms/skin_substitute_hcpcs_reference.csv b/data/cms/skin_substitute_hcpcs_reference.csv index 8735b9e..23cbc64 100644 --- a/data/cms/skin_substitute_hcpcs_reference.csv +++ b/data/cms/skin_substitute_hcpcs_reference.csv @@ -1,12 +1,12 @@ "hcpcs_code","short_descriptor","code_type","body_site","wound_size","effective_date","termination_date","status","notes" -"15271","Skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Replaced CPT 15330" -"15272","Skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Add-on to 15271" -"15273","Skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15331" -"15274","Skin substitute graft, trunk/arms/legs, >=100 sq cm; each addl 100 sq cm","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Add-on to 15273" -"15275","Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; first 25 sq cm or less","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Replaced CPT 15335" -"15276","Skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Add-on to 15275" -"15277","Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15336" -"15278","Skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Add-on to 15277" +"15271","","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Replaced CPT 15330" +"15272","","Application (CPT)","Trunk/arms/legs","<100 sq cm","2012-01-01","","Active","Add-on to 15271" +"15273","","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15331" +"15274","","Application (CPT)","Trunk/arms/legs",">=100 sq cm","2012-01-01","","Active","Add-on to 15273" +"15275","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Replaced CPT 15335" +"15276","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2012-01-01","","Active","Add-on to 15275" +"15277","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Replaced CPT 15336" +"15278","","Application (CPT)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2012-01-01","","Active","Add-on to 15277" "C5271","Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; first 25 sq cm or less","Application (OPPS Low-Cost)","Trunk/arms/legs","<100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026 per CY 2026 OPPS final rule" "C5272","Low cost skin substitute graft, trunk/arms/legs, <100 sq cm; each addl 25 sq cm","Application (OPPS Low-Cost)","Trunk/arms/legs","<100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026" "C5273","Low cost skin substitute graft, trunk/arms/legs, >=100 sq cm; first 100 sq cm","Application (OPPS Low-Cost)","Trunk/arms/legs",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026" @@ -15,15 +15,15 @@ "C5276","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, <100 sq cm; each addl 25 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia","<100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026" "C5277","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; first 100 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Deleted effective Jan 1 2026" "C5278","Low cost skin substitute graft, face/scalp/hands/feet/genitalia, >=100 sq cm; each addl 100 sq cm","Application (OPPS Low-Cost)","Face/scalp/hands/feet/digits/genitalia",">=100 sq cm","2014-01-01","2025-12-31","Deleted","Add-on. Deleted effective Jan 1 2026" -"15330","Acellular dermal allograft, trunk/arms/legs; first 100 sq cm or less","Application (CPT - Historical)","Trunk/arms/legs","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15271/15272 in 2012" -"15331","Acellular dermal allograft, trunk/arms/legs; each addl 100 sq cm","Application (CPT - Historical)","Trunk/arms/legs",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15273/15274 in 2012" -"15335","Acellular dermal allograft, face/scalp/hands/feet; first 100 sq cm or less","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15275/15276 in 2012" -"15336","Acellular dermal allograft, face/scalp/hands/feet; each addl 100 sq cm","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15277/15278 in 2012" -"15340","Tissue cultured allogeneic skin substitute; first 25 sq cm or less","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 tissue-cultured allogeneic graft code" -"15360","Tissue cultured autograft; first 25 sq cm or less","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 autologous cultured skin code" -"15400","Xenograft; first 100 sq cm or less","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft application code" -"15401","Xenograft; each addl 100 sq cm","Application (CPT - Historical)","",">100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft add-on code" -"15430","Acellular xenograft implant; first 100 sq cm or less","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 acellular xenograft code" +"15330","","Application (CPT - Historical)","Trunk/arms/legs","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15271/15272 in 2012" +"15331","","Application (CPT - Historical)","Trunk/arms/legs",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15273/15274 in 2012" +"15335","","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia","<=100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15275/15276 in 2012" +"15336","","Application (CPT - Historical)","Face/scalp/hands/feet/digits/genitalia",">100 sq cm","2006-01-01","2011-12-31","Deleted","Replaced by 15277/15278 in 2012" +"15340","","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 tissue-cultured allogeneic graft code" +"15360","","Application (CPT - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Pre-2012 autologous cultured skin code" +"15400","","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft application code" +"15401","","Application (CPT - Historical)","",">100 sq cm","","2011-12-31","Deleted","Pre-2012 xenograft add-on code" +"15430","","Application (CPT - Historical)","","<=100 sq cm","","2011-12-31","Deleted","Pre-2012 acellular xenograft code" "G0440","Skin substitute, tissue cultured; first 25 sq cm (OPPS)","Application (OPPS - Historical)","","<=25 sq cm","","2011-12-31","Deleted","Transitional OPPS code for pass-through products" "C9363","Integra Meshed Bilayer Wound Matrix, per sq cm","Product (C-code Historical)","","","","","Deleted","Transitional pass-through C-code; replaced by Q-code" "C9358","SurgiMend Collagen Matrix (fetal), per 0.5 sq cm","Product (C-code Historical)","","","","","Active","Bovine collagen - fetal origin" diff --git a/dev/scripts/ingest_asp.py b/dev/scripts/ingest_asp.py index 970ed32..aaba410 100644 --- a/dev/scripts/ingest_asp.py +++ b/dev/scripts/ingest_asp.py @@ -1,7 +1,8 @@ """Ingest CMS Average Sales Price (ASP) Drug Pricing Files. -Reads all quarterly ASP ZIP files from data/cms/asp/, normalises column -names across 20+ years of CMS format drift, filters **strictly** to +Reads all quarterly ASP ZIP files from data/cms/asp/ through the drug tier's +parser (``drugs.asp.parse_quarter_zip``: column-name drift, section-508 CSV +before its XLS twin, one file per quarter), filters **strictly** to skin substitute HCPCS codes defined in data/cms/skin_substitute_hcpcs.csv, and produces: @@ -15,8 +16,8 @@ quarterly ZIP and the file within it that was parsed. Source URLs are read from data/cms/asp_source_urls.csv (canonical URLs provided from CMS.gov, one per quarter 2005-Q1 through 2026-Q2). -Column mapping rationale ------------------------- +Column mapping rationale (the map itself now lives in ``drugs.asp.COL_MAP``) +--------------------------------------------------------------------------- CMS has changed column names across 20+ years of ASP files: 2005–2006: "HCPCS Code", "Short Description", "HCPCS Code Dosage", @@ -53,23 +54,34 @@ Usage: from __future__ import annotations import csv -import io -import re -import zipfile from pathlib import Path import pandas as pd +from drugs.asp import newest_per_quarter, parse_quarter_zip, pick_pricing_file # noqa: F401 + # --------------------------------------------------------------------------- # Paths # --------------------------------------------------------------------------- ROOT = Path(__file__).resolve().parents[2] ASP_DIR = ROOT / "data" / "cms" / "asp" HCPCS_REF = ROOT / "data" / "cms" / "skin_substitute_hcpcs.csv" +HCPCS_REFERENCE = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv" URL_REF = ROOT / "data" / "cms" / "asp_source_urls.csv" DUCKDB_PATH = ROOT / "data" / "aco.duckdb" OUTPUT_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv" +OUT_COLS = [ + "quarter", + "hcpcs_code", + "short_description", + "asp_per_unit", + "payment_limit", + "dosage_or_unit", + "source_url", + "source_file", +] + def _load_source_urls() -> dict[str, str]: """Load canonical source URLs: {quarter: url}.""" @@ -81,142 +93,74 @@ def _load_source_urls() -> dict[str, str]: return urls -# --------------------------------------------------------------------------- -# Column normalisation -# --------------------------------------------------------------------------- -# Mapping from observed CMS column names (lowercased) to our standard names. -# Each entry documents which CMS file years use that variant. -COL_MAP = { - # HCPCS code — consistent across all years - "hcpcs code": "hcpcs_code", - "hcpcs_code": "hcpcs_code", - # Short description — consistent - "short description": "short_description", - "short_description": "short_description", - # Dosage/unit — varies by year - "hcpcs code dosage": "dosage_or_unit", # 2005-2006, 2020-2025 - "hcpcs_code_dosage": "dosage_or_unit", - "dosage": "dosage_or_unit", # 2007-2019 - "dosage or unit": "dosage_or_unit", # 2026+ - # Payment limit — consistent (ASP + 6%) - "payment limit": "payment_limit", - "payment_limit": "payment_limit", -} - -KEEP_COLS = ["hcpcs_code", "short_description", "dosage_or_unit", "payment_limit"] +def require_rows(df: pd.DataFrame) -> None: + """Refuse to replace ``skin_subs.asp_quarterly`` with nothing: a parser + fault must not publish an empty table over the replica.""" + if df.empty: + raise RuntimeError( + "no skin-substitute rows parsed; leaving skin_subs.asp_quarterly as it is" + ) -def normalise_columns(df: pd.DataFrame) -> pd.DataFrame: - """Lower-case columns, apply COL_MAP, keep only KEEP_COLS.""" - df.columns = [str(c).strip().lower() for c in df.columns] - df = df.rename(columns=COL_MAP) - present = [c for c in KEEP_COLS if c in df.columns] - return df[present] - - -# --------------------------------------------------------------------------- -# Readers -# --------------------------------------------------------------------------- - - -def _find_header_row(lines: list[str]) -> int: - """Return the 0-based row index of the column-header row. - - The header row starts with 'HCPCS Code,' (possibly quoted). - Must NOT match note lines that incidentally mention 'HCPCS code'. - """ - for i, line in enumerate(lines): - stripped = line.strip().lower() - if stripped.startswith("hcpcs code,") or stripped.startswith('"hcpcs code",'): - return i - return -1 - - -def read_csv_from_zip(zf: zipfile.ZipFile, name: str) -> pd.DataFrame: - """Read a section-508 CSV inside a ZIP, skipping preamble rows.""" - raw = zf.read(name).decode("latin-1") - lines = raw.splitlines() - hdr = _find_header_row(lines) - if hdr < 0: - return pd.DataFrame() - buf = "\n".join(lines[hdr:]) - df = pd.read_csv(io.StringIO(buf), dtype=str, on_bad_lines="skip") - return normalise_columns(df) - - -def read_xls_from_zip(zf: zipfile.ZipFile, name: str) -> pd.DataFrame: - """Read .xls inside a ZIP, skipping preamble rows.""" - import xlrd - - data = zf.read(name) - wb = xlrd.open_workbook(file_contents=data) - sh = wb.sheet_by_index(0) - hdr_row = -1 - for r in range(min(30, sh.nrows)): - first_cell = str(sh.cell_value(r, 0)).strip().lower() - if first_cell == "hcpcs code": - hdr_row = r - break - if hdr_row < 0: - return pd.DataFrame() - headers = [str(sh.cell_value(hdr_row, c)).strip() for c in range(sh.ncols)] - rows = [] - for r in range(hdr_row + 1, sh.nrows): - row = [str(sh.cell_value(r, c)).strip() for c in range(sh.ncols)] - if any(row): - rows.append(row) - df = pd.DataFrame(rows, columns=headers) - return normalise_columns(df) - - -def pick_pricing_file(names: list[str]) -> str | None: - """From a list of files in a ZIP, pick the ASP pricing file. - - Prefer CSV (section 508) over XLS. Skip NOC, crosswalk, NDC, - and "not payable" files. - """ - skip_patterns = ["noc", "crosswalk", "ndc", "not payable", "not_payable"] - candidates: list[str] = [] - for n in names: - low = n.lower() - if any(s in low for s in skip_patterns): - continue - if low.endswith((".csv", ".xls", ".xlsx")): - candidates.append(n) - - csvs = [c for c in candidates if c.lower().endswith(".csv")] - if csvs: - for c in csvs: - low = c.lower() - if "payment limit" in low or "pricing" in low: - return c - return csvs[0] - - xlss = [c for c in candidates if c.lower().endswith((".xls", ".xlsx"))] - if xlss: - for c in xlss: - low = c.lower() - if "payment limit" in low or "pricing" in low or "asp" in low: - return c - return xlss[0] - - return None - - -# --------------------------------------------------------------------------- -# Numeric cleaning -# --------------------------------------------------------------------------- - - -def clean_numeric(series: pd.Series) -> pd.Series: - """Strip $ signs and convert to float, coercing errors to NaN.""" - return ( - series.astype(str) - .str.replace("$", "", regex=False) - .str.replace(",", "", regex=False) - .str.strip() - .pipe(pd.to_numeric, errors="coerce") +def build_skin_df() -> pd.DataFrame: + """Every payment-limit row of a skin-substitute code (strict filter: only + codes in skin_substitute_hcpcs.csv), one file per quarter, with + ``asp_per_unit = payment_limit / 1.06`` (CMS payment limit = ASP + 6% for + drugs/biologicals; no longer meaningful for the flat 2026+ rate, kept for + consistency).""" + skin_codes = set( + pd.read_csv(HCPCS_REF, dtype=str)["hcpcs_code"].str.strip().str.upper() ) + print(f"Loaded {len(skin_codes)} skin-substitute HCPCS codes from reference") + source_urls = _load_source_urls() + print(f"Loaded {len(source_urls)} source URLs") + zips = newest_per_quarter(ASP_DIR.glob("asp_*.zip")) + print(f"Found {len(zips)} quarterly ZIP files\n") + + records: list[dict[str, object]] = [] + seen_q4: set[str] = set() + total = 0 + for quarter, zpath in sorted(zips.items()): + try: + rows, _ = parse_quarter_zip( + zpath, quarter, source_url=source_urls.get(quarter, "") + ) + except Exception as e: # a bad zip must not hide the other quarters + print(f" ERR {quarter}: {e}") + continue + payable = [r for r in rows if r.list_kind == "payment"] + if not payable: + print(f" SKIP {quarter}: no pricing rows found in ZIP") + continue + total += len(payable) + print(f" OK {quarter}: {len(payable):>5} rows from {payable[0].source_file}") + for r in payable: + if r.hcpcs in skin_codes: + records.append( + { + "quarter": quarter, + "hcpcs_code": r.hcpcs, + "short_description": r.short_desc, + "payment_limit": r.payment_limit, + "dosage_or_unit": r.dosage, + "source_url": r.source_url, + "source_file": r.source_file, + } + ) + elif r.hcpcs.startswith("Q4") and len(r.hcpcs) in (4, 5): + seen_q4.add(r.hcpcs) + print(f"\nTotal rows across all quarters: {total}") + print(f"Skin substitute rows (strict filter): {len(records)}") + if seen_q4: + print(f" Excluded Q4 codes NOT in skin sub universe: {sorted(seen_q4)}") + + skin_df = pd.DataFrame.from_records(records) + if skin_df.empty: + return pd.DataFrame(columns=OUT_COLS) + skin_df["payment_limit"] = pd.to_numeric(skin_df["payment_limit"], errors="coerce") + # ASP per unit = payment_limit / 1.06 + skin_df["asp_per_unit"] = (skin_df["payment_limit"] / 1.06).round(3) + return skin_df[OUT_COLS].sort_values(["quarter", "hcpcs_code"]) # --------------------------------------------------------------------------- @@ -225,106 +169,17 @@ def clean_numeric(series: pd.Series) -> pd.Series: def main() -> None: - # Load skin-substitute HCPCS codes (strict filter — no regex fallback) - hcpcs_ref = pd.read_csv(HCPCS_REF, dtype=str) - skin_codes = set(hcpcs_ref["hcpcs_code"].str.strip().str.upper()) - print(f"Loaded {len(skin_codes)} skin-substitute HCPCS codes from reference") - - # Load canonical source URLs - source_urls = _load_source_urls() - print(f"Loaded {len(source_urls)} source URLs") - - # Ingest all quarters - zips = sorted(ASP_DIR.glob("asp_*.zip")) - print(f"Found {len(zips)} quarterly ZIP files\n") - - frames: list[pd.DataFrame] = [] - for zpath in zips: - m = re.match(r"asp_(\d{4}-Q\d)", zpath.name) - if not m: - print(f" SKIP {zpath.name} (cannot parse quarter)") - continue - quarter = m.group(1) - - try: - with zipfile.ZipFile(zpath) as zf: - target = pick_pricing_file(zf.namelist()) - if target is None: - print(f" SKIP {quarter}: no pricing file found in ZIP") - continue - - if target.lower().endswith(".csv"): - df = read_csv_from_zip(zf, target) - else: - df = read_xls_from_zip(zf, target) - - if df.empty: - print(f" SKIP {quarter}: empty after parsing {target}") - continue - - df["quarter"] = quarter - df["source_file"] = target - df["source_url"] = source_urls.get(quarter, "") - frames.append(df) - print(f" OK {quarter}: {len(df):>5} rows from {target}") - except Exception as e: - print(f" ERR {quarter}: {e}") - - if not frames: - raise RuntimeError("No data parsed from any file") - - all_df = pd.concat(frames, ignore_index=True) - print(f"\nTotal rows across all quarters: {len(all_df)}") - - # Normalise HCPCS codes - all_df["hcpcs_code"] = all_df["hcpcs_code"].astype(str).str.strip().str.upper() - - # Filter STRICTLY to skin substitute code universe - # NO regex fallback — only codes in skin_substitute_hcpcs.csv - skin_df = all_df[all_df["hcpcs_code"].isin(skin_codes)].copy() - print(f"Skin substitute rows (strict filter): {len(skin_df)}") - - # Show what we excluded - q4_but_not_skin = all_df[ - all_df["hcpcs_code"].str.match(r"^Q4\d{2,3}$") - & ~all_df["hcpcs_code"].isin(skin_codes) - ]["hcpcs_code"].unique() - if len(q4_but_not_skin) > 0: - print( - f" Excluded Q4 codes NOT in skin sub universe: {sorted(q4_but_not_skin)}" - ) - - if not skin_df.empty: - skin_df["payment_limit"] = clean_numeric(skin_df["payment_limit"]) - # ASP per unit = payment_limit / 1.06 - # (CMS payment limit = ASP + 6% for drugs/biologicals) - skin_df["asp_per_unit"] = (skin_df["payment_limit"] / 1.06).round(3) - - # Reorder columns - out_cols = [ - "quarter", - "hcpcs_code", - "short_description", - "asp_per_unit", - "payment_limit", - "dosage_or_unit", - "source_url", - "source_file", - ] - for c in out_cols: - if c not in skin_df.columns: - skin_df[c] = None - skin_df = skin_df[out_cols].sort_values(["quarter", "hcpcs_code"]) + skin_df = build_skin_df() + require_rows(skin_df) # Write CSV skin_df.to_csv(OUTPUT_CSV, index=False) print(f"\nWrote {len(skin_df)} rows to {OUTPUT_CSV}") - if len(skin_df) > 0: - quarters = sorted(skin_df["quarter"].unique()) - codes = sorted(skin_df["hcpcs_code"].unique()) - print(f" Quarters: {len(quarters)} ({quarters[0]} to {quarters[-1]})") - print(f" Unique HCPCS codes: {len(codes)}") + quarters = sorted(skin_df["quarter"].unique()) + codes = sorted(skin_df["hcpcs_code"].unique()) + print(f" Quarters: {len(quarters)} ({quarters[0]} to {quarters[-1]})") + print(f" Unique HCPCS codes: {len(codes)}") # Load into DuckDB print(f"\nLoading into DuckDB at {DUCKDB_PATH} ...") @@ -335,24 +190,26 @@ def main() -> None: with duckdb_batch("aco") as con: con.execute("CREATE SCHEMA IF NOT EXISTS skin_subs") con.execute("DROP TABLE IF EXISTS skin_subs.asp_quarterly") - if not skin_df.empty: + con.execute( + """ + CREATE TABLE skin_subs.asp_quarterly AS + SELECT * FROM read_csv_auto(?, header=true) + """, + [str(OUTPUT_CSV)], + ) + count = con.execute("SELECT count(*) FROM skin_subs.asp_quarterly").fetchone()[ + 0 + ] + print(f" Loaded {count} rows into skin_subs.asp_quarterly") + if HCPCS_REFERENCE.exists(): + # the scrubbed reference (CPT descriptors blanked; codes and + # HCPCS Level II text kept) + con.execute("DROP TABLE IF EXISTS skin_subs.hcpcs_reference") con.execute( - """ - CREATE TABLE skin_subs.asp_quarterly AS - SELECT * FROM read_csv_auto(?, header=true) - """, - [str(OUTPUT_CSV)], + "CREATE TABLE skin_subs.hcpcs_reference AS " + "SELECT * FROM read_csv_auto(?, header=true)", + [str(HCPCS_REFERENCE)], ) - count = con.execute( - "SELECT count(*) FROM skin_subs.asp_quarterly" - ).fetchone()[0] - print(f" Loaded {count} rows into skin_subs.asp_quarterly") - else: - print(" WARNING: No skin substitute data found in ASP files") - print( - " This may be expected — skin subs may not appear in main ASP pricing files" - ) - print(" They may be in separate NOC or tissue coding files") # Refresh the notebook read replica (#510). print(f"replica → {publish_replica('aco')}") diff --git a/src/cli/drugs.py b/src/cli/drugs.py index f0bd902..2c5b086 100644 --- a/src/cli/drugs.py +++ b/src/cli/drugs.py @@ -2,12 +2,18 @@ from __future__ import annotations +import re +import zipfile from typing import Any import typer +from drugs import asp as A +from drugs import http as dh +from drugs import tables as T from drugs.tables import table_counts -from market.tables import read_gaps, write_gap +from market.http import Refused +from market.tables import clear_gap, read_gaps, write_gap from pfs.codetables import is_missing_table_error app = typer.Typer(no_args_is_help=True) @@ -89,3 +95,162 @@ def status() -> None: typer.echo("no gaps recorded") for g in gaps: typer.echo(f"gap: {g.source}: {g.note} ({g.recorded_on})") + + +# ── asp ────────────────────────────────────────────────────────────── +ASP_GAP = "cms-asp:asp" + + +def _cite(store: Any, url: str, title: str, table: str) -> str: + if not url: + return "" + return dh.cite_file(store, url=url, title=title, provider="cms-asp", table=table) + + +def _quarters_option(values: list[str] | None) -> list[str]: + for v in values or []: + if not re.fullmatch(r"[0-9]{4}-Q[1-4]", v): + raise typer.BadParameter(f"{v!r} is not a quarter like 2026-Q4") + return list(values or []) + + +@app.command() +def asp( + fetch: bool = typer.Option(False, "--fetch", help="Hoard the page and its zips."), + refresh: bool = typer.Option(False, "--refresh", help="Re-download every zip."), + quarter: list[str] = typer.Option(None, "--quarter", help="Only these quarters."), + dry_run: bool = typer.Option( + False, "--dry-run", help="List the links; change nothing." + ), + write: bool = typer.Option( + False, "--write", help="Write drugs.asp_payment and drugs.asp_noc." + ), +) -> None: + """Quarterly Part B payment limits for every code (payable, not-payable + and NOC rows) from the hoarded CMS ASP files.""" + sel = set(_quarters_option(quarter)) + write = write and not dry_run + cl = None + if fetch: + cl = dh.client("cms") + try: + A.fetch_page(cl) + except Refused as exc: + _gap(ASP_GAP, f"{A.ASP_PAGE} refused: {exc}", write=write) + return + page = A.hoarded_page() + links = [ + lk + for lk in (A.parse_asp_page(page.read_text(errors="replace")) if page else []) + if lk.kind in ("pricing", "noc", "crosswalk") and (not sel or lk.quarter in sel) + ] + steps = A.plan_fetch(links, refresh=refresh) + if dry_run: + if not links: + typer.echo("no page hoarded; run with --fetch") + for st in steps: + typer.echo(f"{st.link.kind} {st.link.quarter} {st.action} {st.link.url}") + return + refusal = "" + if fetch: + for st in steps: + if st.action == "have": + continue + try: + A.apply_plan(cl, st) + except Refused as exc: + refusal = f"refused: {exc}" + break + typer.echo(f"{st.action} {st.link.kind} {st.link.quarter}") + + def pick(kind: str, prefix: str) -> dict[str, Any]: + got = A.newest_per_quarter(A.hoard_dir(kind).glob(f"{prefix}_*.zip")) + return {q: p for q, p in got.items() if not sel or q in sel} + + pricing, nocs, xwalks = ( + pick("pricing", "asp"), + pick("noc", "noc"), + pick("crosswalk", "xwalk"), + ) + url_of = {(lk.quarter, lk.kind): lk.url for lk in links} + canon = A.canonical_urls() + store = _store() if write else None + + def source(q: str, kind: str, title: str, table: str) -> tuple[str, str]: + url = url_of.get((q, kind)) or (canon.get(q, "") if kind == "pricing" else "") + return url, (_cite(store, url, title, table) if write else "") + + payments: dict[str, list[T.AspRow]] = {} + noc_rows: dict[str, list[T.NocRow]] = {} + files: dict[tuple[str, str], Any] = {} + unreadable: list[str] = [] + for q, path in sorted(pricing.items()): + url, key = source( + q, "pricing", f"CMS ASP pricing file {q}", "drugs.asp_payment" + ) + try: + rows, bundled = A.parse_quarter_zip(path, q, source_url=url, item_key=key) + except zipfile.BadZipFile: + unreadable.append(f"{q} pricing unreadable") + continue + payments[q] = rows + files[(q, "drugs.asp_payment")] = path + if bundled and q not in nocs: + noc_rows[q] = bundled + files[(q, "drugs.asp_noc")] = path + for q, path in sorted(nocs.items()): + url, key = source(q, "noc", f"CMS ASP NOC pricing file {q}", "drugs.asp_noc") + try: + _, rows = A.parse_quarter_zip(path, q, source_url=url, item_key=key) + except zipfile.BadZipFile: + unreadable.append(f"{q} noc unreadable") + continue + noc_rows[q] = rows + files[(q, "drugs.asp_noc")] = path + + for q in sorted(payments): + rows = payments[q] + kinds = [r.list_kind for r in rows] + typer.echo( + f"{q}: {kinds.count('payment')} payment, {kinds.count('not-payable')} not-payable," + f" {len(noc_rows.get(q, []))} noc" + ) + for q in sorted(set(noc_rows) - set(payments)): + typer.echo(f"{q}: {len(noc_rows[q])} noc") + expected = sorted(sel) if sel else A.expected_quarters(A.today()) + typer.echo( + f"pricing {len(payments)}/{len(expected)} quarters," + f" {sum(len(r) for r in payments.values())} rows," + f" {sum(r.list_kind == 'not-payable' for v in payments.values() for r in v)} not-payable;" + f" noc {len(noc_rows)} quarters, {sum(len(r) for r in noc_rows.values())} rows" + ) + have = { + "pricing": set(pricing), + "noc": set(nocs) | set(noc_rows), + "crosswalk": set(xwalks), + } + listed = {(lk.quarter, lk.kind) for lk in links} + pairs = unreadable + A.gap_pairs(expected, have, listed) + note = "; ".join(x for x in (refusal, A.gap_note(pairs)) if x) + if note: + typer.echo(f"gap: {ASP_GAP}: {note}") + if not write: + return + from cms.ingest_log import log_ingest + + with _batch() as con: + T.ensure_tables(con) + for q, rows in payments.items(): + T.write_asp(con, q, rows) + for q, nrows in noc_rows.items(): + T.write_noc(con, q, nrows) + for (q, table), path in sorted(files.items()): + n = len(payments[q]) if table == "drugs.asp_payment" else len(noc_rows[q]) + log_ingest( + con, module="drugs.asp", table_name=table, rows=n, source_file=str(path) + ) + if note: + write_gap(con, ASP_GAP, note) + else: + clear_gap(con, ASP_GAP) + _publish() diff --git a/src/drugs/asp.py b/src/drugs/asp.py new file mode 100644 index 0000000..24fd6ae --- /dev/null +++ b/src/drugs/asp.py @@ -0,0 +1,594 @@ +"""CMS Medicare Part B drug payment-limit ("ASP") files, every quarter. + +One parser serves two readers: ``stack drugs asp`` (all codes, plus the +"drugs not payable" lists and the not-otherwise-classified drugs) and the +skin-substitute ingest (``dev/scripts/ingest_asp.py``). + +Rules held here: + +* CMS's short descriptions of CPT (Level I) codes are AMA text and are never + kept: a licensed code's description is ``""``. HCPCS Level II text is CMS's + own and is kept. +* A zip holds one member per kind (pricing, not-payable, NOC); when CMS ships + both a section-508 ``.csv`` and its ``.xls`` twin, the ``.csv`` is read and + the twin is not, so no row is counted twice. +* Files are hoarded under ``data/cms/asp`` and never overwritten; an updated + posting lands beside the old file and is the one read. +* A quarter with no file is a gap, never a zero. +""" + +from __future__ import annotations + +import csv +import io +import json +import re +import zipfile +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass +from datetime import date +from html.parser import HTMLParser +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urljoin, urlparse + +from drugs import http as dh +from drugs.codes import level, licensed +from drugs.tables import ( # noqa: F401 (re-exported) + AspRow, + NocRow, + read_asp, + read_noc, + write_asp, + write_noc, +) + +ASP_PAGE = "https://www.cms.gov/medicare/payment/part-b-drugs/asp-pricing-files" +CMS = "https://www.cms.gov" +FIRST_QUARTER = "2005-Q1" +KINDS = ("pricing", "noc", "crosswalk", "vaccine", "other") + + +def today() -> date: + return date.today() + + +# ── the page ───────────────────────────────────────────────────────── +@dataclass(frozen=True) +class AspLink: + quarter: str + kind: str + url: str + text: str + + +class _Anchors(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.found: list[tuple[str, str]] = [] + self._href: str | None = None + self._text: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + self._href = dict(attrs).get("href") + self._text = [] + + def handle_data(self, data: str) -> None: + if self._href is not None: + self._text.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag == "a" and self._href is not None: + text = re.sub(r"\s+", " ", "".join(self._text).replace("\xa0", " ")) + self.found.append((self._href, text.strip())) + self._href = None + + +_MONTH_Q = {"jan": 1, "apr": 2, "jul": 3, "oct": 4} +_TEXT_Q = re.compile(r"\b(jan|apr|jul|oct)[a-z]*\.?\s+(\d{4})\b") +_HREF_Q = re.compile(r"(january|april|july|october)[-_ ]?(\d{4})") +_HREF_YQ = re.compile(r"(\d{4})[-_ ](january|april|july|october)") + + +def _quarter_of(text: str, href: str) -> str: + m = _TEXT_Q.search(text.lower()) + if m: + return f"{m.group(2)}-Q{_MONTH_Q[m.group(1)]}" + low = href.lower() + m = _HREF_Q.search(low) + if m: + return f"{m.group(2)}-Q{_MONTH_Q[m.group(1)[:3]]}" + m = _HREF_YQ.search(low) + if m: + return f"{m.group(1)}-Q{_MONTH_Q[m.group(2)[:3]]}" + return "" + + +_KIND_MARKS = ( + ("crosswalk", ("crosswalk", "ndc")), + ("noc", ("noc",)), + ("pricing", ("asp pricing", "payment limit")), +) + + +def _kind_of(text: str) -> str: + """The first kind the link text names: a bundle such as "ASP Pricing + File/ASP NOC Pricing File/ASP NDC-HCPCS Crosswalk" is its pricing zip.""" + low = text.lower() + if "vaccine" in low: + return "vaccine" + best, where = "other", len(low) + 1 + for kind, marks in _KIND_MARKS: + for mark in marks: + i = low.find(mark) + if 0 <= i < where: + best, where = kind, i + return best + + +def parse_asp_page(html: str) -> list[AspLink]: + """Every zip link of the ASP pricing-files page, with its quarter and + kind. Vaccine and unrecognised links have quarter ``""``.""" + p = _Anchors() + p.feed(html) + out: list[AspLink] = [] + for href, text in p.found: + url = direct_url(href) + if not urlparse(url).path.lower().endswith(".zip"): + continue + kind = _kind_of(text) + quarter = _quarter_of(text, url) + if kind in ("vaccine", "other") or not quarter: + kind, quarter = ("vaccine" if kind == "vaccine" else "other"), "" + out.append(AspLink(quarter, kind, url, text)) + return out + + +def direct_url(href: str) -> str: + """The file URL behind a CMS click-through link + (``.../license.asp?file=

`` or ``.../license/ama?file=

`` -> + ``https://www.cms.gov

``); a relative href is made absolute.""" + parts = urlparse(href) + if parts.path.lower().endswith(("/license.asp", "/license/ama")): + target = parse_qs(parts.query).get("file", [""])[0] + if target: + return urljoin(CMS, target) + return urljoin(CMS, href) + + +def page_path() -> Path: + return dh.path("drugs", "cms", "asp-pricing-files.html") + + +def fetch_page(cl: Any) -> Path: + """Hoard the page (a changed page lands beside the old one).""" + return dh.fetch( + cl, ASP_PAGE, page_path(), provider="cms", source="cms-asp", refresh=True + ) + + +def hoarded_page() -> Path | None: + p = dh.newest(page_path()) + return p if p.is_file() else None + + +# ── hoard layout ───────────────────────────────────────────────────── +_DIRS = { + "pricing": ("asp", "asp"), + "noc": ("asp/noc", "noc"), + "crosswalk": ("asp/xwalk", "xwalk"), +} + + +def hoard_dir(kind: str) -> Path: + if kind not in _DIRS: + raise ValueError(f"no hoard for kind {kind!r}") + return dh.path("cms", *_DIRS[kind][0].split("/")) + + +def hoard_path(link: AspLink) -> Path: + return hoard_dir(link.kind) / f"{_DIRS[link.kind][1]}_{link.quarter}.zip" + + +_ZIP_Q = re.compile(r"^(?:asp|noc|xwalk)_(\d{4}-Q[1-4])(?:\.[0-9a-f]{8})?\.zip$") + + +def newest_per_quarter(paths: Iterable[Path]) -> dict[str, Path]: + """One path per quarter: the newest copy (``market.http.newest``'s order: + write time, then name), so an updated posting beside the original wins and + a quarter is never read twice.""" + best: dict[str, Path] = {} + for p in paths: + m = _ZIP_Q.match(p.name) + if not m: + continue + q = m.group(1) + key = (p.stat().st_mtime_ns, p.name) + if q not in best or key > (best[q].stat().st_mtime_ns, best[q].name): + best[q] = p + return best + + +def expected_quarters(today: date) -> list[str]: + """Every quarter from 2005-Q1 to the one containing *today*.""" + out: list[str] = [] + y, q = 2005, 1 + while (y, q) <= (today.year, (today.month - 1) // 3 + 1): + out.append(f"{y}-Q{q}") + y, q = (y + 1, 1) if q == 4 else (y, q + 1) + return out + + +# ── the manifest and what to fetch ────────────────────────────────── +def manifest_urls() -> set[str]: + mf = dh.data_root() / "drugs" / dh.MANIFEST + if not mf.exists(): + return set() + return {json.loads(x)["url"] for x in mf.read_text().splitlines() if x.strip()} + + +def canonical_urls() -> dict[str, str]: + """``data/cms/asp_source_urls.csv``: the URL each of the first 86 pricing + zips was downloaded from.""" + p = dh.path("cms", "asp_source_urls.csv") + if not p.exists(): + return {} + with p.open(newline="") as fh: + return {r["quarter"]: r["url"] for r in csv.DictReader(fh)} + + +@dataclass(frozen=True) +class Step: + link: AspLink + dest: Path + action: str # have | adopt | fetch | fetch-beside + + +def plan_fetch(links: Sequence[AspLink], *, refresh: bool) -> list[Step]: + """What to do with each pricing, NOC and crosswalk link: ``have`` (its + URL is in the manifest), ``adopt`` (the file on disk was downloaded from + this URL, per ``asp_source_urls.csv``; record it), ``fetch`` (nothing on + disk) or ``fetch-beside`` (a different posting of a hoarded quarter, or + *refresh*; it lands beside the old file).""" + known = manifest_urls() + canon = canonical_urls() + steps: list[Step] = [] + for link in links: + if link.kind not in _DIRS or not link.quarter: + continue + dest = hoard_path(link) + if not dest.exists(): + action = "fetch" + elif refresh: + action = "fetch-beside" + elif link.url in known: + action = "have" + elif link.kind == "pricing" and canon.get(link.quarter) == link.url: + action = "adopt" + else: + action = "fetch-beside" + steps.append(Step(link, dest, action)) + return steps + + +def apply_plan(cl: Any, step: Step) -> Path: + """Carry out one step; returns the hoarded path (the newest copy).""" + if step.action == "have": + return dh.newest(step.dest) + if step.action == "adopt": + dh.manifest( + step.dest, step.link.url, "adopted: downloaded from asp_source_urls.csv" + ) + return step.dest + return dh.fetch( + cl, + step.link.url, + step.dest, + provider="cms", + source="cms-asp", + refresh=True, + note=f"{step.link.kind} {step.link.quarter}", + ) + + +def gap_pairs( + expected: Sequence[str], + have: dict[str, set[str]], + listed: set[tuple[str, str]], +) -> list[str]: + """``" "`` for every expected quarter with no pricing file + and every NOC or crosswalk file the page lists but the hoard lacks.""" + pairs = [f"{q} pricing" for q in expected if q not in have["pricing"]] + for q, kind in sorted(listed): + if kind != "pricing" and q not in have.get(kind, set()): + pairs.append(f"{q} {kind}") + return pairs + + +def gap_note(pairs: Sequence[str], show: int = 12) -> str: + if not pairs: + return "" + note = "no hoarded file for: " + ", ".join(pairs[:show]) + if len(pairs) > show: + note += f" (+{len(pairs) - show} more)" + return note + + +# ── reading a file ─────────────────────────────────────────────────── +#: observed CMS column names (lower-cased) -> ours +COL_MAP = { + "hcpcs code": "hcpcs_code", + "hcpcs_code": "hcpcs_code", + "short description": "short_description", + "short_description": "short_description", + "long description": "short_description", # the not-payable list + "hcpcs code dosage": "dosage_or_unit", + "hcpcs_code_dosage": "dosage_or_unit", + "dosage": "dosage_or_unit", + "dosage or unit": "dosage_or_unit", + "payment limit": "payment_limit", + "payment_limit": "payment_limit", + "co-insurance percentage": "coinsurance_pct", + "notes": "notes", + "manufacturer": "manufacturer", # the not-payable list + "drug generic name (trade name)": "drug_name", # NOC files + "drug generic name": "drug_name", + "drug name (generic)": "drug_name", # the 2005 NOC sheet +} + +_HEADER_STARTS = ( + "hcpcs code,", + '"hcpcs code",', + "drug generic name", + '"drug generic name', + "drug name (generic)", +) + + +def find_header_row(lines: Sequence[str]) -> int: + """0-based index of the column-header row, or -1. The header starts with + ``HCPCS Code,`` (or ``Drug Generic Name`` in NOC files); a note line that + merely mentions HCPCS codes does not match.""" + for i, line in enumerate(lines): + if line.strip().lower().startswith(_HEADER_STARTS): + return i + return -1 + + +def _skip_member(low: str) -> bool: + return any( + s in low for s in ("noc", "crosswalk", "ndc", "not payable", "not_payable") + ) + + +def pick_pricing_file(names: Sequence[str]) -> str | None: + """The ASP pricing member of a zip: ``.csv`` before ``.xls``; NOC, + crosswalk, NDC and "not payable" members are skipped.""" + candidates = [ + n + for n in names + if not _skip_member(n.lower()) and n.lower().endswith((".csv", ".xls", ".xlsx")) + ] + csvs = [c for c in candidates if c.lower().endswith(".csv")] + if csvs: + for c in csvs: + low = c.lower() + if "payment limit" in low or "pricing" in low: + return c + return csvs[0] + xlss = [c for c in candidates if c.lower().endswith((".xls", ".xlsx"))] + if xlss: + for c in xlss: + low = c.lower() + if "payment limit" in low or "pricing" in low or "asp" in low: + return c + return xlss[0] + return None + + +_MEMBER_MARKS = { + "not-payable": ("not payable", "not_payable", "notpayable"), + "noc": ("noc",), +} + + +def pick_member(names: Sequence[str], kind: str) -> str | None: + """The one member of kind ``"not-payable"`` or ``"noc"``: its ``.csv`` + when CMS ships a twin, else its ``.xls``.""" + if kind not in _MEMBER_MARKS: + raise ValueError(f"no member kind {kind!r}") + marks = _MEMBER_MARKS[kind] + found = [ + n + for n in names + if n.lower().endswith((".csv", ".xls", ".xlsx")) + and any(m in n.lower() for m in marks) + and not ("crosswalk" in n.lower() or "ndc" in n.lower()) + ] + csvs = [n for n in found if n.lower().endswith(".csv")] + return (csvs or found or [None])[0] + + +def _cell(v: Any) -> str: + if isinstance(v, float) and v == int(v): + return str(int(v)) + return str(v).strip() + + +def read_table( + zf: zipfile.ZipFile, + name: str, + *, + open_workbook: Callable[..., Any] | None = None, + sheet: str = "", +) -> list[dict[str, str]]: + """The rows of a zip member as ``{our column: text}`` (only the columns of + :data:`COL_MAP`); preamble rows are skipped and blank rows dropped. A CSV + is latin-1 with its header row found; an ``.xls`` goes through + ``xlrd.open_workbook`` (injectable for tests): its first sheet, or with + *sheet* the first sheet whose name contains that text (the 2005 files keep + the NOC table on a second sheet).""" + if name.lower().endswith((".xls", ".xlsx")): + return _read_xls(zf, name, open_workbook, sheet) + lines = zf.read(name).decode("latin-1").splitlines() + hdr = find_header_row(lines) + if hdr < 0: + return [] + reader = csv.reader(io.StringIO("\n".join(lines[hdr:]), newline="")) + header = [COL_MAP.get(c.strip().lower()) for c in next(reader)] + rows: list[dict[str, str]] = [] + for raw in reader: + if len(raw) > len(header): + if any(c.strip() for c in raw[len(header) :]): + continue # more fields than columns: a malformed line + raw = raw[: len(header)] + row: dict[str, str] = {} + for col, val in zip(header, raw, strict=False): + if col and col not in row: + row[col] = val + if any(v.strip() for v in row.values()): + rows.append(row) + return rows + + +def _read_xls( + zf: zipfile.ZipFile, + name: str, + open_workbook: Callable[..., Any] | None, + sheet: str, +) -> list[dict[str, str]]: + if open_workbook is None: + import xlrd + + open_workbook = xlrd.open_workbook + wb = open_workbook(file_contents=zf.read(name)) + index = 0 + if sheet: + found = [i for i, n in enumerate(wb.sheet_names()) if sheet in n.lower()] + if not found: + return [] + index = found[0] + sh = wb.sheet_by_index(index) + hdr = -1 + for r in range(min(30, sh.nrows)): + first = str(sh.cell_value(r, 0)).strip().lower() + if first == "hcpcs code" or first.startswith( + ("drug generic name", "drug name (generic)") + ): + hdr = r + break + if hdr < 0: + return [] + header = [ + COL_MAP.get(_cell(sh.cell_value(hdr, c)).lower()) for c in range(sh.ncols) + ] + rows: list[dict[str, str]] = [] + for r in range(hdr + 1, sh.nrows): + vals = [_cell(sh.cell_value(r, c)) for c in range(sh.ncols)] + if not any(vals): + continue + row: dict[str, str] = {} + for col, val in zip(header, vals, strict=True): + if col and col not in row: + row[col] = val + rows.append(row) + return rows + + +def _number(raw: str | None) -> float | None: + s = (raw or "").replace("$", "").replace(",", "").strip() + if not s: + return None + try: + return float(s) + except ValueError: + return None + + +def _asp_rows( + rows: Sequence[dict[str, str]], + *, + quarter: str, + list_kind: str, + member: str, + source_url: str, + item_key: str, +) -> list[AspRow]: + out: list[AspRow] = [] + for r in rows: + code = r.get("hcpcs_code", "").strip().upper() + if not code: + continue + payable = list_kind == "payment" + out.append( + AspRow( + quarter=quarter, + hcpcs=code, + level=level(code), + list_kind=list_kind, + dosage=r.get("dosage_or_unit", ""), + payment_limit=_number(r.get("payment_limit")) if payable else None, + coinsurance_pct=_number(r.get("coinsurance_pct")) if payable else None, + short_desc="" if licensed(code) else r.get("short_description", ""), + notes=r.get("notes", ""), + source_file=member, + source_url=source_url, + item_key=item_key, + manufacturer=r.get("manufacturer", "").strip() if not payable else "", + ) + ) + return out + + +def parse_quarter_zip( + path: Path, + quarter: str, + *, + source_url: str, + item_key: str = "", + open_workbook: Callable[..., Any] | None = None, +) -> tuple[list[AspRow], list[NocRow]]: + """The payment, not-payable and NOC rows of one quarter's zip: one member + per kind. A licensed (CPT/CDT) code gets ``short_desc=""``.""" + asp: list[AspRow] = [] + noc: list[NocRow] = [] + with zipfile.ZipFile(path) as zf: + names = zf.namelist() + pricing = pick_pricing_file(names) + for kind, member in ( + ("payment", pricing), + ("not-payable", pick_member(names, "not-payable")), + ): + if member: + asp += _asp_rows( + read_table(zf, member, open_workbook=open_workbook), + quarter=quarter, + list_kind=kind, + member=member, + source_url=source_url, + item_key=item_key, + ) + member = pick_member(names, "noc") + table: list[dict[str, str]] = [] + if member: + table = read_table(zf, member, open_workbook=open_workbook) + elif pricing and pricing.lower().endswith((".xls", ".xlsx")): + table = read_table(zf, pricing, open_workbook=open_workbook, sheet="noc") + member = f"{pricing}#NOC" + if member: + for r in table: + name = r.get("drug_name", "").strip() + if name: + noc.append( + NocRow( + quarter=quarter, + drug_name=name, + dosage=r.get("dosage_or_unit", ""), + payment_limit=_number(r.get("payment_limit")), + source_file=member, + source_url=source_url, + item_key=item_key, + ) + ) + return asp, noc diff --git a/src/drugs/codes.py b/src/drugs/codes.py index 042f028..1aaa0b9 100644 --- a/src/drugs/codes.py +++ b/src/drugs/codes.py @@ -5,18 +5,18 @@ from __future__ import annotations import re from datetime import date, timedelta -_L1 = re.compile(r"^\d{4}[0-9A-Z]$") -_L2 = re.compile(r"^[ABCEGHJKLMPQRSTUV]\d{4}$") -_CDT = re.compile(r"^D\d{4}$") +_L1 = re.compile(r"[0-9]{4}[0-9A-Z]") +_L2 = re.compile(r"[ABCEGHJKLMPQRSTUV][0-9]{4}") +_CDT = re.compile(r"D[0-9]{4}") def level(code: str) -> str: """``"I"`` (CPT), ``"II"`` (HCPCS Level II), ``"CDT"`` or ``""``.""" - if _L1.match(code): + if _L1.fullmatch(code): return "I" - if _L2.match(code): + if _L2.fullmatch(code): return "II" - if _CDT.match(code): + if _CDT.fullmatch(code): return "CDT" return "" @@ -30,9 +30,9 @@ def ndc11(raw: str) -> str | None: """The 11-digit NDC of a 4-4-2, 5-3-2, 5-4-1 or 5-4-2 hyphenated code, or of 11 bare digits; anything else is ``None``.""" s = raw.strip() - if re.fullmatch(r"\d{11}", s): + if re.fullmatch(r"[0-9]{11}", s): return s - m = re.fullmatch(r"(\d+)-(\d+)-(\d+)", s) + m = re.fullmatch(r"([0-9]+)-([0-9]+)-([0-9]+)", s) if not m: return None a, b, c = m.groups() @@ -48,27 +48,36 @@ def ndc11(raw: str) -> str | None: return None -_APPL = {"N": "NDA", "A": "ANDA", "BLA": "BLA"} +_APPL = {"N": "NDA", "NDA": "NDA", "A": "ANDA", "ANDA": "ANDA", "BLA": "BLA"} def appl_id(kind: str, number: str) -> str: """``("N","21436")`` -> ``NDA021436``; an id already spelled out passes.""" - if re.fullmatch(r"[A-Z]+\d{6}", kind): + if re.fullmatch(r"[A-Z]+[0-9]{6}", kind): return kind - return f"{_APPL.get(kind, kind)}{int(number):06d}" + if kind not in _APPL: + raise ValueError(f"unknown application kind {kind!r}") + return f"{_APPL[kind]}{int(number):06d}" + + +#: the longest range :func:`expand` will list; a wider one is a data error +MAX_RANGE = 500 def expand(lo: str, hi: str) -> list[str]: """Every code from *lo* to *hi* when they share a letter prefix and a - numeric tail; otherwise just *lo*.""" - a = re.fullmatch(r"([A-Z]+)(\d+)", lo) - b = re.fullmatch(r"([A-Z]+)(\d+)", hi) + numeric tail; otherwise just *lo*. A range wider than :data:`MAX_RANGE` + codes raises ``ValueError``.""" + a = re.fullmatch(r"([A-Z]+)([0-9]+)", lo) + b = re.fullmatch(r"([A-Z]+)([0-9]+)", hi) if not a or not b or a.group(1) != b.group(1): return [lo] width = len(a.group(2)) start, end = int(a.group(2)), int(b.group(2)) if end < start or len(b.group(2)) != width: return [lo] + if end - start + 1 > MAX_RANGE: + raise ValueError(f"range too large: {lo}-{hi} is {end - start + 1} codes") return [f"{a.group(1)}{n:0{width}d}" for n in range(start, end + 1)] diff --git a/src/drugs/tables.py b/src/drugs/tables.py index 07fa6b6..051c978 100644 --- a/src/drugs/tables.py +++ b/src/drugs/tables.py @@ -7,11 +7,35 @@ tables. Nothing opens a database at import. from __future__ import annotations from collections.abc import Iterable, Sequence -from dataclasses import astuple, fields +from dataclasses import astuple, dataclass, fields from typing import Any _DDL = """ -CREATE SCHEMA IF NOT EXISTS drugs +CREATE SCHEMA IF NOT EXISTS drugs; +CREATE TABLE IF NOT EXISTS drugs.asp_payment ( + quarter VARCHAR, + hcpcs VARCHAR, + level VARCHAR, + list_kind VARCHAR, + dosage VARCHAR, + payment_limit DOUBLE, + coinsurance_pct DOUBLE, + short_desc VARCHAR, + notes VARCHAR, + source_file VARCHAR, + source_url VARCHAR, + item_key VARCHAR, + manufacturer VARCHAR +); +CREATE TABLE IF NOT EXISTS drugs.asp_noc ( + quarter VARCHAR, + drug_name VARCHAR, + dosage VARCHAR, + payment_limit DOUBLE, + source_file VARCHAR, + source_url VARCHAR, + item_key VARCHAR +) """ @@ -61,3 +85,79 @@ def _select(con: Any, cls: type, table: str, where: str = "", params: Sequence = def _chunks(items: Sequence[Any], n: int = 500) -> Iterable[Sequence[Any]]: for i in range(0, len(items), n): yield items[i : i + n] + + +# ── asp_payment / asp_noc ──────────────────────────────────────────── +@dataclass(frozen=True) +class AspRow: + """One line of a quarterly Part B payment-limit file. ``list_kind`` is + ``payment`` or ``not-payable`` (no Medicaid rebate agreement; carries the + file's manufacturer). ``short_desc`` is ``""`` for a licensed (CPT/CDT) + code. ``payment_limit`` is the CMS payment allowance (ASP + 6%), never + the ASP itself.""" + + quarter: str + hcpcs: str + level: str + list_kind: str + dosage: str + payment_limit: float | None + coinsurance_pct: float | None + short_desc: str + notes: str + source_file: str + source_url: str + item_key: str + manufacturer: str = "" + + +@dataclass(frozen=True) +class NocRow: + """One line of a quarterly not-otherwise-classified drug pricing file.""" + + quarter: str + drug_name: str + dosage: str + payment_limit: float | None + source_file: str + source_url: str + item_key: str + + +def write_asp(con: Any, quarter: str, rows: Sequence[AspRow]) -> int: + """Replace *quarter* of ``drugs.asp_payment`` with *rows*.""" + ensure_tables(con) + con.execute("DELETE FROM drugs.asp_payment WHERE quarter = ?", [quarter]) + return _insert(con, "drugs.asp_payment", rows) + + +def read_asp( + con: Any, *, hcpcs: str | None = None, quarter: str | None = None +) -> list[AspRow]: + where, params = [], [] + if hcpcs is not None: + where.append("hcpcs = ?") + params.append(hcpcs) + if quarter is not None: + where.append("quarter = ?") + params.append(quarter) + rows = _select(con, AspRow, "drugs.asp_payment", " AND ".join(where), params) + return sorted( + (AspRow(*r) for r in rows), + key=lambda r: (r.quarter, r.hcpcs, r.list_kind, r.dosage), + ) + + +def write_noc(con: Any, quarter: str, rows: Sequence[NocRow]) -> int: + """Replace *quarter* of ``drugs.asp_noc`` with *rows*.""" + ensure_tables(con) + con.execute("DELETE FROM drugs.asp_noc WHERE quarter = ?", [quarter]) + return _insert(con, "drugs.asp_noc", rows) + + +def read_noc(con: Any, quarter: str | None = None) -> list[NocRow]: + where, params = ("quarter = ?", [quarter]) if quarter is not None else ("", []) + rows = _select(con, NocRow, "drugs.asp_noc", where, params) + return sorted( + (NocRow(*r) for r in rows), key=lambda r: (r.quarter, r.drug_name, r.dosage) + ) diff --git a/tests/cli/test_drugs_cli.py b/tests/cli/test_drugs_cli.py index 4e95e35..f78ac78 100644 --- a/tests/cli/test_drugs_cli.py +++ b/tests/cli/test_drugs_cli.py @@ -2,12 +2,20 @@ from __future__ import annotations +import io +import json +import zipfile + import duckdb +import httpx import pytest from typer.testing import CliRunner import cli.drugs as dg +from bib.store import Store from cli import app +from drugs import asp as A +from drugs import http as dh from drugs import tables as T from market import tables as MT @@ -140,3 +148,376 @@ def test_status_gap_read_error_propagates(env, monkeypatch): monkeypatch.setattr(dg, "_read", lambda: _Con()) assert isinstance(runner.invoke(app, ["drugs", "status"]).exception, RuntimeError) + + +# ── stack drugs asp ────────────────────────────────────────────────── +PRICING = ( + "Payment Allowance Limits,,,,,,,,,,,\r\n" + "HCPCS Code,Short Description,HCPCS Code Dosage,Payment Limit," + "Co-insurance Percentage,Vaccine AWP%,Vaccine Limit,Blood AWP%,Blood limit," + "Clotting Factor,Notes\r\n" + "90371,x,1 ML,140.209,20.000,,,,,,\r\n" + "J9035,Bevacizumab injection,10 MG,66.123,20.000,,,,,,\r\n" +) +NP = ( + "HCPCS Code,Long Description,Manufacturer\r\nJ7330,Chondrocytes implant,Vericel\r\n" +) +NOC = "Drug Generic Name (Trade Name),Dosage,Payment Limit,Notes\r\nExamplumab,10 MG,12.5,\r\n" + + +def _zip(members): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for n, b in members.items(): + zf.writestr(n, b) + return buf.getvalue() + + +PAGE = """ +July 2026 Medicare Part B Payment Limit Files +July 2026 NDC-HCPCS Crosswalk +April 2018 ASP Pricing File +April 2018 NOC Pricing File +2026-2027 Seasonal Vaccine Price File +""" + +ROUTES = { + "/medicare/payment/part-b-drugs/asp-pricing-files": PAGE.encode(), + "/files/zip/july-2026-medicare-part-b-payment-limit-files.zip": _zip( + { + "section 508 version of July 2026 Payment Limit File.csv": PRICING, + "section 508 version of July 2026 Drugs Not Payable.csv": NP, + } + ), + "/files/zip/july-2026-ndc-hcpcs-crosswalk.zip": _zip({"x.csv": "a"}), + "/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip": _zip({"NOC.csv": NOC}), + "/Medicare/x/downloads/2018-April-ASP-Pricing-File.zip": _zip({"ASP.csv": PRICING}), +} + + +class _NoWait: + def wait(self): + pass + + +@pytest.fixture +def asp_env(env, monkeypatch, tmp_path): + c, published = env + monkeypatch.setattr(dh, "data_root", lambda: tmp_path) + monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait()) + dh._MANIFESTS.clear() + store = Store(":memory:", storage_dir=tmp_path / "st") + monkeypatch.setattr(dg, "_store", lambda: store) + seen: list[httpx.Request] = [] + + def handler(req): + seen.append(req) + body = ROUTES.get(req.url.path) + return ( + httpx.Response(200, content=body) + if body is not None + else httpx.Response(404) + ) + + monkeypatch.setattr( + dh, + "client", + lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)), + ) + monkeypatch.setattr(A, "today", lambda: __import__("datetime").date(2026, 8, 1)) + return c, published, seen, tmp_path, store + + +def test_asp_cli_fetch_write(asp_env): + c, published, seen, root, store = asp_env + MT.write_gap(c, "cms-asp:asp", "stale") + res = runner.invoke( + app, + [ + "drugs", + "asp", + "--fetch", + "--write", + "--quarter", + "2026-Q3", + "--quarter", + "2018-Q2", + ], + ) + assert res.exit_code == 0, res.output + pay = [r for r in T.read_asp(c) if r.quarter == "2026-Q3"] + assert {r.quarter for r in T.read_asp(c)} == {"2018-Q2", "2026-Q3"} + assert [(r.quarter, r.hcpcs, r.list_kind) for r in pay] == [ + ("2026-Q3", "90371", "payment"), + ("2026-Q3", "J7330", "not-payable"), + ("2026-Q3", "J9035", "payment"), + ] + assert pay[0].short_desc == "" and pay[2].short_desc == "Bevacizumab injection" + assert pay[1].manufacturer == "Vericel" + assert all( + r.item_key + and r.source_url.startswith("https://www.cms.gov/files/zip/july-2026") + for r in pay + ) + noc = T.read_noc(c) + assert [(n.quarter, n.drug_name, n.payment_limit) for n in noc] == [ + ("2018-Q2", "Examplumab", 12.5) + ] + assert ( + noc[0].source_url + == "https://www.cms.gov/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip" + ) + assert { + k + for (k,) in c.execute( + "SELECT DISTINCT table_name FROM cms.ingest_log" + ).fetchall() + } == { + "drugs.asp_payment", + "drugs.asp_noc", + } + assert published == [True] + assert [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"] == [] + assert "stale" not in " ".join(g.note for g in MT.read_gaps(c)) + # every request carried the honest user agent through the real client factory + hoard = root / "cms" / "asp" + assert (hoard / "asp_2026-Q3.zip").exists() + assert (hoard / "noc" / "noc_2018-Q2.zip").exists() + assert (hoard / "xwalk" / "xwalk_2026-Q3.zip").exists() + assert (root / "drugs" / "cms" / "asp-pricing-files.html").exists() + manifest = [ + json.loads(x) for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines() + ] + assert len(manifest) == 5 + # vaccine zip is never fetched + assert not any("vaccine" in str(r.url) for r in seen) + assert "2026-Q3" in res.output and "not-payable" in res.output + cites = [i for i in store.list_items() if "ASP" in i.title] + assert len(cites) == 3 + # a second run fetches nothing but the page + n = len(seen) + again = runner.invoke( + app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3", "--quarter", "2018-Q2"] + ) + assert again.exit_code == 0, again.output + assert len(seen) == n + 1 + + +def test_asp_cli_gap_note_written_for_missing(asp_env): + c, published, seen, root, store = asp_env + res = runner.invoke( + app, + [ + "drugs", + "asp", + "--fetch", + "--write", + "--quarter", + "2026-Q3", + "--quarter", + "2026-Q2", + ], + ) + assert res.exit_code == 0, res.output + gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"] + assert len(gaps) == 1 and "2026-Q2 pricing" in gaps[0].note + assert "gap: cms-asp:asp: no hoarded file for: 2026-Q2 pricing" in res.output + + +def test_asp_cli_dry_run_lists_links(asp_env): + c, published, seen, root, store = asp_env + first = runner.invoke(app, ["drugs", "asp", "--fetch", "--dry-run"]) + assert first.exit_code == 0, first.output + assert [r.url.path for r in seen] == [ + "/medicare/payment/part-b-drugs/asp-pricing-files" + ] + out = first.output + assert ( + "pricing 2026-Q3 fetch https://www.cms.gov/files/zip/july-2026-medicare-part-b-payment-limit-files.zip" + in out + ) + assert "noc 2018-Q2 fetch" in out and "crosswalk 2026-Q3 fetch" in out + assert "vaccine" not in out + assert not (root / "cms").exists() + assert T.table_counts(c) == {} + assert published == [] + # --write is ignored by a dry run + res = runner.invoke( + app, ["drugs", "asp", "--dry-run", "--write", "--quarter", "2018-Q2"] + ) + assert res.exit_code == 0 and published == [] and "noc 2018-Q2 fetch" in res.output + assert "crosswalk 2026-Q3" not in res.output + + +def test_asp_cli_refusal_prints_gap(asp_env, monkeypatch): + c, published, seen, root, store = asp_env + monkeypatch.setattr( + dh, + "client", + lambda source, key=None: httpx.Client( + transport=httpx.MockTransport(lambda r: httpx.Response(403)) + ), + ) + res = runner.invoke(app, ["drugs", "asp", "--fetch"]) + assert res.exit_code == 0, res.output + lines = [x for x in res.output.splitlines() if x.strip()] + assert len(lines) == 1 and lines[0].startswith("gap: cms-asp:asp:") + assert published == [] + assert T.table_counts(c) == {} and not (root / "cms").exists() + res = runner.invoke(app, ["drugs", "asp", "--fetch", "--write"]) + assert [g.source for g in MT.read_gaps(c)] == ["cms-asp:asp"] + assert published == [True] + assert T.table_counts(c) == {} + + +def test_asp_cli_refusal_on_a_zip_stops_fetching(asp_env, monkeypatch): + c, published, seen, root, store = asp_env + + def handler(req): + if req.url.path.endswith("asp-pricing-files"): + return httpx.Response(200, content=PAGE.encode()) + return httpx.Response(403) + + monkeypatch.setattr( + dh, + "client", + lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)), + ) + res = runner.invoke( + app, ["drugs", "asp", "--fetch", "--write", "--quarter", "2026-Q3"] + ) + assert res.exit_code == 0, res.output + assert "gap: cms-asp:asp: refused" in res.output + assert "2026-Q3 pricing" in res.output + assert T.read_asp(c) == [] + + +def test_asp_cli_parse_only_reads_hoard_without_cites(asp_env): + c, published, seen, root, store = asp_env + assert ( + runner.invoke( + app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3"] + ).exit_code + == 0 + ) + assert T.table_counts(c) == {} + seen.clear() + res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q3"]) + assert res.exit_code == 0 and seen == [] + assert "2026-Q3" in res.output and "3 payment" not in res.output + assert "2 payment" in res.output and "1 not-payable" in res.output + assert store.list_items() == [] and published == [] + + +def test_asp_cli_unreadable_zip_is_a_gap(asp_env): + c, published, seen, root, store = asp_env + d = root / "cms" / "asp" + d.mkdir(parents=True) + (d / "asp_2026-Q3.zip").write_bytes(b"not a zip") + res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"]) + assert res.exit_code == 0, res.output + gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"] + assert gaps and "2026-Q3 pricing unreadable" in gaps[0].note + assert T.read_asp(c) == [] + + +def test_asp_cli_source_url_falls_back_to_csv(asp_env): + c, published, seen, root, store = asp_env + d = root / "cms" / "asp" + d.mkdir(parents=True) + (d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING})) + (root / "cms" / "asp_source_urls.csv").write_text( + "quarter,url\n2026-Q3,https://cms.test/canon.zip\n" + ) + res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"]) + assert res.exit_code == 0, res.output + assert {r.source_url for r in T.read_asp(c)} == {"https://cms.test/canon.zip"} + assert all(r.item_key for r in T.read_asp(c)) + + +def test_asp_cli_no_url_no_cite(asp_env): + c, published, seen, root, store = asp_env + d = root / "cms" / "asp" + d.mkdir(parents=True) + (d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING})) + res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"]) + assert res.exit_code == 0, res.output + assert {(r.source_url, r.item_key) for r in T.read_asp(c)} == {("", "")} + + +def test_asp_cli_refresh_refetches(asp_env): + c, published, seen, root, store = asp_env + assert ( + runner.invoke( + app, ["drugs", "asp", "--fetch", "--quarter", "2018-Q2"] + ).exit_code + == 0 + ) + n = len(seen) + ROUTES_BEFORE = dict(ROUTES) + res = runner.invoke( + app, ["drugs", "asp", "--fetch", "--refresh", "--quarter", "2018-Q2"] + ) + assert ( + res.exit_code == 0 and len(seen) == n + 3 + ) # page, the pricing zip and the NOC zip + assert ROUTES == ROUTES_BEFORE + + +def test_asp_cli_unfiltered_uses_expected_quarters(asp_env): + c, published, seen, root, store = asp_env + res = runner.invoke(app, ["drugs", "asp"]) + assert res.exit_code == 0, res.output + assert "gap: cms-asp:asp: no hoarded file for: 2005-Q1 pricing" in res.output + assert "(+" in res.output and published == [] + assert seen == [] + + +def test_asp_today_is_real_date(): + import datetime + + assert A.today() == datetime.date.today() + + +def test_asp_cli_rejects_bad_quarter(asp_env): + res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q5"]) + assert res.exit_code != 0 and "not a quarter" in res.output + + +def test_asp_cli_unreadable_noc_zip_is_a_gap(asp_env): + c, published, seen, root, store = asp_env + d = root / "cms" / "asp" / "noc" + d.mkdir(parents=True) + (d / "noc_2018-Q2.zip").write_bytes(b"not a zip") + res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2018-Q2"]) + assert res.exit_code == 0, res.output + assert "2018-Q2 noc unreadable" in res.output + assert T.read_noc(c) == [] + + +def test_asp_cli_bundled_noc_from_pricing_zip(asp_env): + c, published, seen, root, store = asp_env + d = root / "cms" / "asp" + d.mkdir(parents=True) + (d / "asp_2006-Q2.zip").write_bytes( + _zip({"ASPbyHCPCS.csv": PRICING, "ASPNOC.csv": NOC}) + ) + res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2006-Q2"]) + assert res.exit_code == 0, res.output + assert [n.drug_name for n in T.read_noc(c)] == ["Examplumab"] + assert "1 noc" in res.output + + +def test_asp_cli_dry_run_without_page(asp_env): + res = runner.invoke(app, ["drugs", "asp", "--dry-run"]) + assert res.exit_code == 0 and "no page hoarded; run with --fetch" in res.output + + +def test_asp_cli_noc_only_quarter_is_listed(asp_env): + c, published, seen, root, store = asp_env + d = root / "cms" / "asp" / "noc" + d.mkdir(parents=True) + (d / "noc_2018-Q2.zip").write_bytes(_zip({"NOC.csv": NOC})) + res = runner.invoke(app, ["drugs", "asp", "--quarter", "2018-Q2"]) + assert res.exit_code == 0, res.output + assert "2018-Q2: 1 noc" in res.output diff --git a/tests/dev/test_ingest_asp_parity.py b/tests/dev/test_ingest_asp_parity.py new file mode 100644 index 0000000..33c72f1 --- /dev/null +++ b/tests/dev/test_ingest_asp_parity.py @@ -0,0 +1,134 @@ +"""The skin-substitute ingest reads the same ASP parser as the drug tier, and +its output is unchanged (1,974 rows, 2009-Q1..2025-Q4).""" + +from __future__ import annotations + +import csv +import importlib.util +import math +import zipfile +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +ASP_DIR = ROOT / "data" / "cms" / "asp" +SKIN_CSV = ASP_DIR / "skin_subs_asp_quarterly.csv" +REF_CSV = ROOT / "data" / "cms" / "skin_substitute_hcpcs_reference.csv" + +_spec = importlib.util.spec_from_file_location( + "_ingest_asp", ROOT / "dev" / "scripts" / "ingest_asp.py" +) +ingest = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(ingest) + +needs_data = pytest.mark.skipif( + not (SKIN_CSV.exists() and any(ASP_DIR.glob("asp_*.zip"))), + reason="needs the hoarded data/cms/asp files", +) + + +def _legacy_pick(names: list[str]) -> str | None: + """The selection the dev script made before the shared parser (frozen).""" + skip = ["noc", "crosswalk", "ndc", "not payable", "not_payable"] + cand = [ + n + for n in names + if not any(s in n.lower() for s in skip) + and n.lower().endswith((".csv", ".xls", ".xlsx")) + ] + csvs = [c for c in cand if c.lower().endswith(".csv")] + if csvs: + for c in csvs: + if "payment limit" in c.lower() or "pricing" in c.lower(): + return c + return csvs[0] + xlss = [c for c in cand if c.lower().endswith((".xls", ".xlsx"))] + if xlss: + for c in xlss: + low = c.lower() + if "payment limit" in low or "pricing" in low or "asp" in low: + return c + return xlss[0] + return None + + +@needs_data +def test_pick_pricing_file_over_every_hoarded_zip(): + from drugs import asp + + n = 0 + for z in sorted(ASP_DIR.glob("asp_*.zip")): + names = zipfile.ZipFile(z).namelist() + want = _legacy_pick(names) + assert ingest.pick_pricing_file(names) == want, z.name + assert asp.pick_pricing_file(names) == want, z.name + n += 1 + assert n >= 86 + + +@needs_data +def test_skin_rows_match_hoarded_csv(): + df = ingest.build_skin_df() + got = sorted( + df.to_dict("records"), + key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]), + ) + with SKIN_CSV.open(newline="") as fh: + want = sorted( + csv.DictReader(fh), + key=lambda r: (r["quarter"], r["hcpcs_code"], r["source_file"]), + ) + assert len(want) == 1974 + assert len(got) == len(want), "an extra or missing skin row appeared" + for g, w in zip(got, want, strict=True): + for col in ( + "quarter", + "hcpcs_code", + "short_description", + "dosage_or_unit", + "source_file", + "source_url", + ): + gv = g[col] + gv = "" if gv is None or (isinstance(gv, float) and math.isnan(gv)) else gv + assert gv == w[col], (g["quarter"], g["hcpcs_code"], col) + gp = g["payment_limit"] + if w["payment_limit"] == "": + assert gp is None or math.isnan(gp) + else: + assert abs(gp - float(w["payment_limit"])) < 1e-9 + assert abs(g["asp_per_unit"] - float(w["asp_per_unit"])) < 1e-9 + + +def test_reference_csv_carries_no_cpt_descriptor_text(): + with REF_CSV.open(newline="") as fh: + rows = list(csv.DictReader(fh)) + assert len(rows) == 34 + assert list(rows[0]) == [ + "hcpcs_code", + "short_descriptor", + "code_type", + "body_site", + "wound_size", + "effective_date", + "termination_date", + "status", + "notes", + ] + cpt = [r for r in rows if "CPT" in r["code_type"]] + assert {r["hcpcs_code"] for r in cpt} >= {"15271", "15278", "15330", "15430"} + assert all(r["short_descriptor"] == "" for r in cpt) + # CMS's own HCPCS Level II text stays + level2 = {r["hcpcs_code"]: r["short_descriptor"] for r in rows if r not in cpt} + assert level2["C5271"].startswith("Low cost skin substitute graft") + assert level2["G0440"] and level2["C9363"] + assert all(r["short_descriptor"] for r in rows if r not in cpt) + + +def test_refuses_to_replace_table_with_nothing(): + import pandas as pd + + with pytest.raises(RuntimeError, match="no skin-substitute rows"): + ingest.require_rows(pd.DataFrame()) + ingest.require_rows(pd.DataFrame({"a": [1]})) diff --git a/tests/drugs/fixtures/asp_page_head.html b/tests/drugs/fixtures/asp_page_head.html new file mode 100644 index 0000000..62908e2 --- /dev/null +++ b/tests/drugs/fixtures/asp_page_head.html @@ -0,0 +1,17 @@ + +ASP Pricing Files + +

+
+ +
+ diff --git a/tests/drugs/test_asp.py b/tests/drugs/test_asp.py new file mode 100644 index 0000000..115900a --- /dev/null +++ b/tests/drugs/test_asp.py @@ -0,0 +1,694 @@ +import json +import os +import zipfile +from datetime import date +from pathlib import Path + +import duckdb +import httpx +import pytest + +from drugs import asp +from drugs import http as dh +from drugs import tables as T + +FIXTURE = Path(__file__).parent / "fixtures" / "asp_page_head.html" + + +class _NoWait: + def wait(self): + pass + + +@pytest.fixture +def root(monkeypatch, tmp_path): + monkeypatch.setattr(dh, "data_root", lambda: tmp_path) + monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait()) + dh._MANIFESTS.clear() + return tmp_path + + +# ── the page ───────────────────────────────────────────────────────── +def test_parse_asp_page_kinds_and_quarters(): + links = asp.parse_asp_page(FIXTURE.read_text()) + got = {(lk.quarter, lk.kind): lk.url for lk in links} + assert got[("2026-Q4", "pricing")] == ( + "https://www.cms.gov/files/zip/" + "october-2026-medicare-part-b-payment-limit-files-final.zip" + ) + assert got[("2026-Q4", "crosswalk")] == ( + "https://www.cms.gov/files/zip/october-2026-ndc-hcpcs-crosswalk-final.zip" + ) + vaccine = [lk for lk in links if lk.kind == "vaccine"] + assert len(vaccine) == 2 and {lk.quarter for lk in vaccine} == {""} + assert ("2018-Q2", "noc") in got and ("2018-Q1", "pricing") in got + assert got[("2005-Q1", "pricing")] == ( + "https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/" + "McrPartBDrugAvgSalesPrice/downloads/jan05aspbyhcpcsv6_033106.zip" + ) + assert ("2005-Q1", "crosswalk") in got + assert len(links) == 8 + assert links[0].text == "October 2026 Medicare Part B Payment Limit Files" + + +def test_parse_asp_page_edge_cases(): + html = """ + 2023 ASP Pricing File (ZIP) + 2023 NOC Pricing File + ASP Pricing File + January 2006 ASP Pricing File/ASP NOC Pricing File/ASP NDC-HCPCS Crosswalk + July 2006 ASP NDC-HCPCS Crosswalk - Updated + October 2009 ASP NOC Pricing File + Something else entirely + ASP Pricing File + Guide to the files (PDF) + no href at all + Oct. 2005 ASP Pricing File and ASP NOC Pricing File + """ + links = asp.parse_asp_page(html) + got = [(lk.quarter, lk.kind) for lk in links] + assert got == [ + ("2023-Q1", "pricing"), + ("2023-Q2", "noc"), + ("2018-Q3", "pricing"), + ("2006-Q1", "pricing"), + ("2006-Q3", "crosswalk"), + ("2009-Q4", "noc"), + ("", "other"), + ("", "other"), + ("2005-Q4", "pricing"), + ] + assert links[1].text == "2023 NOC Pricing File" + + +def test_direct_url_unwraps_license(): + p = "/Medicare/Medicare-Fee-for-Service-Part-B-Drugs/x/downloads/a.zip" + want = "https://www.cms.gov" + p + assert asp.direct_url(f"https://www.cms.gov/apps/ama/license.asp?file={p}") == want + assert asp.direct_url(f"/apps/ama/license.asp?file={p}") == want + assert asp.direct_url(f"https://www.cms.gov/license/ama?file={p}") == want + assert asp.direct_url("/files/zip/a.zip") == "https://www.cms.gov/files/zip/a.zip" + assert asp.direct_url("https://x.test/a.zip") == "https://x.test/a.zip" + + +def test_hoard_path_and_newest_per_quarter(root): + def lk(kind): + return asp.AspLink("2026-Q2", kind, "u", "t") + + assert asp.hoard_path(lk("pricing")) == root / "cms/asp/asp_2026-Q2.zip" + assert asp.hoard_path(lk("noc")) == root / "cms/asp/noc/noc_2026-Q2.zip" + assert asp.hoard_path(lk("crosswalk")) == root / "cms/asp/xwalk/xwalk_2026-Q2.zip" + with pytest.raises(ValueError): + asp.hoard_path(lk("vaccine")) + + +def test_one_zip_per_quarter_newest_wins(tmp_path): + a = tmp_path / "asp_2026-Q2.zip" + b = tmp_path / "asp_2026-Q2.ab12cd34.zip" + c = tmp_path / "asp_2026-Q1.zip" + junk = tmp_path / "asp_notes.zip" + for p in (a, b, c, junk): + p.write_bytes(b"x") + os.utime(a, (1_000, 1_000)) + os.utime(b, (2_000, 2_000)) + got = asp.newest_per_quarter([a, b, c, junk]) + assert got == {"2026-Q1": c, "2026-Q2": b} + # the older copy still wins when it is the newer file + os.utime(a, (3_000, 3_000)) + assert asp.newest_per_quarter([a, b])["2026-Q2"] == a + assert asp.newest_per_quarter([]) == {} + + +def test_expected_quarters(): + q = asp.expected_quarters(date(2026, 11, 13)) + assert q[0] == "2005-Q1" and q[-1] == "2026-Q4" and len(q) == 88 + assert asp.expected_quarters(date(2005, 1, 1)) == ["2005-Q1"] + assert asp.expected_quarters(date(2005, 3, 31)) == ["2005-Q1"] + assert asp.expected_quarters(date(2006, 2, 1)) == [ + "2005-Q1", + "2005-Q2", + "2005-Q3", + "2005-Q4", + "2006-Q1", + ] + + +# ── the member logic ───────────────────────────────────────────────── +def test_col_map_covers_dev_script_and_new_columns(): + dev = { + "hcpcs code": "hcpcs_code", + "hcpcs_code": "hcpcs_code", + "short description": "short_description", + "short_description": "short_description", + "hcpcs code dosage": "dosage_or_unit", + "hcpcs_code_dosage": "dosage_or_unit", + "dosage": "dosage_or_unit", + "dosage or unit": "dosage_or_unit", + "payment limit": "payment_limit", + "payment_limit": "payment_limit", + } + for k, v in dev.items(): + assert asp.COL_MAP[k] == v + assert asp.COL_MAP["co-insurance percentage"] == "coinsurance_pct" + assert asp.COL_MAP["notes"] == "notes" + assert asp.COL_MAP["long description"] == "short_description" + assert asp.COL_MAP["manufacturer"] == "manufacturer" + assert asp.COL_MAP["drug generic name (trade name)"] == "drug_name" + + +def test_find_header_row(): + assert asp.find_header_row(["a", "Note: HCPCS code x,y", "HCPCS Code,Foo"]) == 2 + assert asp.find_header_row(['"HCPCS Code","Foo"']) == 0 + assert asp.find_header_row(["", "Drug Generic Name (Trade Name),Dosage"]) == 1 + assert asp.find_header_row(["Drug Name (Generic),Dosage"]) == 0 + assert asp.find_header_row(["nothing here"]) == -1 + assert asp.find_header_row([]) == -1 + + +NAMES_2015Q3 = [ + "Jul 15 ASP Pricing File - updated 090115.xls", + "section 508 version of Jul 15 ASP Pricing File - updated 090115.csv", +] +NAMES_2026Q2 = [ + "April 2026 Drugs Not Payable Under Part B 030926.xls", + "April 2026 Medicare Part B Payment Limit File 032426.xls", + "section 508 version of April 2026 Drugs Not Payable Under Part B 030926.csv", + "section 508 version of April 2026 Medicare Part B Payment Limit File 032426.csv", +] + + +def test_pick_pricing_file_matches_dev_script(): + assert asp.pick_pricing_file(["Jan05ASPbyHCPCSv6_033106.xls"]) == ( + "Jan05ASPbyHCPCSv6_033106.xls" + ) + assert asp.pick_pricing_file(NAMES_2015Q3) == NAMES_2015Q3[1] + assert asp.pick_pricing_file( + [ + "January 2026 Medicare Part B Payment Limit File updated 022526.xls", + "section 508 version of January 2026 Medicare Part B Payment Limit " + "File updated 022526.csv", + ] + ).endswith("022526.csv") + assert asp.pick_pricing_file(NAMES_2026Q2) == NAMES_2026Q2[3] + assert asp.pick_pricing_file(["a.xls", "b.pdf"]) == "a.xls" + assert asp.pick_pricing_file(["ASP.xlsx", "other.xls"]) == "ASP.xlsx" + assert ( + asp.pick_pricing_file(["other.csv", "payment limit.csv"]) == "payment limit.csv" + ) + assert asp.pick_pricing_file(["x.csv"]) == "x.csv" + assert ( + asp.pick_pricing_file(["Oct 2025 NOC Pricing File.csv", "readme.txt"]) is None + ) + assert asp.pick_pricing_file(["Crosswalk.csv", "ndc.xls"]) is None + + +def test_pick_member_prefers_csv_over_its_xls_twin(): + # P7: one member per kind per zip + assert asp.pick_member(NAMES_2026Q2, "not-payable") == NAMES_2026Q2[2] + assert asp.pick_member(NAMES_2026Q2, "noc") is None + names = [ + "1 - Apr06_ASPbyHCPCS_122107.xls", + "1 - Apr06_ASPbyHCPCS_122107.csv", + "1 - Apr06_ASPNOC_20Sep06.xls", + "1 - Apr06_ASPNOC_20Sep06.csv", + "April06ASPNDC-HCPCSCrosswalk_20Mar06.xls", + ] + assert asp.pick_member(names, "noc") == "1 - Apr06_ASPNOC_20Sep06.csv" + assert asp.pick_member(names[2:3] + names[4:], "noc") == names[2] + assert asp.pick_member(["Drugs_Not_Payable.xlsx"], "not-payable") == ( + "Drugs_Not_Payable.xlsx" + ) + assert asp.pick_member(["x.csv"], "noc") is None + with pytest.raises(ValueError): + asp.pick_member(names, "crosswalk") + + +# ── reading tables ─────────────────────────────────────────────────── +PRICING_HEAD = ( + "Payment Allowance Limits for Medicare Part B Drugs,,,,,,,,,,,\r\n" + ",,,,,,,,,,,\r\n" + "Note 1: HCPCS code lines mention HCPCS code here,,,,,,,,,,,\r\n" + "HCPCS Code,Short Description,HCPCS Code Dosage,Payment Limit," + "Co-insurance Percentage,Vaccine AWP%,Vaccine Limit,Blood AWP%,Blood limit," + "Clotting Factor,Notes\r\n" +) +PRICING_ROWS = ( + "90371,x,1 ML,140.209,20.000,,,,,,\r\n" + 'J9035,Bevacizumab injection,10 MG,"$66.123",20.000,,,,,,\r\n' +) +NP_CSV = ( + "Drugs Not Payable Under Part B,,,\r\n" + ",,,\r\n" + "HCPCS Code,Long Description,Manufacturer,,,,\r\n" + 'J7330," Autologous cultured chondrocytes, implant ",Vericel Corporation,,,,\r\n' + "90999,x,Acme,,,,\r\n" + ",,,,,,\r\n" +) +NOC_CSV = ( + "Payment Allowance Limits for NOC Drugs,,,\r\n" + ",,,\r\n" + "Drug Generic Name (Trade Name),Dosage,Payment Limit,Notes\r\n" + "Examplumab,10 MG,12.5,\r\n" + "Vasopressin (Long Grove),1 UNIT,$2.462,\r\n" + ",,,\r\n" +) + + +def make_zip(path: Path, members: dict[str, str | bytes]) -> Path: + with zipfile.ZipFile(path, "w") as zf: + for name, body in members.items(): + zf.writestr( + name, body if isinstance(body, bytes) else body.encode("latin-1") + ) + return path + + +def test_cpt_short_description_never_stored(tmp_path): + z = make_zip( + tmp_path / "asp_2026-Q2.zip", + { + "section 508 version of April 2026 Payment Limit File.csv": PRICING_HEAD + + PRICING_ROWS + }, + ) + rows, noc = asp.parse_quarter_zip( + z, "2026-Q2", source_url="https://cms.test/a.zip", item_key="K1" + ) + assert noc == [] + by = {r.hcpcs: r for r in rows} + assert by["90371"].level == "I" and by["90371"].short_desc == "" + assert by["90371"].dosage == "1 ML" and by["90371"].payment_limit == 140.209 + assert by["90371"].coinsurance_pct == 20.0 + j = by["J9035"] + assert j.level == "II" and j.short_desc == "Bevacizumab injection" + assert j.payment_limit == 66.123 and j.list_kind == "payment" + assert j.quarter == "2026-Q2" and j.item_key == "K1" + assert j.source_url == "https://cms.test/a.zip" + assert j.source_file == "section 508 version of April 2026 Payment Limit File.csv" + assert "Bevacizumab" not in " ".join(r.short_desc for r in rows if r.level == "I") + + +def test_numbers_codes_and_blank_rows(tmp_path): + body = ( + PRICING_HEAD + + ' j9035 ,Bevacizumab,10 MG,"$1,066.50",,,,,,,see note\r\n' + + "J0000,Unparseable,1 MG,n/a,abc,,,,,,\r\n" + + "J0001,Short row\r\n" + + ",,,,,,,,,,\r\n" + + " ,blank code,,,,,,,,,\r\n" + + "J0002,Too many,1 MG,1.0,20,,,,,,,extra,fields\r\n" + + "D0120,dental,1,2.0,,,,,,,\r\n" + ) + z = make_zip(tmp_path / "asp_2020-Q1.zip", {"ASP Pricing File.csv": body}) + rows, _ = asp.parse_quarter_zip(z, "2020-Q1", source_url="") + by = {r.hcpcs: r for r in rows} + # a line with more fields than columns is malformed and skipped + assert set(by) == {"J9035", "J0000", "J0001", "D0120"} + assert by["J9035"].payment_limit == 1066.5 and by["J9035"].notes == "see note" + assert by["J0000"].payment_limit is None and by["J0000"].coinsurance_pct is None + assert by["J0001"].dosage == "" and by["J0001"].short_desc == "Short row" + assert by["D0120"].level == "CDT" and by["D0120"].short_desc == "" + assert by["J9035"].item_key == "" and by["J9035"].source_url == "" + + +def test_not_payable_rows_kept(tmp_path): + z = make_zip( + tmp_path / "asp_2026-Q2.zip", + { + "April 2026 Drugs Not Payable Under Part B 030926.xls": b"not read", + "section 508 version of April 2026 Drugs Not Payable Under Part B 030926.csv": NP_CSV, + "section 508 version of April 2026 Payment Limit File.csv": PRICING_HEAD + + PRICING_ROWS, + }, + ) + rows, _ = asp.parse_quarter_zip(z, "2026-Q2", source_url="u") + np_rows = [r for r in rows if r.list_kind == "not-payable"] + assert len(np_rows) == 2 and len(rows) == 4 + j = next(r for r in np_rows if r.hcpcs == "J7330") + assert j.payment_limit is None and j.coinsurance_pct is None + assert j.manufacturer == "Vericel Corporation" + assert j.short_desc == " Autologous cultured chondrocytes, implant " + assert j.source_file.endswith("Under Part B 030926.csv") + lvl1 = next(r for r in np_rows if r.hcpcs == "90999") + assert lvl1.level == "I" and lvl1.short_desc == "" and lvl1.manufacturer == "Acme" + assert all(r.manufacturer == "" for r in rows if r.list_kind == "payment") + + +def test_noc_member_parsed(tmp_path): + z = make_zip( + tmp_path / "noc_2025-Q4.zip", + { + "Oct 2025 NOC Pricing File 090425.xls": b"not read", + "section 508 version of Oct 2025 NOC Pricing File 090425.csv": NOC_CSV, + }, + ) + rows, noc = asp.parse_quarter_zip( + z, "2025-Q4", source_url="https://cms.test/n.zip", item_key="K2" + ) + assert rows == [] + assert [(n.drug_name, n.dosage, n.payment_limit) for n in noc] == [ + ("Examplumab", "10 MG", 12.5), + ("Vasopressin (Long Grove)", "1 UNIT", 2.462), + ] + assert noc[0].quarter == "2025-Q4" and noc[0].item_key == "K2" + assert noc[0].source_url == "https://cms.test/n.zip" + assert noc[0].source_file.endswith("090425.csv") + + +def test_bundled_zip_gives_both_and_one_member_per_kind(tmp_path): + # 2006-style zip: pricing, NOC and crosswalk, each as .xls and .csv twins + z = make_zip( + tmp_path / "asp_2006-Q2.zip", + { + "1 - Apr06_ASPbyHCPCS.xls": b"x", + "1 - Apr06_ASPbyHCPCS.csv": PRICING_HEAD + PRICING_ROWS, + "1 - Apr06_ASPNOC.xls": b"x", + "1 - Apr06_ASPNOC.csv": NOC_CSV, + "April06ASPNDC-HCPCSCrosswalk.csv": "HCPCS Code,NDC\r\nJ9035,1\r\n", + }, + ) + rows, noc = asp.parse_quarter_zip(z, "2006-Q2", source_url="") + assert [r.hcpcs for r in rows] == ["90371", "J9035"] + assert len(noc) == 2 + + +def test_header_missing_gives_no_rows(tmp_path): + z = make_zip( + tmp_path / "asp_2020-Q1.zip", {"ASP Pricing File.csv": "a,b\r\n1,2\r\n"} + ) + assert asp.parse_quarter_zip(z, "2020-Q1", source_url="") == ([], []) + z2 = make_zip(tmp_path / "asp_2020-Q2.zip", {"readme.txt": "hi"}) + assert asp.parse_quarter_zip(z2, "2020-Q2", source_url="") == ([], []) + + +class _Sheet: + def __init__(self, grid): + self.grid = grid + self.nrows = len(grid) + self.ncols = max(len(r) for r in grid) + + def cell_value(self, r, c): + row = self.grid[r] + return row[c] if c < len(row) else "" + + +class _Book: + def __init__(self, grid, sheets=None): + self.sheets = [("Sheet1", _Sheet(grid))] + [ + (name, _Sheet(g)) for name, g in (sheets or []) + ] + + def sheet_names(self): + return [n for n, _ in self.sheets] + + def sheet_by_index(self, i): + return self.sheets[i][1] + + +XLS_GRID = [ + ["Payment Allowance Limits"], + [""], + ["HCPCS Code", "Short Description", "HCPCS Code Dosage", "Payment Limit"], + [90371.0, "x", "1 ML", 140.209], + ["J9035 ", "Bevacizumab injection", "10 MG", 66.123], + ["", "", "", ""], + ["J0001", "Whole", "1 MG", 5.0], +] + + +def test_xls_member_through_fake_workbook(tmp_path): + z = make_zip( + tmp_path / "asp_2005-Q1.zip", {"Jan05ASPbyHCPCSv6_033106.xls": b"BIFF"} + ) + seen = {} + + def open_wb(**kw): + seen.update(kw) + return _Book(XLS_GRID) + + rows, noc = asp.parse_quarter_zip( + z, "2005-Q1", source_url="u", open_workbook=open_wb + ) + assert seen == {"file_contents": b"BIFF"} + assert noc == [] + assert [ + (r.hcpcs, r.level, r.short_desc, r.dosage, r.payment_limit) for r in rows + ] == [ + ("90371", "I", "", "1 ML", 140.209), + ("J9035", "II", "Bevacizumab injection", "10 MG", 66.123), + ("J0001", "II", "Whole", "1 MG", 5.0), + ] + csv_z = make_zip( + tmp_path / "asp_2005-Q2.zip", + {"ASP.csv": PRICING_HEAD + PRICING_ROWS}, + ) + csv_rows, _ = asp.parse_quarter_zip(csv_z, "2005-Q2", source_url="u") + assert [(r.hcpcs, r.dosage, r.payment_limit) for r in csv_rows] == [ + (r.hcpcs, r.dosage, r.payment_limit) for r in rows[:2] + ] + + +def test_xls_without_header_and_default_workbook(tmp_path, monkeypatch): + z = make_zip(tmp_path / "asp_2005-Q1.zip", {"ASP.xls": b"BIFF"}) + assert asp.parse_quarter_zip( + z, "2005-Q1", source_url="", open_workbook=lambda **kw: _Book([["a"], ["b"]]) + ) == ([], []) + import sys + import types + + fake = types.SimpleNamespace(open_workbook=lambda **kw: _Book(XLS_GRID)) + monkeypatch.setitem(sys.modules, "xlrd", fake) + rows, _ = asp.parse_quarter_zip(z, "2005-Q1", source_url="") + assert len(rows) == 3 + + +def test_noc_xls_member(tmp_path): + grid = [ + ["Drug Generic Name (Trade Name)", "Dosage", "Payment Limit"], + ["Abatacept", "250 MG", 477.0], + ["", "", ""], + ] + z = make_zip(tmp_path / "noc_2008-Q1.zip", {"Jan08 NOC.xls": b"BIFF"}) + _, noc = asp.parse_quarter_zip( + z, "2008-Q1", source_url="", open_workbook=lambda **kw: _Book(grid) + ) + assert [(n.drug_name, n.dosage, n.payment_limit) for n in noc] == [ + ("Abatacept", "250 MG", 477.0) + ] + + +# ── manifest and gaps ──────────────────────────────────────────────── +def test_manifest_urls_and_known_urls(root): + assert asp.manifest_urls() == set() + d = root / "cms" / "asp" + d.mkdir(parents=True) + f = d / "asp_2026-Q1.zip" + f.write_bytes(b"z") + dh.manifest(f, "https://cms.test/q1.zip") + assert asp.manifest_urls() == {"https://cms.test/q1.zip"} + (root / "cms" / "asp_source_urls.csv").write_text( + "quarter,url\n2026-Q2,https://cms.test/q2.zip\n" + ) + assert asp.canonical_urls() == {"2026-Q2": "https://cms.test/q2.zip"} + + +def test_plan_fetch_actions(root): + d = root / "cms" / "asp" + d.mkdir(parents=True) + (d / "asp_2026-Q1.zip").write_bytes(b"q1") + (d / "asp_2026-Q2.zip").write_bytes(b"q2") + (d / "asp_2025-Q4.zip").write_bytes(b"q4") + dh.manifest(d / "asp_2025-Q4.zip", "https://cms.test/2025q4.zip") + (root / "cms" / "asp_source_urls.csv").write_text( + "quarter,url\n2026-Q2,https://cms.test/q2-a.zip\n2026-Q1,https://cms.test/q1.zip\n" + ) + + def lk(q, kind, url): + return asp.AspLink(q, kind, url, "t") + + links = [ + lk("2025-Q4", "pricing", "https://cms.test/2025q4.zip"), # in manifest + lk("2026-Q1", "pricing", "https://cms.test/q1.zip"), # canonical, adopt + lk("2026-Q2", "pricing", "https://cms.test/q2-b.zip"), # updated posting + lk("2026-Q3", "pricing", "https://cms.test/q3.zip"), # new + lk("2026-Q3", "noc", "https://cms.test/n3.zip"), + lk("", "vaccine", "https://cms.test/v.zip"), + ] + plan = asp.plan_fetch(links, refresh=False) + assert [(p.link.quarter, p.link.kind, p.action) for p in plan] == [ + ("2025-Q4", "pricing", "have"), + ("2026-Q1", "pricing", "adopt"), + ("2026-Q2", "pricing", "fetch-beside"), + ("2026-Q3", "pricing", "fetch"), + ("2026-Q3", "noc", "fetch"), + ] + assert plan[3].dest == d / "asp_2026-Q3.zip" + forced = asp.plan_fetch(links, refresh=True) + assert {p.action for p in forced[:3]} == {"fetch-beside"} + assert forced[3].action == "fetch" + + +def test_gap_pairs_and_note(): + have = {"pricing": {"2026-Q1", "2026-Q3"}, "noc": set(), "crosswalk": set()} + listed = {("2026-Q3", "noc"), ("2026-Q3", "crosswalk")} + exp = ["2026-Q1", "2026-Q2", "2026-Q3"] + pairs = asp.gap_pairs(exp, have, listed) + assert pairs == ["2026-Q2 pricing", "2026-Q3 crosswalk", "2026-Q3 noc"] + assert asp.gap_note(pairs).startswith("no hoarded file for: 2026-Q2 pricing") + assert asp.gap_note([]) == "" + many = [f"{2000 + i}-Q1 pricing" for i in range(40)] + note = asp.gap_note(many) + assert note.endswith("(+28 more)") and "2011-Q1 pricing" in note + + +def test_missing_quarter_is_gap_not_zero(tmp_path): + z1 = make_zip( + tmp_path / "asp_2026-Q1.zip", {"ASP.csv": PRICING_HEAD + PRICING_ROWS} + ) + z3 = make_zip( + tmp_path / "asp_2026-Q3.zip", {"ASP.csv": PRICING_HEAD + PRICING_ROWS} + ) + got = asp.newest_per_quarter([z1, z3]) + have = {"pricing": set(got), "noc": set(), "crosswalk": set()} + pairs = asp.gap_pairs(["2026-Q1", "2026-Q2", "2026-Q3"], have, set()) + assert "2026-Q2 pricing" in asp.gap_note(pairs) + con = duckdb.connect(":memory:") + for q, z in (("2026-Q1", z1), ("2026-Q3", z3)): + rows, _ = asp.parse_quarter_zip(z, q, source_url="") + asp.write_asp(con, q, rows) + assert asp.read_asp(con, quarter="2026-Q2") == [] + assert len(asp.read_asp(con, quarter="2026-Q1")) == 2 + + +# ── a transport-level updated posting lands beside ────────────────── +def test_updated_posting_lands_beside(root): + d = root / "cms" / "asp" + d.mkdir(parents=True) + old = d / "asp_2026-Q2.zip" + old.write_bytes(b"old posting") + dh.manifest(old, "https://cms.test/A.zip") + link = asp.AspLink("2026-Q2", "pricing", "https://cms.test/B.zip", "t") + (plan,) = asp.plan_fetch([link], refresh=False) + assert plan.action == "fetch-beside" + seen = [] + + def handler(req): + seen.append(str(req.url)) + return httpx.Response(200, content=b"new posting") + + cl = httpx.Client(transport=httpx.MockTransport(handler)) + out = asp.apply_plan(cl, plan) + assert seen == ["https://cms.test/B.zip"] + assert out != old and out.read_bytes() == b"new posting" + assert old.read_bytes() == b"old posting" + urls = [ + json.loads(x)["url"] + for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines() + ] + assert urls == ["https://cms.test/A.zip", "https://cms.test/B.zip"] + # it is now the newest copy of the quarter + assert asp.newest_per_quarter(d.glob("asp_*.zip")) == {"2026-Q2": out} + # a second plan sees B in the manifest + (again,) = asp.plan_fetch([link], refresh=False) + assert again.action == "have" + + +def test_apply_plan_adopt_and_have(root): + d = root / "cms" / "asp" + d.mkdir(parents=True) + f = d / "asp_2026-Q1.zip" + f.write_bytes(b"q1") + (root / "cms" / "asp_source_urls.csv").write_text( + "quarter,url\n2026-Q1,https://cms.test/q1.zip\n" + ) + link = asp.AspLink("2026-Q1", "pricing", "https://cms.test/q1.zip", "t") + (plan,) = asp.plan_fetch([link], refresh=False) + assert plan.action == "adopt" + boom = httpx.Client(transport=httpx.MockTransport(lambda r: pytest.fail("fetched"))) + assert asp.apply_plan(boom, plan) == f + rec = json.loads((root / "drugs/MANIFEST.jsonl").read_text()) + assert ( + rec["url"] == "https://cms.test/q1.zip" and "asp_source_urls.csv" in rec["note"] + ) + (plan2,) = asp.plan_fetch([link], refresh=False) + assert plan2.action == "have" and asp.apply_plan(boom, plan2) == f + + +def test_page_hoard(root): + seen = [] + + def handler(req): + seen.append(req.headers["user-agent"]) + return httpx.Response(200, content=b"one") + + cl = httpx.Client( + headers={"User-Agent": "ua"}, transport=httpx.MockTransport(handler) + ) + p = asp.fetch_page(cl) + assert ( + p == root / "drugs/cms/asp-pricing-files.html" + and p.read_bytes() == b"one" + ) + assert asp.hoarded_page() == p + cl2 = httpx.Client( + transport=httpx.MockTransport( + lambda r: httpx.Response(200, content=b"two") + ) + ) + p2 = asp.fetch_page(cl2) + assert p2 != p and p.read_bytes() == b"one" + assert asp.hoarded_page() == p2 + assert ( + asp.ASP_PAGE + == "https://www.cms.gov/medicare/payment/part-b-drugs/asp-pricing-files" + ) + + +def test_hoarded_page_absent(root): + assert asp.hoarded_page() is None + + +# ── tables ─────────────────────────────────────────────────────────── +def test_write_noc_replaces_quarter(): + con = duckdb.connect(":memory:") + a = T.NocRow("2025-Q4", "A", "1 MG", 1.0, "f", "u", "k") + b = T.NocRow("2025-Q3", "B", "1 MG", None, "f", "u", "k") + asp.write_noc(con, "2025-Q4", [a]) + asp.write_noc(con, "2025-Q3", [b]) + asp.write_noc(con, "2025-Q4", [T.NocRow("2025-Q4", "C", "", 2.0, "f", "u", "")]) + assert [r.drug_name for r in asp.read_noc(con)] == ["B", "C"] + assert [r.drug_name for r in asp.read_noc(con, "2025-Q3")] == ["B"] + + +def test_2005_style_noc_sheet_inside_the_pricing_workbook(tmp_path): + noc_grid = [ + ["Note"], + ["Drug Name (Generic)", "Dosage", "Payment Limit", "Notes"], + ["Allopurinol Sodium", "500 MG", "455.714", "Revised"], + ["Amikacin Sulfate", "50 MG", "$0.492", ""], + ] + z = make_zip(tmp_path / "asp_2005-Q1.zip", {"Jan05ASPbyHCPCSv6.xls": b"BIFF"}) + rows, noc = asp.parse_quarter_zip( + z, + "2005-Q1", + source_url="u", + open_workbook=lambda **kw: _Book(XLS_GRID, [("Jan05_NOC_rev", noc_grid)]), + ) + assert len(rows) == 3 + assert [(n.drug_name, n.payment_limit) for n in noc] == [ + ("Allopurinol Sodium", 455.714), + ("Amikacin Sulfate", 0.492), + ] + assert noc[0].source_file == "Jan05ASPbyHCPCSv6.xls#NOC" + # no NOC sheet: no NOC rows + _, none = asp.parse_quarter_zip( + z, "2005-Q1", source_url="u", open_workbook=lambda **kw: _Book(XLS_GRID) + ) + assert none == [] + + +def test_manufacturer_is_stripped(tmp_path): + body = "HCPCS Code,Long Description,Manufacturer\r\nJ0209,Some drug,\tHope Pharmaceuticals \r\n" + z = make_zip(tmp_path / "asp_2026-Q4.zip", {"Drugs Not Payable.csv": body}) + (row,), _ = asp.parse_quarter_zip(z, "2026-Q4", source_url="") + assert row.manufacturer == "Hope Pharmaceuticals" and row.list_kind == "not-payable" diff --git a/tests/drugs/test_codes.py b/tests/drugs/test_codes.py index c99da51..2af65af 100644 --- a/tests/drugs/test_codes.py +++ b/tests/drugs/test_codes.py @@ -1,5 +1,7 @@ from datetime import date +import pytest + from drugs import codes @@ -34,7 +36,8 @@ def test_appl_id(): assert codes.appl_id("A", "90123") == "ANDA090123" assert codes.appl_id("BLA", "125057") == "BLA125057" assert codes.appl_id("NDA021436", "") == "NDA021436" - assert codes.appl_id("X", "5") == "X000005" + assert codes.appl_id("NDA", "21436") == "NDA021436" + assert codes.appl_id("ANDA", "90123") == "ANDA090123" def test_expand(): @@ -54,3 +57,31 @@ def test_quarters_and_fy(): assert codes.quarter_end("2026-Q1") == date(2026, 3, 31) assert codes.fy_effective("2026") == date(2025, 10, 1) assert codes.fy_effective("2026-04") == date(2026, 4, 1) + + +def test_level_and_ndc_reject_trailing_newline_and_fullwidth_digits(): + assert codes.level("J9035\n") == "" + assert codes.level("90371") == "" + assert codes.level("90371") == "" + assert codes.licensed("90371\n") is False + assert codes.ndc11("000214338011") is None + assert codes.ndc11("00021433801") is None + assert codes.ndc11("0002-1433-80") is None + assert codes.ndc11("0002-1433-80") == "00002143380" + + +def test_appl_id_rejects_unknown_kinds(): + with pytest.raises(ValueError, match="unknown application kind"): + codes.appl_id("X", "5") + with pytest.raises(ValueError): + codes.appl_id("", "5") + + +def test_expand_caps_range_size(): + assert len(codes.expand("J0001", "J0500")) == 500 + assert codes.MAX_RANGE == 500 + with pytest.raises(ValueError, match="range too large"): + codes.expand("J0001", "J9999") + with pytest.raises(ValueError): + codes.expand("J0001", "J0501") + assert codes.expand("J9001", "J9002") == ["J9001"] diff --git a/tests/drugs/test_imports.py b/tests/drugs/test_imports.py index 08c7138..3905e91 100644 --- a/tests/drugs/test_imports.py +++ b/tests/drugs/test_imports.py @@ -5,7 +5,7 @@ from __future__ import annotations import subprocess import sys -MODULES = ["drugs.http", "drugs.codes", "drugs.tables"] +MODULES = ["drugs.http", "drugs.codes", "drugs.tables", "drugs.asp"] _PROBE = """ import builtins, importlib, sys diff --git a/tests/drugs/test_tables.py b/tests/drugs/test_tables.py index 088eecf..2e5e825 100644 --- a/tests/drugs/test_tables.py +++ b/tests/drugs/test_tables.py @@ -1,4 +1,5 @@ import duckdb +import pytest from drugs import tables as T @@ -16,7 +17,7 @@ def test_table_names_empty(): con = duckdb.connect(":memory:") assert T.table_counts(con) == {} T.ensure_tables(con) - assert T.table_counts(con) == {} + assert T.table_counts(con) == {"drugs.asp_noc": 0, "drugs.asp_payment": 0} def test_helpers(): @@ -37,3 +38,92 @@ def test_helpers(): 1, ] con.execute("CREATE TABLE drugs.x (a INTEGER)") if False else None + + +ROUNDTRIP = { + "asp_payment": ( + T.write_asp, + T.read_asp, + [ + T.AspRow( + "2026-Q2", + "J9035", + "II", + "payment", + "10 MG", + 66.123, + 20.0, + "Bevacizumab injection", + "n", + "f.csv", + "https://x.test/a.zip", + "K1", + ), + T.AspRow( + "2026-Q2", + "J7330", + "II", + "not-payable", + "", + None, + None, + "Autologous cultured chondrocytes, implant", + "", + "np.csv", + "https://x.test/a.zip", + "K1", + "Vericel Corporation", + ), + ], + ), + "asp_noc": ( + T.write_noc, + T.read_noc, + [ + T.NocRow( + "2026-Q2", + "Examplumab", + "10 MG", + 12.5, + "n.csv", + "https://x.test/n.zip", + "K2", + ), + T.NocRow("2026-Q2", "Other", "", None, "n.csv", "https://x.test/n.zip", ""), + ], + ), +} + + +@pytest.mark.parametrize("name", sorted(ROUNDTRIP)) +def test_roundtrip(name): + write, read, rows = ROUNDTRIP[name] + con = duckdb.connect(":memory:") + assert write(con, "2026-Q2", rows) == len(rows) + assert sorted(read(con), key=repr) == sorted(rows, key=repr) + # writing a quarter again replaces it, other quarters are untouched + assert write(con, "2026-Q2", rows[:1]) == 1 + assert read(con) == rows[:1] + assert T.table_counts(con)[f"drugs.{name}"] == 1 + + +def test_write_empty_quarter_clears_it(): + con = duckdb.connect(":memory:") + rows = ROUNDTRIP["asp_payment"][2] + T.write_asp(con, "2026-Q2", rows) + assert T.write_asp(con, "2026-Q2", []) == 0 + assert T.read_asp(con) == [] + + +def test_read_asp_filters(): + con = duckdb.connect(":memory:") + a, b = ROUNDTRIP["asp_payment"][2] + c = T.AspRow( + "2026-Q1", "J9035", "II", "payment", "10 MG", 60.0, 20.0, "x", "", "f", "u", "" + ) + T.write_asp(con, "2026-Q2", [a, b]) + T.write_asp(con, "2026-Q1", [c]) + assert [r.quarter for r in T.read_asp(con, hcpcs="J9035")] == ["2026-Q1", "2026-Q2"] + assert T.read_asp(con, quarter="2026-Q1") == [c] + assert T.read_asp(con, hcpcs="J7330", quarter="2026-Q2") == [b] + assert T.read_asp(con, hcpcs="ZZZZZ") == []