Files
stack/dev/scripts/ingest_opps.py
kert 1aa288c344
Some checks failed
CI / lint (push) Failing after 48s
CI / notebooks-smoke (push) Successful in 2m0s
Deploy / changes (push) Successful in 18s
CI / test (push) Successful in 4m23s
Infra CI / notebooks (push) Successful in 1m5s
Infra CI / zotero (push) Successful in 16s
Infra CI / docs (push) Successful in 19s
Infra CI / api (push) Successful in 1m19s
Infra CI / bcda (push) Successful in 1m48s
Infra CI / mc (push) Successful in 18s
Infra CI / llm (push) Successful in 56s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 2m34s
Lean Pay / pay (push) Successful in 3m22s
Deploy / llm (push) Successful in 1m43s
Deploy / mc (push) Has been skipped
Deploy / bcda (push) Successful in 2m10s
Deploy / report (push) Successful in 16s
feat(pay): load the published numbers the fence was missing
The wage index, pass-through rates, the missing APC years, the CY2027
work GPCI, equipment usage, IPPS standardized amounts, MS-DRG weights,
and the MSSP and REACH financial results now come from the CMS files.
The snapshot records those observed ranges. Maintenance stays the
published 5/100. There is still no IPPS or LEAD pricer.
2026-11-17 17:04:51 -05:00

481 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Ingest CMS OPPS addendum files into DuckDB.
Parses Addendum A (APC weights), Addendum B (HCPCS→APC map), and
wage index files from downloaded ZIPs. Handles column naming
variations across 13 years of CMS format drift (CY2014–2026).
Produces:
* opps.apc_weight (Addendum A — APC relative weights)
* opps.addendum_b (Addendum B — HCPCS→APC crosswalk)
* opps.wage_index (Wage index by CBSA)
Usage:
uv run python dev/scripts/download_opps_files.py # download first
uv run python dev/scripts/ingest_opps.py # then ingest
uv run python dev/scripts/ingest_opps.py --year 2026
"""
from __future__ import annotations
import argparse
import io
import re
import zipfile
from pathlib import Path
import duckdb
import pandas as pd
ROOT = Path(__file__).resolve().parents[2]
OPPS_DIR = ROOT / "data" / "cms" / "opps"
DUCKDB_PATH = ROOT / "data" / "aco.duckdb"
# ---------------------------------------------------------------------------
# Column normalisation — CMS changes headers across years
# ---------------------------------------------------------------------------
ADDENDUM_A_COL_MAP = {
"apc": "apc",
"apc group": "apc",
"apc group number": "apc",
"group title": "group_title",
"apc group title": "group_title",
"status indicator": "status_indicator",
"si": "status_indicator",
"relative weight": "relative_weight",
"payment rate": "payment_rate",
"national unadjusted payment": "payment_rate",
"minimum unadjusted copayment": "minimum_unadjusted_copayment",
}
ADDENDUM_B_COL_MAP = {
"hcpcs code": "hcpcs",
"hcpcs": "hcpcs",
"cpt/hcpcs": "hcpcs",
"short descriptor": "short_description",
"short description": "short_description",
"si": "status_indicator",
"status indicator": "status_indicator",
"apc": "apc",
"apc group": "apc",
"apc group number": "apc",
"apc title": "apc_title",
"relative weight": "relative_weight",
"payment rate": "payment_rate",
"national unadjusted payment": "payment_rate",
"minimum unadjusted copayment": "minimum_unadjusted_copayment",
}
WAGE_INDEX_COL_MAP = {
"cbsa": "cbsa",
"cbsa number": "cbsa",
"cbsa name": "cbsa_name",
"urban area title": "cbsa_name",
"state": "state",
"wage index": "wage_index",
"pre-reclassification wage index": "wage_index",
"reclassified wage index": "reclassified_wage_index",
"post-reclassification wage index": "reclassified_wage_index",
}
def normalise_columns(df: pd.DataFrame, col_map: dict[str, str]) -> pd.DataFrame:
"""Lowercase columns, apply mapping, keep only mapped columns."""
df.columns = [str(c).strip().lower() for c in df.columns]
df = df.rename(columns=col_map)
present = [c for c in set(col_map.values()) if c in df.columns]
return df[present]
# ---------------------------------------------------------------------------
# Readers — handle ZIP contents (CSV or Excel inside)
# ---------------------------------------------------------------------------
def find_data_file(zf: zipfile.ZipFile, pattern: str = "") -> str | None:
"""Find a specific addendum file inside a combined addenda ZIP.
CMS OPPS addenda ZIPs contain multiple addenda (A, B, C, D1, D2, etc.)
as both XLSX and CSV (508 version). Prefer XLSX in the root, fall back
to CSV in the 508 subfolder.
"""
skip = ["__macosx", ".ds_store", "readme", "note"]
candidates = []
for name in zf.namelist():
low = name.lower()
if any(s in low for s in skip):
continue
if low.endswith((".csv", ".xlsx", ".xls")):
candidates.append(name)
if pattern:
# Match "Addendum A" / "Addendum B" etc., separator-insensitively:
# CMS names use spaces ("Addendum B.xlsx") and hyphens
# ("CMS-1613-FC-Addendum-B.xlsx") interchangeably across years.
pn = re.sub(r"[^a-z0-9]", "", pattern.lower())
filtered = [c for c in candidates if pn in re.sub(r"[^a-z0-9]", "", c.lower())]
# Exclude the "Data Addendum B" geometric-mean-cost file — it carries
# SI/APC/Comment-Indicator only, no payment rate, so it never
# reconciles. Its name varies ("Data-Addendum-B", "DataAddB",
# "Data Add B"); normalise and drop anything containing "dataadd".
real = [
c for c in filtered if "dataadd" not in re.sub(r"[^a-z0-9]", "", c.lower())
]
pool = real or filtered
# Prefer xlsx, then xls, then csv; prefer root over a 508 subfolder.
for in_root in (True, False):
for ext in (".xlsx", ".xls", ".csv"):
hits = [
f
for f in pool
if f.lower().endswith(ext) and (("/" not in f) == in_root)
]
if hits:
return sorted(hits)[0]
if pool:
return sorted(pool)[0]
return candidates[0] if candidates else None
def _find_header_row(df: pd.DataFrame, col_map: dict[str, str]) -> int | None:
"""Find the row index containing column headers in a preamble-laden CMS file.
Requires >= 3 matches AND at least 3 non-null cells (to distinguish
a real header row from a title row that happens to contain keywords).
None means no row cleared that bar. A header on row 0 is 0, not a miss.
"""
target_cols = [k for k in col_map if k not in ("", "year")]
for i, row in df.iterrows():
vals = [str(v).strip().lower() for v in row if pd.notna(v) and str(v).strip()]
if len(vals) < 3:
continue
matches = sum(
1 for v in vals for k in target_cols if v == k or (len(k) > 3 and k in v)
)
if matches >= 3:
return int(i)
return None
def _excel_payment_sheet(
data: bytes, col_map: dict[str, str]
) -> tuple[str, int] | None:
"""First sheet whose opening rows are a payment header.
Some Addendum A workbooks put an APC-category lookup on sheet 0 and
the relative weights on the next sheet. A sheet counts only when one
of its first 20 rows matches at least three column-map headers. The
first such sheet wins, so a workbook whose first sheet is already the
payment sheet stays on that sheet.
"""
book = pd.ExcelFile(io.BytesIO(data))
try:
for sheet in book.sheet_names:
try:
preview = pd.read_excel(
book, sheet_name=sheet, dtype=str, header=None, nrows=20
)
except Exception:
continue
hdr = _find_header_row(preview, col_map)
if hdr is not None:
return str(sheet), hdr
finally:
book.close()
return None
def read_from_zip(
zpath: Path, col_map: dict[str, str], pattern: str = ""
) -> pd.DataFrame:
"""Read and normalise a data file from inside a ZIP.
Handles CMS preamble rows in both CSV and Excel formats.
"""
try:
with zipfile.ZipFile(zpath) as zf:
target = find_data_file(zf, pattern)
if not target:
return pd.DataFrame()
data = zf.read(target)
if target.lower().endswith(".csv"):
raw = data.decode("latin-1")
lines = raw.splitlines()
# Find header row
hdr_idx = 0
target_cols = list(col_map.keys())[:5]
for i, line in enumerate(lines[:20]):
low = line.strip().lower()
if sum(1 for k in target_cols if k in low) >= 2:
hdr_idx = i
break
buf = "\n".join(lines[hdr_idx:])
df = pd.read_csv(io.StringIO(buf), dtype=str, on_bad_lines="skip")
else:
found = _excel_payment_sheet(data, col_map)
if found is None:
print(f" {zpath.name}/{pattern}: no payment sheet")
return pd.DataFrame()
sheet, hdr_row = found
df = pd.read_excel(
io.BytesIO(data),
sheet_name=sheet,
dtype=str,
header=hdr_row,
)
if sheet:
print(
f" {pattern or target}: sheet {sheet!r} header row {hdr_row}"
)
return normalise_columns(df, col_map)
except Exception as e:
print(f" ERROR: {zpath.name}/{pattern}: {e}")
return pd.DataFrame()
def clean_numeric(series: pd.Series) -> pd.Series:
"""Strip $ and commas, convert to float."""
return (
series.astype(str)
.str.replace("$", "", regex=False)
.str.replace(",", "", regex=False)
.str.strip()
.pipe(pd.to_numeric, errors="coerce")
)
# ---------------------------------------------------------------------------
# Main ingestion
# ---------------------------------------------------------------------------
def ingest_all(con: duckdb.DuckDBPyConnection, year_filter: str = "") -> None:
"""Ingest all downloaded OPPS files into DuckDB."""
con.execute("CREATE SCHEMA IF NOT EXISTS opps")
apc_frames: list[pd.DataFrame] = []
addb_frames: list[pd.DataFrame] = []
wage_frames: list[pd.DataFrame] = []
year_dirs = sorted(OPPS_DIR.iterdir())
if year_filter:
year_dirs = [d for d in year_dirs if d.name == year_filter]
for year_dir in year_dirs:
if not year_dir.is_dir():
continue
year = year_dir.name
print(f"\n--- CY{year} ---")
# Combined addenda ZIP (CY2021+)
addenda_zip = year_dir / f"opps_{year}_addenda.zip"
if not addenda_zip.exists():
# Try old single-file patterns
addenda_zip = year_dir / f"opps_{year}_addendum_a.zip"
if addenda_zip.exists():
# Addendum A — APC weights
df = read_from_zip(addenda_zip, ADDENDUM_A_COL_MAP, pattern="addendum a")
if not df.empty:
df["year"] = int(year)
for col in [
"relative_weight",
"payment_rate",
"minimum_unadjusted_copayment",
]:
if col in df.columns:
df[col] = clean_numeric(df[col])
apc_frames.append(df)
print(f" Addendum A: {len(df)} APCs")
# Addendum B — HCPCS→APC crosswalk
df = read_from_zip(addenda_zip, ADDENDUM_B_COL_MAP, pattern="addendum b")
if not df.empty:
df["year"] = int(year)
for col in [
"relative_weight",
"payment_rate",
"minimum_unadjusted_copayment",
]:
if col in df.columns:
df[col] = clean_numeric(df[col])
addb_frames.append(df)
print(f" Addendum B: {len(df)} HCPCS codes")
# Load into DuckDB
print("\n--- Loading into DuckDB ---")
def _load(table: str, frames: list[pd.DataFrame]) -> None:
"""Replace `table` — but with --year, merge instead of wipe.
A year-filtered run only parses that year's files, so a bare
DROP/CREATE here used to erase every other year (#509). When
filtering, keep the existing table's other-year rows and rebuild
from their union; pandas concat also unions columns, so CMS
format drift between years stays handled either way.
"""
merged = pd.concat(frames, ignore_index=True)
if year_filter:
exists = con.execute(
"SELECT 1 FROM information_schema.tables "
"WHERE table_schema = 'opps' AND table_name = ?",
[table],
).fetchone()
if exists:
other_years = con.execute(
f"SELECT * FROM opps.{table} WHERE year != ?", # noqa: S608
[int(year_filter)],
).df()
merged = pd.concat([other_years, merged], ignore_index=True)
con.execute(f"DROP TABLE IF EXISTS opps.{table}")
con.execute(f"CREATE TABLE opps.{table} AS SELECT * FROM merged") # noqa: S608
print(f" opps.{table}: {len(merged)} rows ({merged['year'].nunique()} years)")
if apc_frames:
_load("apc_weight", apc_frames)
if addb_frames:
_load("addendum_b", addb_frames)
if wage_frames:
_load("wage_index", wage_frames)
# Skin sub specific: extract skin sub codes from Addendum B history.
# Derived from the (merged) table rather than this run's frames, so a
# --year run keeps the other years here too.
if addb_frames:
print("\n--- Skin substitute history in Addendum B ---")
con.execute("DROP TABLE IF EXISTS opps.skin_sub_addendum_b")
con.execute(
"CREATE TABLE opps.skin_sub_addendum_b AS "
"SELECT * FROM opps.addendum_b "
"WHERE regexp_matches(CAST(hcpcs AS VARCHAR), '^Q4\\d{2,3}$|^C527[1-8]$')"
)
n, years, codes = con.execute(
"SELECT count(*), count(DISTINCT year), count(DISTINCT hcpcs) "
"FROM opps.skin_sub_addendum_b"
).fetchone()
print(
f" opps.skin_sub_addendum_b: {n} rows ({years} years, {codes} unique codes)"
)
# SI distribution for skin subs (column absent in some year formats)
has_si = con.execute(
"SELECT 1 FROM information_schema.columns "
"WHERE table_schema = 'opps' AND table_name = 'skin_sub_addendum_b' "
"AND column_name = 'status_indicator'"
).fetchone()
si_rows = (
con.execute(
"SELECT year, status_indicator, count(*) "
"FROM opps.skin_sub_addendum_b "
"GROUP BY year, status_indicator ORDER BY year, status_indicator"
).fetchall()
if has_si
else []
)
if si_rows:
print(" Status indicators for skin subs by year:")
for yr, si, ct in si_rows:
print(f" CY{yr} SI={si}: {ct} codes")
# Pass-through rates are Addendum B rows with status indicator G
# (drugs and biologicals) or H (devices). Addendum D1/D2 are indicator
# definitions, not payment rates. Built from the table, so a --year
# run keeps every other year.
load_pass_through(con)
def load_pass_through(con: duckdb.DuckDBPyConnection) -> int:
"""Copy Addendum B status indicators G and H into opps.pass_through.
``asp_per_unit`` stays null. ``drugs.asp_payment.payment_limit`` is
the ASP+6% allowance, and dividing it by 1.06 would invent an ASP.
Status indicator K is a separately payable drug, not pass-through.
"""
exists = con.execute(
"SELECT 1 FROM information_schema.tables "
"WHERE table_schema = 'opps' AND table_name = 'addendum_b'"
).fetchone()
if not exists:
return 0
has_si = con.execute(
"SELECT 1 FROM information_schema.columns "
"WHERE table_schema = 'opps' AND table_name = 'addendum_b' "
"AND column_name = 'status_indicator'"
).fetchone()
if not has_si:
return 0
con.execute("CREATE SCHEMA IF NOT EXISTS opps")
con.execute("DROP TABLE IF EXISTS opps.pass_through")
con.execute(
"""
CREATE TABLE opps.pass_through AS
SELECT
TRIM(CAST(hcpcs AS VARCHAR)) AS hcpcs,
short_description,
CASE UPPER(TRIM(CAST(status_indicator AS VARCHAR)))
WHEN 'G' THEN 'drug'
WHEN 'H' THEN 'device'
END AS pass_through_type,
UPPER(TRIM(CAST(status_indicator AS VARCHAR))) AS status_indicator,
payment_rate,
CAST(NULL AS DOUBLE) AS asp_per_unit,
year
FROM opps.addendum_b
WHERE UPPER(TRIM(CAST(status_indicator AS VARCHAR))) IN ('G', 'H')
"""
)
n = con.execute("SELECT count(*) FROM opps.pass_through").fetchone()[0]
print(f" opps.pass_through: {n} rows (Addendum B SI G and H)")
return int(n)
def main() -> None:
parser = argparse.ArgumentParser(description="Ingest CMS OPPS files")
parser.add_argument("--year", help="Ingest only this year")
parser.add_argument(
"--no-lake",
action="store_true",
help="skip publishing to the DuckLake lakehouse",
)
args = parser.parse_args()
print("Ingesting CMS OPPS files into DuckDB ...")
# duckdb_batch preflights the single-writer lock (retry + name the
# holding PID) instead of dying on a raw IOException when a notebook
# kernel holds a connection (#508).
from conf.connect import duckdb_batch, publish_replica
with duckdb_batch("aco") as con:
ingest_all(con, year_filter=args.year or "")
# Final inventory
print("\n--- OPPS tables ---")
for r in con.execute("""
SELECT table_name FROM information_schema.tables
WHERE table_schema = 'opps' ORDER BY table_name
""").fetchall():
cnt = con.execute(f"SELECT count(*) FROM opps.{r[0]}").fetchone()[0] # noqa: S608
print(f" opps.{r[0]:30s}: {cnt:>6} rows")
# Refresh the notebook read replica so long-running readers see the
# new data without ever locking this primary (#510).
replica = publish_replica("aco")
print(f"replica → {replica}")
if not args.no_lake:
print("\n--- Publishing to DuckLake ---")
import _lake
_lake.publish_lake(("opps",))
print("\nDone.")
if __name__ == "__main__":
main()