Files
stack/dev/scripts/ingest_ipps_tables.py
kert 1aa288c344
Some checks failed
CI / lint (push) Failing after 48s
CI / notebooks-smoke (push) Successful in 2m0s
Deploy / changes (push) Successful in 18s
CI / test (push) Successful in 4m23s
Infra CI / notebooks (push) Successful in 1m5s
Infra CI / zotero (push) Successful in 16s
Infra CI / docs (push) Successful in 19s
Infra CI / api (push) Successful in 1m19s
Infra CI / bcda (push) Successful in 1m48s
Infra CI / mc (push) Successful in 18s
Infra CI / llm (push) Successful in 56s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 2m34s
Lean Pay / pay (push) Successful in 3m22s
Deploy / llm (push) Successful in 1m43s
Deploy / mc (push) Has been skipped
Deploy / bcda (push) Successful in 2m10s
Deploy / report (push) Successful in 16s
feat(pay): load the published numbers the fence was missing
The wage index, pass-through rates, the missing APC years, the CY2027
work GPCI, equipment usage, IPPS standardized amounts, MS-DRG weights,
and the MSSP and REACH financial results now come from the CMS files.
The snapshot records those observed ranges. Maintenance stays the
published 5/100. There is still no IPPS or LEAD pricer.
2026-11-17 17:04:51 -05:00

732 lines
26 KiB
Python

"""Load published IPPS tables and the MSSP / ACO REACH settlement files.
Wage index: IPPS final-rule Table 3 (CBSA). Calendar year N of OPPS uses
fiscal year N. Years whose Table 3 is split into average-hourly-wage
files (FY2014 and FY2015) use Table 4A and 4B for the wage index and
Table 4C for the reclassified index. A correction-notice file replaces
the final-rule file only when it is a full reprint, not a short delta.
Table 5 is the MS-DRG weight. When CMS publishes both a weight and a
10% cap-applied weight, the cap-applied weight is the one stored.
Table 1A-1E is the national standardized amounts. Each published dollar
is one point.
There is still no claim-level IPPS pricer. IME, DSH, and outliers are
not computed here.
"""
from __future__ import annotations
import csv
import io
import re
import zipfile
from pathlib import Path
from urllib.parse import unquote
import openpyxl
import pandas as pd
ROOT = Path(__file__).resolve().parents[2]
IPPS_DIR = ROOT / "data" / "cms" / "ipps"
MSSP_DIR = ROOT / "data" / "cms" / "mssp"
REACH_DIR = ROOT / "data" / "cms" / "reach"
_WAGE_KINDS = ("table3", "table4a", "table4b", "table4c")
def _norm_header(value: object) -> str:
text = re.sub(r"\s+", " ", str(value or "")).strip().lower()
text = text.replace("\xa0", " ")
text = re.sub(r"^\d+\s*", "", text)
return text.strip(" *")
def _money(value: object) -> float | None:
if value is None or (isinstance(value, float) and pd.isna(value)):
return None
if isinstance(value, bool):
return None
if isinstance(value, int | float):
return float(value)
text = str(value).strip().replace("$", "").replace(",", "").replace('"', "")
if text.lower() in {"", "not applicable", "n/a", "na", "------", "*"}:
return None
try:
return float(text)
except ValueError:
return None
def _cn_rank(name: str) -> int:
"""Higher wins when two files both reprint the table.
A numbered correction notice outranks an unnumbered one. The FY2016
CAA reprint outranks the correction notice. A short delta still loses
on row count before this rank is consulted.
"""
low = Path(name).name.lower()
if "caa" in low:
return 3
numbered = re.search(r"cn(\d+)", low)
if numbered:
return int(numbered.group(1))
if (
"correction" in low
or "correcting" in low
or re.match(r"ca\b", low)
or re.search(r"(^|[^a-z])cn([^a-z]|$)", low)
):
return 1
return 0
def _member_kind(name: str) -> str | None:
"""Table id from the member name. ``Tables 3`` and ``Table_5`` both count.
``3`` inside a rule number (``1735``) does not. Table 3A and 3B are
average hourly wage, not the index, so a letter or digit after the 3
is not Table 3.
"""
base = Path(name).name.lower()
if not base.endswith(".txt"):
return None
if re.search(r"tables?[_\s-]*3([^0-9ab]|$)", base):
return "table3"
if re.search(r"tables?[_\s-]*4a([^0-9a-z]|$)", base):
return "table4a"
if re.search(r"tables?[_\s-]*4b([^0-9a-z]|$)", base):
return "table4b"
if re.search(r"tables?[_\s-]*4c([^0-9a-z]|$)", base):
return "table4c"
if re.search(r"tables?[_\s-]*5([^0-9]|$)", base):
return "table5"
return None
def _read_zip_text(zf: zipfile.ZipFile, name: str) -> str:
return zf.read(name).decode("latin-1", "replace")
def _rows(text: str) -> list[list[str]]:
return list(csv.reader(io.StringIO(text), delimiter="\t"))
def _header_index(rows: list[list[str]], *needles: str) -> int | None:
for index, row in enumerate(rows[:15]):
norms = [_norm_header(cell) for cell in row]
if any(any(needle in norm for needle in needles) for norm in norms):
if any(
norm in {"cbsa", "cbsa code", "ms-drg"} or norm.startswith("ms-drg")
for norm in norms
):
return index
return None
def _wage_columns(header: list[str], *, reclassified_only: bool) -> dict[str, int]:
found: dict[str, int] = {}
for index, cell in enumerate(header):
norm = _norm_header(cell)
if norm in {"cbsa", "cbsa code"} and "cbsa" not in found:
found["cbsa"] = index
elif norm in {
"area name",
"urban area",
"rural area",
"nonurban area",
"area",
"state or urban area",
}:
found["cbsa_name"] = index
elif norm in {"state", "state*"} or (
norm.startswith("state") and "code" not in norm
):
found.setdefault("state", index)
elif _is_reclassified_wage(norm):
found["reclassified_wage_index"] = index
elif _is_plain_wage_index(norm):
if reclassified_only:
# Table 4C labels the reclassified index as "Wage Index".
# A real reclassified column, when the file has both, wins.
found.setdefault("reclassified_wage_index", index)
else:
found.setdefault("wage_index", index)
return found
def _is_reclassified_wage(norm: str) -> bool:
"""The CBSA reclassified index, not the Puerto Rico or GAF column."""
if "puerto" in norm or "gaf" in norm:
return False
return "reclassified" in norm and "wage index" in norm
def _is_plain_wage_index(norm: str) -> bool:
if "reclassified" in norm or "puerto" in norm or "gaf" in norm:
return False
return norm == "wage index" or norm.startswith("wage index ")
def parse_wage_text(
text: str, year: int, *, reclassified_only: bool = False
) -> pd.DataFrame:
"""One Table 3 (or Table 4A/4B/4C) text file."""
rows = _rows(text)
header_at = _header_index(rows, "wage index", "cbsa")
if header_at is None:
return pd.DataFrame()
columns = _wage_columns(rows[header_at], reclassified_only=reclassified_only)
if "cbsa" not in columns:
return pd.DataFrame()
if "wage_index" not in columns and "reclassified_wage_index" not in columns:
return pd.DataFrame()
parsed: list[dict[str, object]] = []
for row in rows[header_at + 1 :]:
if columns["cbsa"] >= len(row):
continue
cbsa = str(row[columns["cbsa"]]).strip()
if not cbsa or not re.search(r"\d", cbsa):
continue
item: dict[str, object] = {
"cbsa": cbsa,
"cbsa_name": None,
"state": None,
"wage_index": None,
"reclassified_wage_index": None,
"year": year,
}
if "cbsa_name" in columns and columns["cbsa_name"] < len(row):
item["cbsa_name"] = str(row[columns["cbsa_name"]]).strip() or None
if "state" in columns and columns["state"] < len(row):
state = str(row[columns["state"]]).strip()
item["state"] = state or None
for key in ("wage_index", "reclassified_wage_index"):
if key in columns and columns[key] < len(row):
item[key] = _money(row[columns[key]])
if item["wage_index"] is None and item["reclassified_wage_index"] is None:
continue
parsed.append(item)
return pd.DataFrame(parsed)
def _pick_full_reprint(zf: zipfile.ZipFile, names: list[str], parser) -> str | None:
"""A correction notice wins only when it reprints the table, not a delta."""
if not names:
return None
scored: list[tuple[int, int, str]] = []
for name in names:
frame = parser(_read_zip_text(zf, name))
scored.append((len(frame), _cn_rank(name), name))
best = max(count for count, _, _ in scored)
if best == 0:
return None
full = [item for item in scored if item[0] >= 0.8 * best]
full.sort()
return full[-1][2]
def _wage_root(zpath: Path) -> tuple[zipfile.ZipFile, str]:
outer = zipfile.ZipFile(zpath)
nested = [name for name in outer.namelist() if name.lower().endswith(".zip")]
if not nested:
return outer, zpath.name
new = [name for name in nested if "new" in name.lower()]
chosen = new[0] if new else nested[0]
inner = zipfile.ZipFile(io.BytesIO(outer.read(chosen)))
outer.close()
return inner, chosen
def read_wage_year(zpath: Path, year: int) -> pd.DataFrame:
"""Wage index rows for one fiscal year. Empty if the zip has no index table."""
root, _source = _wage_root(zpath)
try:
grouped: dict[str, list[str]] = {kind: [] for kind in _WAGE_KINDS}
for name in root.namelist():
kind = _member_kind(name)
if kind in grouped:
grouped[kind].append(name)
if grouped["table3"]:
chosen = _pick_full_reprint(
root,
grouped["table3"],
lambda text: parse_wage_text(text, year),
)
if chosen is None:
return pd.DataFrame()
return parse_wage_text(_read_zip_text(root, chosen), year)
frames: list[pd.DataFrame] = []
for kind in ("table4a", "table4b"):
chosen = _pick_full_reprint(
root,
grouped[kind],
lambda text: parse_wage_text(text, year),
)
if chosen:
frames.append(parse_wage_text(_read_zip_text(root, chosen), year))
reclass_name = _pick_full_reprint(
root,
grouped["table4c"],
lambda text: parse_wage_text(text, year, reclassified_only=True),
)
base = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
if reclass_name is None:
return base
reclass = parse_wage_text(
_read_zip_text(root, reclass_name), year, reclassified_only=True
)
if base.empty:
return reclass
reclass = reclass.rename(columns={"reclassified_wage_index": "_reclass"})
keep = reclass[["cbsa", "state", "_reclass"]].drop_duplicates()
merged = base.merge(keep, on=["cbsa", "state"], how="left")
merged["reclassified_wage_index"] = merged["_reclass"].combine_first(
merged["reclassified_wage_index"]
)
return merged.drop(columns=["_reclass"])
finally:
root.close()
def parse_drg_text(text: str, year: int) -> pd.DataFrame:
"""Table 5. The cap-applied weight is the paid weight when both are published."""
rows = _rows(text)
header_at = None
for index, row in enumerate(rows[:12]):
norms = [_norm_header(cell) for cell in row]
if any(norm.startswith("ms-drg") for norm in norms) and any(
"weight" in norm for norm in norms
):
header_at = index
break
if header_at is None:
return pd.DataFrame()
header = [_norm_header(cell) for cell in rows[header_at]]
weight_at = next(
(
i
for i, cell in enumerate(header)
if "cap applied" in cell or "10% cap" in cell
),
None,
)
if weight_at is None:
weight_at = next(
(
i
for i, cell in enumerate(header)
if cell == "weights" or cell.endswith(" weights")
),
None,
)
drg_at = next(
(i for i, cell in enumerate(header) if cell.startswith("ms-drg")), None
)
geo_at = next((i for i, cell in enumerate(header) if "geometric" in cell), None)
ari_at = next((i for i, cell in enumerate(header) if "arithmetic" in cell), None)
if weight_at is None or drg_at is None:
return pd.DataFrame()
parsed: list[dict[str, object]] = []
for row in rows[header_at + 1 :]:
if drg_at >= len(row) or weight_at >= len(row):
continue
code = str(row[drg_at]).strip()
if not code.isdigit():
continue
weight = _money(row[weight_at])
if weight is None:
continue
parsed.append(
{
"ms_drg": code.zfill(3),
"relative_weight": weight,
"geometric_mean_los": _money(row[geo_at])
if geo_at is not None and geo_at < len(row)
else None,
"arithmetic_mean_los": _money(row[ari_at])
if ari_at is not None and ari_at < len(row)
else None,
"fiscal_year": year,
}
)
return pd.DataFrame(parsed)
def read_drg_year(zpath: Path, year: int) -> pd.DataFrame:
with zipfile.ZipFile(zpath) as zf:
names = [name for name in zf.namelist() if _member_kind(name) == "table5"]
chosen = _pick_full_reprint(zf, names, lambda text: parse_drg_text(text, year))
if chosen is None:
return pd.DataFrame()
return parse_drg_text(_read_zip_text(zf, chosen), year)
def _hospital_band(text: str) -> str:
low = text.lower()
if "greater than 1" in low:
wage = "wi_gt_1"
elif "less than or equal" in low:
wage = "wi_le_1"
else:
wage = ""
if "not submit quality" in low or "did not submit quality" in low:
quality = "no_quality"
elif "quality" in low:
quality = "quality"
else:
quality = ""
if "not a meaningful" in low:
ehr = "no_ehr"
elif "meaningful" in low or "ehr" in low:
ehr = "ehr"
else:
ehr = ""
if not quality and not ehr and not wage:
if "full update" in low:
return "full_update"
if "reduced update" in low:
return "reduced_update"
return "_".join(part for part in (quality, ehr, wage) if part)
def _share_label(text: str) -> str:
low = _norm_header(text)
if "nonlabor" in low:
return "nonlabor"
if low == "labor" or low.startswith("labor"):
return "labor"
if low.startswith("full update"):
return "full_update"
if "reduced update" in low:
return "reduced_update"
if low == "rate":
return "rate"
return ""
def _row_label(text: str) -> str:
low = _norm_header(text)
if low.startswith("national"):
return "national"
if "puerto rico" in low:
return "puerto_rico"
if "standard federal" in low:
return "standard_federal_rate"
return ""
def _pick_tables(candidates: list[tuple[int, int, str, pd.DataFrame]]) -> pd.DataFrame:
"""One frame per table code. A short correction sheet does not replace a full one."""
by_code: dict[str, list[tuple[int, int, str, pd.DataFrame]]] = {}
for rank, _count, title, frame in candidates:
if frame.empty or "table_code" not in frame.columns:
continue
for code, part in frame.groupby("table_code", sort=False):
by_code.setdefault(str(code), []).append((len(part), rank, title, part))
chosen: list[pd.DataFrame] = []
for parts in by_code.values():
best = max(item[0] for item in parts)
full = [item for item in parts if item[0] >= 0.8 * best]
full.sort()
chosen.append(full[-1][3])
if not chosen:
return pd.DataFrame()
return pd.concat(chosen, ignore_index=True)
def parse_standardized_xlsx(data: bytes, year: int) -> pd.DataFrame:
"""Each dollar in Tables 1A-1E. 'Not Applicable' is not a number.
A workbook often has a final-rule sheet and a correction sheet. The
correction sheet is kept only when it reprints that table. A table
the correction sheet omits (FY2015 Table 1E) stays from the final rule.
"""
book = openpyxl.load_workbook(io.BytesIO(data), data_only=True)
try:
parsed: list[tuple[int, int, str, pd.DataFrame]] = []
for sheet in book.worksheets:
grid = [[cell for cell in row] for row in sheet.iter_rows(values_only=True)]
frame = _parse_standardized_grid(grid, year)
parsed.append((_cn_rank(sheet.title), len(frame), sheet.title, frame))
finally:
book.close()
if sum(1 for _rank, count, _title, _frame in parsed if count) > 1:
for rank, count, title, _frame in parsed:
print(f" sheet {title!r} cn={rank} rows={count}")
return _pick_tables(parsed)
def _parse_standardized_grid(grid: list[list[object]], year: int) -> pd.DataFrame:
"""One worksheet of Tables 1A-1E."""
table = ""
parsed: list[dict[str, object]] = []
for index, row in enumerate(grid):
joined = " ".join(str(cell) for cell in row if cell is not None)
found = re.search(r"TABLE\s+1([A-E])\b", joined, flags=re.I)
if found:
table = f"1{found.group(1).upper()}"
continue
if not table:
continue
amounts = [_money(cell) for cell in row]
if not any(amount is not None for amount in amounts):
continue
# The label row is the nearest prior row that names labor/nonlabor/rate/update.
share_row = None
band_row = None
for look in range(index - 1, max(-1, index - 4), -1):
labels = [_share_label(str(cell or "")) for cell in grid[look]]
if any(labels):
share_row = labels
if look > 0:
band_row = [
_hospital_band(str(cell or "")) for cell in grid[look - 1]
]
break
if share_row is None:
continue
label = ""
for cell, amount in zip(row, amounts, strict=False):
if amount is None and _row_label(str(cell or "")):
label = _row_label(str(cell or ""))
break
for col, amount in enumerate(amounts):
if amount is None:
continue
share = share_row[col] if col < len(share_row) else ""
if not share:
for left in range(col, -1, -1):
if left < len(share_row) and share_row[left]:
share = share_row[left]
break
band = ""
if band_row:
for left in range(col, -1, -1):
if left < len(band_row) and band_row[left]:
band = band_row[left]
break
parts = [part for part in (label, band, share) if part]
if not parts:
continue
parsed.append(
{
"fiscal_year": year,
"table_code": table,
"component": "_".join(parts),
"amount": amount,
}
)
return pd.DataFrame(parsed)
def read_standardized_year(zpath: Path, year: int) -> pd.DataFrame:
with zipfile.ZipFile(zpath) as zf:
books = [name for name in zf.namelist() if name.lower().endswith(".xlsx")]
if not books:
return pd.DataFrame()
books.sort(key=lambda name: (_cn_rank(name), name))
return parse_standardized_xlsx(zf.read(books[-1]), year)
def _year_from_dirname(path: Path) -> int | None:
match = re.fullmatch(r"fy(\d{4})", path.name)
return int(match.group(1)) if match else None
def read_ipps_dir(root: Path = IPPS_DIR) -> dict[str, pd.DataFrame]:
"""Read every downloaded fiscal year. A missing file is skipped."""
wages: list[pd.DataFrame] = []
drgs: list[pd.DataFrame] = []
amounts: list[pd.DataFrame] = []
for year_dir in sorted(root.glob("fy*")):
year = _year_from_dirname(year_dir)
if year is None:
continue
wage_zip = next(year_dir.glob("wage--*.zip"), None)
drg_zip = next(year_dir.glob("drg--*.zip"), None)
amount_zip = next(year_dir.glob("standardized--*.zip"), None)
if wage_zip is not None:
frame = read_wage_year(wage_zip, year)
print(f" FY{year} wage {len(frame)} from {wage_zip.name}")
if not frame.empty:
wages.append(frame)
if drg_zip is not None:
frame = read_drg_year(drg_zip, year)
print(f" FY{year} drg {len(frame)} from {drg_zip.name}")
if not frame.empty:
drgs.append(frame)
if amount_zip is not None:
frame = read_standardized_year(amount_zip, year)
print(f" FY{year} amounts {len(frame)} from {amount_zip.name}")
if not frame.empty:
amounts.append(frame)
return {
"wage": pd.concat(wages, ignore_index=True) if wages else pd.DataFrame(),
"drg": pd.concat(drgs, ignore_index=True) if drgs else pd.DataFrame(),
"amount": pd.concat(amounts, ignore_index=True) if amounts else pd.DataFrame(),
}
_MSSP_MONEY = (
"earnsaveloss",
"gensaveloss",
"abtotbnchmk",
"abtotexp",
"updatedbnchmk",
"histbnchmk",
"sav_rate",
"qualscore",
"finalsharerate",
)
_REACH_MONEY = (
"shared_savings",
"gross_savings",
"final_benchmark",
"unadj_benchmark",
"tot_cost_care",
"tot_cost_care_net",
"sav_rate",
)
def _perf_year_from_filename(name: str) -> int | None:
"""First calendar year in the file name. ``%20`` is a space, not 2020."""
match = re.search(r"(20\d{2})", unquote(name))
return int(match.group(1)) if match else None
def read_settlement_dir(
directory: Path, money: tuple[str, ...], *, year_column: str
) -> pd.DataFrame:
"""Every catalog CSV in one directory. The filename's first year is the performance year."""
from cms.datacms import sanitize_column_name
frames: list[pd.DataFrame] = []
for path in sorted(directory.glob("*.csv")):
frame = pd.read_csv(
path, dtype=str, encoding="utf-8", encoding_errors="replace"
)
frame.columns = [sanitize_column_name(column) for column in frame.columns]
year = _perf_year_from_filename(path.name)
if year_column in frame.columns:
parsed = pd.to_numeric(frame[year_column], errors="coerce")
frame[year_column] = parsed.round().astype("Int64").astype(str)
frame.loc[parsed.isna(), year_column] = str(year) if year else pd.NA
elif year is not None:
frame[year_column] = str(year)
frame["source_file"] = path.name
for column in frame.columns:
if column in money:
frame[column] = pd.to_numeric(frame[column], errors="coerce")
else:
frame[column] = frame[column].astype("string")
frames.append(frame)
print(f" {path.name}: {len(frame)} rows year {year}")
if not frames:
return pd.DataFrame()
return pd.concat(frames, ignore_index=True)
def _create_typed(
con, table: str, frame: pd.DataFrame, doubles: set[str], integers: set[str]
) -> None:
"""Load a frame with explicit types so an id column is not inferred as a measure."""
con.register("_src", frame)
try:
casts = []
for column in frame.columns:
if column in doubles:
kind = "DOUBLE"
elif column in integers:
kind = "INTEGER"
else:
kind = "VARCHAR"
quoted = '"' + column.replace('"', '""') + '"'
casts.append(f"CAST({quoted} AS {kind}) AS {quoted}")
con.execute(f"CREATE TABLE {table} AS SELECT {', '.join(casts)} FROM _src")
finally:
con.unregister("_src")
def load_into(con) -> None:
"""Replace the IPPS and settlement tables. Other schemas are left alone."""
tables = read_ipps_dir()
con.execute("CREATE SCHEMA IF NOT EXISTS opps")
con.execute("CREATE SCHEMA IF NOT EXISTS ipps")
con.execute("CREATE SCHEMA IF NOT EXISTS cms")
if not tables["wage"].empty:
con.execute("DROP TABLE IF EXISTS opps.wage_index")
_create_typed(
con,
"opps.wage_index",
tables["wage"],
{"wage_index", "reclassified_wage_index"},
{"year"},
)
if not tables["drg"].empty:
con.execute("DROP TABLE IF EXISTS ipps.drg_weight")
_create_typed(
con,
"ipps.drg_weight",
tables["drg"],
{"relative_weight", "geometric_mean_los", "arithmetic_mean_los"},
{"fiscal_year"},
)
if not tables["amount"].empty:
con.execute("DROP TABLE IF EXISTS ipps.standardized_amount")
_create_typed(
con,
"ipps.standardized_amount",
tables["amount"],
{"amount"},
{"fiscal_year"},
)
mssp = read_settlement_dir(MSSP_DIR, _MSSP_MONEY, year_column="perf_year")
reach = read_settlement_dir(REACH_DIR, _REACH_MONEY, year_column="perf_year")
if not mssp.empty:
con.execute(
"DROP TABLE IF EXISTS cms.performance_year_financial_quality_results"
)
_create_typed(
con,
"cms.performance_year_financial_quality_results",
mssp,
set(_MSSP_MONEY),
set(),
)
if not reach.empty:
con.execute("DROP TABLE IF EXISTS cms.aco_reach_financial_quality_results")
_create_typed(
con,
"cms.aco_reach_financial_quality_results",
reach,
set(_REACH_MONEY),
set(),
)
def main() -> None:
from conf.connect import duckdb_batch, publish_replica
print("Reading published tables ...")
with duckdb_batch("aco") as con:
load_into(con)
for schema, table in (
("opps", "wage_index"),
("ipps", "drg_weight"),
("ipps", "standardized_amount"),
("cms", "performance_year_financial_quality_results"),
("cms", "aco_reach_financial_quality_results"),
):
count = con.execute(f"SELECT count(*) FROM {schema}.{table}").fetchone()[0]
print(f" {schema}.{table}: {count}")
replica = publish_replica("aco")
print("replica", replica)
if __name__ == "__main__":
main()