Some checks failed
CI / lint (push) Failing after 49s
CI / notebooks-smoke (push) Successful in 2m5s
Deploy / changes (push) Successful in 17s
Infra CI / notebooks (push) Successful in 57s
Infra CI / zotero (push) Successful in 16s
Infra CI / docs (push) Successful in 26s
CI / test (push) Successful in 5m11s
Infra CI / api (push) Successful in 1m20s
Infra CI / bcda (push) Successful in 1m53s
Infra CI / mc (push) Successful in 17s
Infra CI / llm (push) Successful in 1m6s
Deploy / notebooks (push) Has been skipped
Lean Pay / pay (push) Successful in 14s
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m40s
Deploy / bcda (push) Successful in 1m47s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 59s
Deploy / report (push) Successful in 13s
Area capital GAF comes from Table 3, or Tables 4A and 4B when that year has no Table 3. National factors come from MAC Implementation File 1. Hospital IME, DSH, CCR, and capital GAF come from the final-rule impact file. The claim payment and the Lean ceiling stay absent. The impact file is not the quarterly Provider Specific File. The operating federal rate and the GAF=1 Table 1D capital amount are unchanged.
824 lines
27 KiB
Python
824 lines
27 KiB
Python
"""Load published IPPS payment factors. Does not price a claim.
|
||
|
||
Three grains, each from the file CMS published for that grain:
|
||
|
||
* Area capital GAF from the wage-index zip already on disk. Table 3,
|
||
or Tables 4A and 4B when that year has no Table 3, plus the
|
||
reclassified GAF from Table 4C. The Puerto Rico GAF column stays out.
|
||
* National factors from MAC Implementation File 1, for the years CMS
|
||
posted that file (FY2018–FY2027). FY2014–FY2017 have no file.
|
||
* Hospital factors from the final-rule impact file. That file models
|
||
the rule. It is not the quarterly Provider Specific File.
|
||
|
||
The operating federal rate and the Table 1D capital amount are left
|
||
where the other ingest put them. This script does not call that ingest
|
||
and does not drop those tables.
|
||
|
||
A correction notice replaces a final rule only when it reprints the
|
||
table. Among full impact-file reprints, the latest month named in the
|
||
path wins, and a higher correction-notice number wins when no month is
|
||
named. January 2014 therefore beats the September 2013 notice even
|
||
though that notice's CN number is higher.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import importlib.util
|
||
import io
|
||
import re
|
||
import sys
|
||
import zipfile
|
||
from pathlib import Path
|
||
|
||
import openpyxl
|
||
import pandas as pd
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
IPPS_DIR = ROOT / "data" / "cms" / "ipps"
|
||
FACTOR_DIR = IPPS_DIR / "factors"
|
||
|
||
_GAF_COLUMNS = (
|
||
"fiscal_year",
|
||
"cbsa",
|
||
"state",
|
||
"cbsa_name",
|
||
"gaf",
|
||
"reclassified_gaf",
|
||
)
|
||
_NATIONAL_COLUMNS = ("fiscal_year", "factor", "area", "amount")
|
||
_PROVIDER_COLUMNS = (
|
||
"fiscal_year",
|
||
"provider",
|
||
"payment_wage_index",
|
||
"cola",
|
||
"resident_to_bed",
|
||
"resident_to_adc",
|
||
"operating_ime",
|
||
"capital_ime",
|
||
"dsh_patient_pct",
|
||
"operating_dsh",
|
||
"capital_dsh",
|
||
"ucp_per_claim",
|
||
"operating_ccr",
|
||
"capital_ccr",
|
||
"gaf",
|
||
"capital_cola",
|
||
)
|
||
_PROVIDER_DOUBLES = set(_PROVIDER_COLUMNS) - {"fiscal_year", "provider"}
|
||
|
||
_MONTH_NUMBER = {
|
||
"jan": 1,
|
||
"january": 1,
|
||
"feb": 2,
|
||
"february": 2,
|
||
"mar": 3,
|
||
"march": 3,
|
||
"apr": 4,
|
||
"april": 4,
|
||
"may": 5,
|
||
"jun": 6,
|
||
"june": 6,
|
||
"jul": 7,
|
||
"july": 7,
|
||
"aug": 8,
|
||
"august": 8,
|
||
"sep": 9,
|
||
"sept": 9,
|
||
"september": 9,
|
||
"oct": 10,
|
||
"october": 10,
|
||
"nov": 11,
|
||
"november": 11,
|
||
"dec": 12,
|
||
"december": 12,
|
||
}
|
||
_MONTH_RE = re.compile(
|
||
r"(?<![a-z])("
|
||
r"january|february|march|april|june|july|august|september|"
|
||
r"jan|feb|mar|apr|may|jun|jul|aug|sept|sep|oct|nov|dec"
|
||
r")\s+(\d{4})"
|
||
)
|
||
_LY_RE = re.compile(r"(^|[^a-z])ly([^a-z]|$)")
|
||
_QUALITY_ONE = re.compile(r"quality\s*=\s*.{0,3}1")
|
||
_QUALITY_ZERO = re.compile(r"quality\s*=\s*.{0,3}0")
|
||
_EHR_YES = re.compile(r"ehr\s*=\s*.{0,3}y")
|
||
|
||
|
||
def _tables():
|
||
"""Wage-index ingest. Same reprint rule, same header cleanup."""
|
||
name = "ingest_ipps_tables"
|
||
loaded = sys.modules.get(name)
|
||
if loaded is not None and hasattr(loaded, "_wage_root"):
|
||
return loaded
|
||
path = Path(__file__).with_name("ingest_ipps_tables.py")
|
||
spec = importlib.util.spec_from_file_location(name, path)
|
||
if spec is None or spec.loader is None:
|
||
raise ImportError(path)
|
||
module = importlib.util.module_from_spec(spec)
|
||
sys.modules[name] = module
|
||
spec.loader.exec_module(module)
|
||
return module
|
||
|
||
|
||
def _money(value: object) -> float | None:
|
||
"""Sibling money parse, rounded so a binary 0.34 stays 0.34."""
|
||
number = _tables()._money(value)
|
||
if number is None:
|
||
return None
|
||
return round(number, 10)
|
||
|
||
|
||
def _empty(columns: tuple[str, ...]) -> pd.DataFrame:
|
||
return pd.DataFrame(columns=list(columns))
|
||
|
||
|
||
def _require_unique(frame: pd.DataFrame, keys: list[str], label: str) -> pd.DataFrame:
|
||
if frame.empty:
|
||
return frame
|
||
collapsed = frame.drop_duplicates()
|
||
if collapsed.duplicated(keys).any():
|
||
clash = collapsed.loc[collapsed.duplicated(keys, keep=False), keys]
|
||
sample = clash.head(5).to_dict("records")
|
||
raise ValueError(f"{label} has conflicting rows for {sample}")
|
||
return collapsed.drop_duplicates(keys)
|
||
|
||
|
||
def _is_reclassified_gaf(norm: str) -> bool:
|
||
if "puerto" in norm:
|
||
return False
|
||
return "reclassified" in norm and "gaf" in norm
|
||
|
||
|
||
def _is_plain_gaf(norm: str) -> bool:
|
||
"""National GAF column. Not Puerto Rico and not the reclassified column."""
|
||
if "reclassified" in norm or "puerto" in norm:
|
||
return False
|
||
return norm == "gaf" or norm.startswith("gaf ") or " gaf" in f" {norm}"
|
||
|
||
|
||
def _gaf_columns(header: list[str], *, reclassified_only: bool) -> dict[str, int]:
|
||
tables = _tables()
|
||
found: dict[str, int] = {}
|
||
for index, cell in enumerate(header):
|
||
norm = tables._norm_header(cell)
|
||
if norm in {"cbsa", "cbsa code"} and "cbsa" not in found:
|
||
found["cbsa"] = index
|
||
elif norm in {
|
||
"area name",
|
||
"urban area",
|
||
"rural area",
|
||
"nonurban area",
|
||
"area",
|
||
"state or urban area",
|
||
}:
|
||
found["cbsa_name"] = index
|
||
elif norm in {"state", "state*"} or (
|
||
norm.startswith("state") and "code" not in norm
|
||
):
|
||
found.setdefault("state", index)
|
||
elif _is_reclassified_gaf(norm):
|
||
found["reclassified_gaf"] = index
|
||
elif _is_plain_gaf(norm):
|
||
# Table 4C labels the reclassified GAF as "GAF".
|
||
if reclassified_only:
|
||
found.setdefault("reclassified_gaf", index)
|
||
else:
|
||
found.setdefault("gaf", index)
|
||
return found
|
||
|
||
|
||
def parse_gaf_text(
|
||
text: str, year: int, *, reclassified_only: bool = False
|
||
) -> pd.DataFrame:
|
||
"""One Table 3 or Table 4A/4B/4C text file."""
|
||
tables = _tables()
|
||
rows = tables._rows(text)
|
||
header_at = tables._header_index(rows, "gaf", "cbsa")
|
||
if header_at is None:
|
||
return _empty(_GAF_COLUMNS)
|
||
columns = _gaf_columns(rows[header_at], reclassified_only=reclassified_only)
|
||
if "cbsa" not in columns:
|
||
return _empty(_GAF_COLUMNS)
|
||
if "gaf" not in columns and "reclassified_gaf" not in columns:
|
||
return _empty(_GAF_COLUMNS)
|
||
parsed: list[dict[str, object]] = []
|
||
for row in rows[header_at + 1 :]:
|
||
if columns["cbsa"] >= len(row):
|
||
continue
|
||
cbsa = str(row[columns["cbsa"]]).strip()
|
||
if not cbsa or not re.search(r"\d", cbsa):
|
||
continue
|
||
item: dict[str, object] = {
|
||
"fiscal_year": year,
|
||
"cbsa": cbsa,
|
||
"state": None,
|
||
"cbsa_name": None,
|
||
"gaf": None,
|
||
"reclassified_gaf": None,
|
||
}
|
||
if "cbsa_name" in columns and columns["cbsa_name"] < len(row):
|
||
item["cbsa_name"] = str(row[columns["cbsa_name"]]).strip() or None
|
||
if "state" in columns and columns["state"] < len(row):
|
||
state = str(row[columns["state"]]).strip()
|
||
item["state"] = state or None
|
||
for key in ("gaf", "reclassified_gaf"):
|
||
if key in columns and columns[key] < len(row):
|
||
item[key] = _money(row[columns[key]])
|
||
if item["gaf"] is None and item["reclassified_gaf"] is None:
|
||
continue
|
||
parsed.append(item)
|
||
if not parsed:
|
||
return _empty(_GAF_COLUMNS)
|
||
return pd.DataFrame(parsed, columns=list(_GAF_COLUMNS))
|
||
|
||
|
||
def read_gaf_year(zpath: Path, year: int) -> pd.DataFrame:
|
||
"""Area GAF for one fiscal year. Empty when the zip has no GAF table."""
|
||
tables = _tables()
|
||
root, _source = tables._wage_root(zpath)
|
||
try:
|
||
grouped: dict[str, list[str]] = {kind: [] for kind in tables._WAGE_KINDS}
|
||
for name in root.namelist():
|
||
kind = tables._member_kind(name)
|
||
if kind in grouped:
|
||
grouped[kind].append(name)
|
||
if grouped["table3"]:
|
||
chosen = tables._pick_full_reprint(
|
||
root,
|
||
grouped["table3"],
|
||
lambda text: parse_gaf_text(text, year),
|
||
)
|
||
if chosen is None:
|
||
return _empty(_GAF_COLUMNS)
|
||
return parse_gaf_text(tables._read_zip_text(root, chosen), year)
|
||
|
||
frames: list[pd.DataFrame] = []
|
||
for kind in ("table4a", "table4b"):
|
||
chosen = tables._pick_full_reprint(
|
||
root,
|
||
grouped[kind],
|
||
lambda text: parse_gaf_text(text, year),
|
||
)
|
||
if chosen:
|
||
frames.append(parse_gaf_text(tables._read_zip_text(root, chosen), year))
|
||
reclass_name = tables._pick_full_reprint(
|
||
root,
|
||
grouped["table4c"],
|
||
lambda text: parse_gaf_text(text, year, reclassified_only=True),
|
||
)
|
||
base = pd.concat(frames, ignore_index=True) if frames else _empty(_GAF_COLUMNS)
|
||
if reclass_name is None:
|
||
return base
|
||
reclass = parse_gaf_text(
|
||
tables._read_zip_text(root, reclass_name),
|
||
year,
|
||
reclassified_only=True,
|
||
)
|
||
if base.empty:
|
||
return reclass
|
||
reclass = reclass.rename(columns={"reclassified_gaf": "_reclass"})
|
||
keep = reclass[["cbsa", "state", "_reclass"]].drop_duplicates()
|
||
merged = base.merge(keep, on=["cbsa", "state"], how="left")
|
||
merged["reclassified_gaf"] = merged["_reclass"].combine_first(
|
||
merged["reclassified_gaf"]
|
||
)
|
||
return merged.drop(columns=["_reclass"])
|
||
finally:
|
||
root.close()
|
||
|
||
|
||
def _update_factor(label: str) -> str | None:
|
||
"""Quality and EHR band. EHR = Y is the penalty, not the bonus."""
|
||
low = label.lower().replace("’", "'").replace("‘", "'")
|
||
if "quality" not in low or "ehr" not in low:
|
||
return None
|
||
quality_one = _QUALITY_ONE.search(low) is not None
|
||
quality_zero = _QUALITY_ZERO.search(low) is not None
|
||
ehr_yes = _EHR_YES.search(low) is not None
|
||
ehr_blank = "blank" in low
|
||
if quality_one and ehr_blank:
|
||
name = "update_quality_ehr"
|
||
elif quality_zero and ehr_blank:
|
||
name = "update_no_quality"
|
||
elif quality_one and ehr_yes:
|
||
name = "update_quality_no_ehr"
|
||
elif quality_zero and ehr_yes:
|
||
name = "update_neither"
|
||
else:
|
||
return None
|
||
if "puerto rico" in low:
|
||
return "pr_" + name
|
||
return name
|
||
|
||
|
||
def _scalar_factor(label: str) -> str | None:
|
||
"""One national factor. The year inside the label is not part of the id."""
|
||
low = _tables()._norm_header(label)
|
||
low = re.sub(r"\bfy\s*\d{4}\b", " ", low)
|
||
low = re.sub(r"\b20\d{2}\b", " ", low)
|
||
low = re.sub(r"\s+", " ", low).strip()
|
||
if "fixed loss" in low and "threshold" in low:
|
||
return "fixed_loss_threshold"
|
||
if "cap policy" in low and "budget neutrality" in low:
|
||
return "weight_cap_budget_neutrality"
|
||
if "budget neutrality" in low and "drg" in low:
|
||
return "ms_drg_budget_neutrality"
|
||
if "ucp" in low and "ccr" in low and "ceiling" in low:
|
||
return "ucp_ccr_ceiling"
|
||
if "operating" in low and "ccr" in low and "ceiling" in low:
|
||
return "operating_ccr_ceiling"
|
||
if "capital" in low and "ccr" in low and "ceiling" in low:
|
||
return "capital_ccr_ceiling"
|
||
if "scaling factor" in low:
|
||
return "ucp_scaling_factor"
|
||
return None
|
||
|
||
|
||
def _cola_area(label: str) -> str | None:
|
||
low = label.lower()
|
||
if "anchorage" in low:
|
||
return "anchorage"
|
||
if "fairbanks" in low:
|
||
return "fairbanks"
|
||
if "juneau" in low:
|
||
return "juneau"
|
||
if "rest of alaska" in low:
|
||
return "rest_of_alaska"
|
||
if "honolulu" in low:
|
||
return "honolulu"
|
||
if "kauai" in low:
|
||
return "kauai"
|
||
if "maui" in low or "kalawao" in low:
|
||
return "maui_kalawao"
|
||
if "hawaii" in low and "county" in low:
|
||
return "hawaii_county"
|
||
return None
|
||
|
||
|
||
def _labor_band(label: str) -> str | None:
|
||
low = label.lower()
|
||
if "wage index" not in low:
|
||
return None
|
||
if "greater than 1" in low:
|
||
return "gt1"
|
||
if "less than or equal" in low:
|
||
return "le1"
|
||
return None
|
||
|
||
|
||
def _first_text(row: list[object]) -> str | None:
|
||
for cell in row:
|
||
if cell is None:
|
||
continue
|
||
text = str(cell).strip()
|
||
if text:
|
||
return text
|
||
return None
|
||
|
||
|
||
def _first_money(row: list[object], start: int = 0) -> float | None:
|
||
for cell in row[start:]:
|
||
amount = _money(cell)
|
||
if amount is not None:
|
||
return amount
|
||
return None
|
||
|
||
|
||
def _labor_header(row: list[object]) -> tuple[int, int] | None:
|
||
labor = None
|
||
nonlabor = None
|
||
for index, cell in enumerate(row):
|
||
norm = _tables()._norm_header(cell)
|
||
if norm == "labor share":
|
||
labor = index
|
||
elif "non-labor" in norm or "nonlabor" in norm:
|
||
nonlabor = index
|
||
if labor is None or nonlabor is None:
|
||
return None
|
||
return labor, nonlabor
|
||
|
||
|
||
def _sheet_rows(sheet) -> list[list[object]]:
|
||
return [list(row) for row in sheet.iter_rows(values_only=True)]
|
||
|
||
|
||
def _parse_esrd(grid: list[list[object]], year: int) -> list[dict[str, object]]:
|
||
"""The rate cell. A 'forthcoming' sentence is not a number."""
|
||
parsed: list[dict[str, object]] = []
|
||
for row in grid:
|
||
amount = _first_money(row)
|
||
if amount is None:
|
||
continue
|
||
parsed.append(
|
||
{
|
||
"fiscal_year": year,
|
||
"factor": "esrd_base_rate",
|
||
"area": "",
|
||
"amount": amount,
|
||
}
|
||
)
|
||
return parsed
|
||
|
||
|
||
def _parse_factor_sheet(
|
||
grid: list[list[object]], year: int, *, sheet_title: str
|
||
) -> list[dict[str, object]]:
|
||
parsed: list[dict[str, object]] = []
|
||
labor_at: tuple[int, int] | None = None
|
||
pending: str | None = None
|
||
cola_on = "cola" in sheet_title.lower()
|
||
|
||
def emit(factor: str, amount: float | None, area: str = "") -> None:
|
||
if amount is None:
|
||
return
|
||
parsed.append(
|
||
{
|
||
"fiscal_year": year,
|
||
"factor": factor,
|
||
"area": area,
|
||
"amount": amount,
|
||
}
|
||
)
|
||
|
||
for row in grid:
|
||
header = _labor_header(row)
|
||
if header is not None:
|
||
labor_at = header
|
||
pending = None
|
||
continue
|
||
label = _first_text(row)
|
||
if label is None:
|
||
continue
|
||
low = label.lower()
|
||
if "cost of living" in low or "cost-of-living" in low:
|
||
cola_on = True
|
||
band = _labor_band(label) if labor_at is not None else None
|
||
if band is not None:
|
||
labor_col, nonlabor_col = labor_at
|
||
emit(f"labor_share_{band}", _money(_cell(row, labor_col)))
|
||
emit(f"nonlabor_share_{band}", _money(_cell(row, nonlabor_col)))
|
||
pending = None
|
||
continue
|
||
update = _update_factor(label)
|
||
if update is not None:
|
||
emit(update, _first_money(row, 1))
|
||
pending = None
|
||
continue
|
||
area = _cola_area(label) if cola_on else None
|
||
if area is not None:
|
||
emit("cola", _first_money(row, 1), area)
|
||
pending = None
|
||
continue
|
||
scalar = _scalar_factor(label)
|
||
if scalar is not None:
|
||
amount = _first_money(row, 1)
|
||
if amount is None:
|
||
pending = scalar
|
||
else:
|
||
emit(scalar, amount)
|
||
pending = None
|
||
continue
|
||
if pending is not None:
|
||
emit(pending, _first_money(row, 0))
|
||
pending = None
|
||
return parsed
|
||
|
||
|
||
def _cell(row: list[object], index: int) -> object:
|
||
if index >= len(row):
|
||
return None
|
||
return row[index]
|
||
|
||
|
||
def parse_mac_xlsx(data: bytes, year: int) -> pd.DataFrame:
|
||
"""MAC Implementation File 1 workbook. The xlsx, not the text export."""
|
||
book = openpyxl.load_workbook(io.BytesIO(data), data_only=True)
|
||
try:
|
||
parsed: list[dict[str, object]] = []
|
||
for sheet in book.worksheets:
|
||
grid = _sheet_rows(sheet)
|
||
if "esrd" in sheet.title.lower():
|
||
parsed.extend(_parse_esrd(grid, year))
|
||
else:
|
||
parsed.extend(_parse_factor_sheet(grid, year, sheet_title=sheet.title))
|
||
finally:
|
||
book.close()
|
||
if not parsed:
|
||
return _empty(_NATIONAL_COLUMNS)
|
||
return pd.DataFrame(parsed, columns=list(_NATIONAL_COLUMNS))
|
||
|
||
|
||
def read_mac_year(zpath: Path, year: int) -> pd.DataFrame:
|
||
with zipfile.ZipFile(zpath) as zf:
|
||
books = [name for name in zf.namelist() if name.lower().endswith(".xlsx")]
|
||
if not books:
|
||
return _empty(_NATIONAL_COLUMNS)
|
||
books.sort(
|
||
key=lambda name: ("factor" not in name.lower(), -zf.getinfo(name).file_size)
|
||
)
|
||
return parse_mac_xlsx(zf.read(books[0]), year)
|
||
|
||
|
||
def _impact_field(norm: str) -> str | None:
|
||
"""Map one impact-file header. Last-year and Puerto Rico columns drop out."""
|
||
if "provider number" in norm:
|
||
return "provider"
|
||
if "puerto" in norm:
|
||
return None
|
||
if _LY_RE.search(norm) or "last year" in norm:
|
||
return None
|
||
if "505" in norm or "reclass" in norm:
|
||
return None
|
||
if "capital" in norm and ("cost of living" in norm or "cola" in norm):
|
||
return "capital_cola"
|
||
if "cost of living" in norm:
|
||
return "cola"
|
||
if "wage index" in norm:
|
||
return "payment_wage_index"
|
||
if norm == "rday" or "resident to average daily census" in norm:
|
||
return "resident_to_adc"
|
||
if "resident to bed" in norm:
|
||
return "resident_to_bed"
|
||
if norm == "tchcp" or norm.startswith("tchcp"):
|
||
return "capital_ime"
|
||
if norm == "tchop" or norm.startswith("tchop"):
|
||
return "operating_ime"
|
||
if norm == "dshpct" or "disproportionate patient percentage" in norm:
|
||
return "dsh_patient_pct"
|
||
if norm in {"dshcpg", "dshcpp"}:
|
||
return "capital_dsh"
|
||
if norm in {"dshopg", "dshopp"}:
|
||
return "operating_dsh"
|
||
if "per claim" in norm and "ucp" in norm:
|
||
return "ucp_per_claim"
|
||
if "capital" in norm and "ccr" in norm:
|
||
return "capital_ccr"
|
||
if "operating" in norm and "ccr" in norm:
|
||
return "operating_ccr"
|
||
if "gaf" in norm:
|
||
return "gaf"
|
||
return None
|
||
|
||
|
||
def _impact_columns(header: list[str]) -> dict[str, int]:
|
||
found: dict[str, int] = {}
|
||
for index, cell in enumerate(header):
|
||
key = _impact_field(_tables()._norm_header(cell))
|
||
if key and key not in found:
|
||
found[key] = index
|
||
return found
|
||
|
||
|
||
def _provider(value: object) -> str | None:
|
||
text = str(value or "").strip()
|
||
if text.isdigit() and 0 < len(text) <= 6:
|
||
return text.zfill(6)
|
||
return None
|
||
|
||
|
||
def parse_impact_text(text: str, year: int) -> pd.DataFrame:
|
||
"""One impact-file reprint. The header row is the one that names the provider."""
|
||
rows = _tables()._rows(text)
|
||
header_at = None
|
||
for index, row in enumerate(rows[:8]):
|
||
norms = [_tables()._norm_header(cell) for cell in row]
|
||
if any("provider number" in norm for norm in norms):
|
||
header_at = index
|
||
break
|
||
if header_at is None:
|
||
return _empty(_PROVIDER_COLUMNS)
|
||
columns = _impact_columns(rows[header_at])
|
||
if "provider" not in columns:
|
||
return _empty(_PROVIDER_COLUMNS)
|
||
parsed: list[dict[str, object]] = []
|
||
for row in rows[header_at + 1 :]:
|
||
if columns["provider"] >= len(row):
|
||
continue
|
||
provider = _provider(row[columns["provider"]])
|
||
if provider is None:
|
||
continue
|
||
item: dict[str, object] = {column: None for column in _PROVIDER_COLUMNS}
|
||
item["fiscal_year"] = year
|
||
item["provider"] = provider
|
||
for key, index in columns.items():
|
||
if key == "provider" or index >= len(row):
|
||
continue
|
||
item[key] = _money(row[index])
|
||
parsed.append(item)
|
||
if not parsed:
|
||
return _empty(_PROVIDER_COLUMNS)
|
||
return pd.DataFrame(parsed, columns=list(_PROVIDER_COLUMNS))
|
||
|
||
|
||
def _reprint_date(path: str) -> tuple[int, int]:
|
||
"""Latest month-year named in the path. (0, 0) when the name has none."""
|
||
found: list[tuple[int, int]] = []
|
||
for match in _MONTH_RE.finditer(path.lower()):
|
||
month = _MONTH_NUMBER[match.group(1)]
|
||
found.append((int(match.group(2)), month))
|
||
if not found:
|
||
return (0, 0)
|
||
return max(found)
|
||
|
||
|
||
def _path_cn_rank(path: str) -> int:
|
||
ranks = [_tables()._cn_rank(part) for part in re.split(r"[\\/]", path)]
|
||
return max(ranks) if ranks else 0
|
||
|
||
|
||
def _impact_members(zpath: Path) -> list[tuple[str, str]]:
|
||
"""Text reprints, one nested zip deep. Descriptions are not reprints."""
|
||
found: list[tuple[str, str]] = []
|
||
|
||
def take(path: str, raw: bytes) -> None:
|
||
low = path.lower()
|
||
if not low.endswith(".txt"):
|
||
return
|
||
if "variable" in low or "description" in low:
|
||
return
|
||
found.append((path, raw.decode("latin-1", "replace")))
|
||
|
||
with zipfile.ZipFile(zpath) as outer:
|
||
for name in outer.namelist():
|
||
if name.lower().endswith(".zip"):
|
||
with zipfile.ZipFile(io.BytesIO(outer.read(name))) as inner:
|
||
for inner_name in inner.namelist():
|
||
take(f"{name}/{inner_name}", inner.read(inner_name))
|
||
else:
|
||
take(name, outer.read(name))
|
||
return found
|
||
|
||
|
||
def choose_impact(zpath: Path, year: int) -> tuple[str, str] | None:
|
||
"""Full reprint with the latest named month, else the higher correction rank."""
|
||
scored: list[tuple[tuple[int, int], int, str, str, int]] = []
|
||
for path, text in _impact_members(zpath):
|
||
count = len(parse_impact_text(text, year))
|
||
scored.append((_reprint_date(path), _path_cn_rank(path), path, text, count))
|
||
if not scored:
|
||
return None
|
||
best = max(item[4] for item in scored)
|
||
if best == 0:
|
||
return None
|
||
full = [item for item in scored if item[4] >= 0.8 * best]
|
||
full.sort(key=lambda item: (item[0], item[1], item[2]))
|
||
chosen = full[-1]
|
||
return chosen[2], chosen[3]
|
||
|
||
|
||
def read_impact_year(zpath: Path, year: int) -> pd.DataFrame:
|
||
chosen = choose_impact(zpath, year)
|
||
if chosen is None:
|
||
return _empty(_PROVIDER_COLUMNS)
|
||
_path, text = chosen
|
||
return parse_impact_text(text, year)
|
||
|
||
|
||
def _year_dirs(ipps_dir: Path) -> list[tuple[int, Path]]:
|
||
found: list[tuple[int, Path]] = []
|
||
for folder in sorted(ipps_dir.glob("fy*")):
|
||
if not folder.is_dir():
|
||
continue
|
||
match = re.fullmatch(r"fy(\d{4})", folder.name)
|
||
if match:
|
||
found.append((int(match.group(1)), folder))
|
||
return found
|
||
|
||
|
||
def read_gaf_dir(ipps_dir: Path) -> pd.DataFrame:
|
||
frames: list[pd.DataFrame] = []
|
||
for year, folder in _year_dirs(ipps_dir):
|
||
zips = sorted(folder.glob("wage--*.zip"))
|
||
if not zips:
|
||
continue
|
||
frame = read_gaf_year(zips[0], year)
|
||
print(f" gaf fy{year}: {len(frame)} from {zips[0].name}")
|
||
if not frame.empty:
|
||
frames.append(frame)
|
||
if not frames:
|
||
return _empty(_GAF_COLUMNS)
|
||
return pd.concat(frames, ignore_index=True)
|
||
|
||
|
||
def _factor_zip(factor_dir: Path, year: int, kind: str) -> Path | None:
|
||
matches = sorted(factor_dir.glob(f"fy{year}-{kind}.zip"))
|
||
return matches[0] if matches else None
|
||
|
||
|
||
def read_mac_dir(factor_dir: Path) -> pd.DataFrame:
|
||
frames: list[pd.DataFrame] = []
|
||
for year in range(2014, 2035):
|
||
path = _factor_zip(factor_dir, year, "mac1")
|
||
if path is None:
|
||
continue
|
||
frame = read_mac_year(path, year)
|
||
print(f" mac fy{year}: {len(frame)} factors")
|
||
if not frame.empty:
|
||
frames.append(frame)
|
||
if not frames:
|
||
return _empty(_NATIONAL_COLUMNS)
|
||
return pd.concat(frames, ignore_index=True)
|
||
|
||
|
||
def read_impact_dir(factor_dir: Path) -> pd.DataFrame:
|
||
frames: list[pd.DataFrame] = []
|
||
for year in range(2014, 2035):
|
||
path = _factor_zip(factor_dir, year, "impact")
|
||
if path is None:
|
||
continue
|
||
chosen = choose_impact(path, year)
|
||
if chosen is None:
|
||
print(f" impact fy{year}: no reprint")
|
||
continue
|
||
name, text = chosen
|
||
frame = parse_impact_text(text, year)
|
||
print(f" impact fy{year}: {len(frame)} from {name}")
|
||
if not frame.empty:
|
||
frames.append(frame)
|
||
if not frames:
|
||
return _empty(_PROVIDER_COLUMNS)
|
||
return pd.concat(frames, ignore_index=True)
|
||
|
||
|
||
def read_tables(
|
||
ipps_dir: Path | None = None, factor_dir: Path | None = None
|
||
) -> dict[str, pd.DataFrame]:
|
||
ipps_root = IPPS_DIR if ipps_dir is None else ipps_dir
|
||
factor_root = FACTOR_DIR if factor_dir is None else factor_dir
|
||
gaf = _require_unique(
|
||
read_gaf_dir(ipps_root),
|
||
["fiscal_year", "cbsa", "state"],
|
||
"ipps.capital_gaf",
|
||
)
|
||
national = _require_unique(
|
||
read_mac_dir(factor_root),
|
||
["fiscal_year", "factor", "area"],
|
||
"ipps.national_factor",
|
||
)
|
||
national["area"] = national["area"].fillna("").astype(str)
|
||
provider = _require_unique(
|
||
read_impact_dir(factor_root),
|
||
["fiscal_year", "provider"],
|
||
"ipps.provider_factor",
|
||
)
|
||
return {"gaf": gaf, "national": national, "provider": provider}
|
||
|
||
|
||
def _create_typed(
|
||
con, table: str, frame: pd.DataFrame, doubles: set[str], integers: set[str]
|
||
) -> None:
|
||
con.register("_src", frame)
|
||
try:
|
||
casts = []
|
||
for column in frame.columns:
|
||
if column in doubles:
|
||
kind = "DOUBLE"
|
||
elif column in integers:
|
||
kind = "INTEGER"
|
||
else:
|
||
kind = "VARCHAR"
|
||
quoted = '"' + column.replace('"', '""') + '"'
|
||
casts.append(f"CAST({quoted} AS {kind}) AS {quoted}")
|
||
con.execute(f"DROP TABLE IF EXISTS {table}")
|
||
con.execute(f"CREATE TABLE {table} AS SELECT {', '.join(casts)} FROM _src")
|
||
finally:
|
||
con.unregister("_src")
|
||
|
||
|
||
def load_into(con, tables: dict[str, pd.DataFrame]) -> None:
|
||
"""Replace only the three factor tables. Wage, DRG, and amounts stay."""
|
||
con.execute("CREATE SCHEMA IF NOT EXISTS ipps")
|
||
gaf = tables["gaf"].reindex(columns=list(_GAF_COLUMNS))
|
||
national = tables["national"].reindex(columns=list(_NATIONAL_COLUMNS))
|
||
national["area"] = national["area"].fillna("").astype(str)
|
||
provider = tables["provider"].reindex(columns=list(_PROVIDER_COLUMNS))
|
||
_create_typed(
|
||
con, "ipps.capital_gaf", gaf, {"gaf", "reclassified_gaf"}, {"fiscal_year"}
|
||
)
|
||
_create_typed(con, "ipps.national_factor", national, {"amount"}, {"fiscal_year"})
|
||
_create_typed(
|
||
con,
|
||
"ipps.provider_factor",
|
||
provider,
|
||
_PROVIDER_DOUBLES,
|
||
{"fiscal_year"},
|
||
)
|
||
|
||
|
||
def main() -> None:
|
||
from conf.connect import duckdb_batch, publish_replica
|
||
|
||
print("Reading published IPPS factors ...")
|
||
tables = read_tables()
|
||
with duckdb_batch("aco") as con:
|
||
load_into(con, tables)
|
||
for table in ("capital_gaf", "national_factor", "provider_factor"):
|
||
count = con.execute(f"SELECT count(*) FROM ipps.{table}").fetchone()[0]
|
||
print(f" ipps.{table}: {count}")
|
||
replica = publish_replica("aco")
|
||
print("replica", replica)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|