Files
stack/dev/scripts/ingest_ipps_factors.py
kert 0d99be1b06
Some checks failed
CI / lint (push) Failing after 49s
CI / notebooks-smoke (push) Successful in 2m5s
Deploy / changes (push) Successful in 17s
Infra CI / notebooks (push) Successful in 57s
Infra CI / zotero (push) Successful in 16s
Infra CI / docs (push) Successful in 26s
CI / test (push) Successful in 5m11s
Infra CI / api (push) Successful in 1m20s
Infra CI / bcda (push) Successful in 1m53s
Infra CI / mc (push) Successful in 17s
Infra CI / llm (push) Successful in 1m6s
Deploy / notebooks (push) Has been skipped
Lean Pay / pay (push) Successful in 14s
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m40s
Deploy / bcda (push) Successful in 1m47s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 59s
Deploy / report (push) Successful in 13s
feat(ipps): load the published payment factors
Area capital GAF comes from Table 3, or Tables 4A and 4B when that year has no Table 3. National factors come from MAC Implementation File 1. Hospital IME, DSH, CCR, and capital GAF come from the final-rule impact file.

The claim payment and the Lean ceiling stay absent. The impact file is not the quarterly Provider Specific File. The operating federal rate and the GAF=1 Table 1D capital amount are unchanged.
2026-11-18 12:12:06 -05:00

824 lines
27 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Load published IPPS payment factors. Does not price a claim.
Three grains, each from the file CMS published for that grain:
* Area capital GAF from the wage-index zip already on disk. Table 3,
or Tables 4A and 4B when that year has no Table 3, plus the
reclassified GAF from Table 4C. The Puerto Rico GAF column stays out.
* National factors from MAC Implementation File 1, for the years CMS
posted that file (FY2018–FY2027). FY2014–FY2017 have no file.
* Hospital factors from the final-rule impact file. That file models
the rule. It is not the quarterly Provider Specific File.
The operating federal rate and the Table 1D capital amount are left
where the other ingest put them. This script does not call that ingest
and does not drop those tables.
A correction notice replaces a final rule only when it reprints the
table. Among full impact-file reprints, the latest month named in the
path wins, and a higher correction-notice number wins when no month is
named. January 2014 therefore beats the September 2013 notice even
though that notice's CN number is higher.
"""
from __future__ import annotations
import importlib.util
import io
import re
import sys
import zipfile
from pathlib import Path
import openpyxl
import pandas as pd
ROOT = Path(__file__).resolve().parents[2]
IPPS_DIR = ROOT / "data" / "cms" / "ipps"
FACTOR_DIR = IPPS_DIR / "factors"
_GAF_COLUMNS = (
"fiscal_year",
"cbsa",
"state",
"cbsa_name",
"gaf",
"reclassified_gaf",
)
_NATIONAL_COLUMNS = ("fiscal_year", "factor", "area", "amount")
_PROVIDER_COLUMNS = (
"fiscal_year",
"provider",
"payment_wage_index",
"cola",
"resident_to_bed",
"resident_to_adc",
"operating_ime",
"capital_ime",
"dsh_patient_pct",
"operating_dsh",
"capital_dsh",
"ucp_per_claim",
"operating_ccr",
"capital_ccr",
"gaf",
"capital_cola",
)
_PROVIDER_DOUBLES = set(_PROVIDER_COLUMNS) - {"fiscal_year", "provider"}
_MONTH_NUMBER = {
"jan": 1,
"january": 1,
"feb": 2,
"february": 2,
"mar": 3,
"march": 3,
"apr": 4,
"april": 4,
"may": 5,
"jun": 6,
"june": 6,
"jul": 7,
"july": 7,
"aug": 8,
"august": 8,
"sep": 9,
"sept": 9,
"september": 9,
"oct": 10,
"october": 10,
"nov": 11,
"november": 11,
"dec": 12,
"december": 12,
}
_MONTH_RE = re.compile(
r"(?<![a-z])("
r"january|february|march|april|june|july|august|september|"
r"jan|feb|mar|apr|may|jun|jul|aug|sept|sep|oct|nov|dec"
r")\s+(\d{4})"
)
_LY_RE = re.compile(r"(^|[^a-z])ly([^a-z]|$)")
_QUALITY_ONE = re.compile(r"quality\s*=\s*.{0,3}1")
_QUALITY_ZERO = re.compile(r"quality\s*=\s*.{0,3}0")
_EHR_YES = re.compile(r"ehr\s*=\s*.{0,3}y")
def _tables():
"""Wage-index ingest. Same reprint rule, same header cleanup."""
name = "ingest_ipps_tables"
loaded = sys.modules.get(name)
if loaded is not None and hasattr(loaded, "_wage_root"):
return loaded
path = Path(__file__).with_name("ingest_ipps_tables.py")
spec = importlib.util.spec_from_file_location(name, path)
if spec is None or spec.loader is None:
raise ImportError(path)
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
spec.loader.exec_module(module)
return module
def _money(value: object) -> float | None:
"""Sibling money parse, rounded so a binary 0.34 stays 0.34."""
number = _tables()._money(value)
if number is None:
return None
return round(number, 10)
def _empty(columns: tuple[str, ...]) -> pd.DataFrame:
return pd.DataFrame(columns=list(columns))
def _require_unique(frame: pd.DataFrame, keys: list[str], label: str) -> pd.DataFrame:
if frame.empty:
return frame
collapsed = frame.drop_duplicates()
if collapsed.duplicated(keys).any():
clash = collapsed.loc[collapsed.duplicated(keys, keep=False), keys]
sample = clash.head(5).to_dict("records")
raise ValueError(f"{label} has conflicting rows for {sample}")
return collapsed.drop_duplicates(keys)
def _is_reclassified_gaf(norm: str) -> bool:
if "puerto" in norm:
return False
return "reclassified" in norm and "gaf" in norm
def _is_plain_gaf(norm: str) -> bool:
"""National GAF column. Not Puerto Rico and not the reclassified column."""
if "reclassified" in norm or "puerto" in norm:
return False
return norm == "gaf" or norm.startswith("gaf ") or " gaf" in f" {norm}"
def _gaf_columns(header: list[str], *, reclassified_only: bool) -> dict[str, int]:
tables = _tables()
found: dict[str, int] = {}
for index, cell in enumerate(header):
norm = tables._norm_header(cell)
if norm in {"cbsa", "cbsa code"} and "cbsa" not in found:
found["cbsa"] = index
elif norm in {
"area name",
"urban area",
"rural area",
"nonurban area",
"area",
"state or urban area",
}:
found["cbsa_name"] = index
elif norm in {"state", "state*"} or (
norm.startswith("state") and "code" not in norm
):
found.setdefault("state", index)
elif _is_reclassified_gaf(norm):
found["reclassified_gaf"] = index
elif _is_plain_gaf(norm):
# Table 4C labels the reclassified GAF as "GAF".
if reclassified_only:
found.setdefault("reclassified_gaf", index)
else:
found.setdefault("gaf", index)
return found
def parse_gaf_text(
text: str, year: int, *, reclassified_only: bool = False
) -> pd.DataFrame:
"""One Table 3 or Table 4A/4B/4C text file."""
tables = _tables()
rows = tables._rows(text)
header_at = tables._header_index(rows, "gaf", "cbsa")
if header_at is None:
return _empty(_GAF_COLUMNS)
columns = _gaf_columns(rows[header_at], reclassified_only=reclassified_only)
if "cbsa" not in columns:
return _empty(_GAF_COLUMNS)
if "gaf" not in columns and "reclassified_gaf" not in columns:
return _empty(_GAF_COLUMNS)
parsed: list[dict[str, object]] = []
for row in rows[header_at + 1 :]:
if columns["cbsa"] >= len(row):
continue
cbsa = str(row[columns["cbsa"]]).strip()
if not cbsa or not re.search(r"\d", cbsa):
continue
item: dict[str, object] = {
"fiscal_year": year,
"cbsa": cbsa,
"state": None,
"cbsa_name": None,
"gaf": None,
"reclassified_gaf": None,
}
if "cbsa_name" in columns and columns["cbsa_name"] < len(row):
item["cbsa_name"] = str(row[columns["cbsa_name"]]).strip() or None
if "state" in columns and columns["state"] < len(row):
state = str(row[columns["state"]]).strip()
item["state"] = state or None
for key in ("gaf", "reclassified_gaf"):
if key in columns and columns[key] < len(row):
item[key] = _money(row[columns[key]])
if item["gaf"] is None and item["reclassified_gaf"] is None:
continue
parsed.append(item)
if not parsed:
return _empty(_GAF_COLUMNS)
return pd.DataFrame(parsed, columns=list(_GAF_COLUMNS))
def read_gaf_year(zpath: Path, year: int) -> pd.DataFrame:
"""Area GAF for one fiscal year. Empty when the zip has no GAF table."""
tables = _tables()
root, _source = tables._wage_root(zpath)
try:
grouped: dict[str, list[str]] = {kind: [] for kind in tables._WAGE_KINDS}
for name in root.namelist():
kind = tables._member_kind(name)
if kind in grouped:
grouped[kind].append(name)
if grouped["table3"]:
chosen = tables._pick_full_reprint(
root,
grouped["table3"],
lambda text: parse_gaf_text(text, year),
)
if chosen is None:
return _empty(_GAF_COLUMNS)
return parse_gaf_text(tables._read_zip_text(root, chosen), year)
frames: list[pd.DataFrame] = []
for kind in ("table4a", "table4b"):
chosen = tables._pick_full_reprint(
root,
grouped[kind],
lambda text: parse_gaf_text(text, year),
)
if chosen:
frames.append(parse_gaf_text(tables._read_zip_text(root, chosen), year))
reclass_name = tables._pick_full_reprint(
root,
grouped["table4c"],
lambda text: parse_gaf_text(text, year, reclassified_only=True),
)
base = pd.concat(frames, ignore_index=True) if frames else _empty(_GAF_COLUMNS)
if reclass_name is None:
return base
reclass = parse_gaf_text(
tables._read_zip_text(root, reclass_name),
year,
reclassified_only=True,
)
if base.empty:
return reclass
reclass = reclass.rename(columns={"reclassified_gaf": "_reclass"})
keep = reclass[["cbsa", "state", "_reclass"]].drop_duplicates()
merged = base.merge(keep, on=["cbsa", "state"], how="left")
merged["reclassified_gaf"] = merged["_reclass"].combine_first(
merged["reclassified_gaf"]
)
return merged.drop(columns=["_reclass"])
finally:
root.close()
def _update_factor(label: str) -> str | None:
"""Quality and EHR band. EHR = Y is the penalty, not the bonus."""
low = label.lower().replace("’", "'").replace("‘", "'")
if "quality" not in low or "ehr" not in low:
return None
quality_one = _QUALITY_ONE.search(low) is not None
quality_zero = _QUALITY_ZERO.search(low) is not None
ehr_yes = _EHR_YES.search(low) is not None
ehr_blank = "blank" in low
if quality_one and ehr_blank:
name = "update_quality_ehr"
elif quality_zero and ehr_blank:
name = "update_no_quality"
elif quality_one and ehr_yes:
name = "update_quality_no_ehr"
elif quality_zero and ehr_yes:
name = "update_neither"
else:
return None
if "puerto rico" in low:
return "pr_" + name
return name
def _scalar_factor(label: str) -> str | None:
"""One national factor. The year inside the label is not part of the id."""
low = _tables()._norm_header(label)
low = re.sub(r"\bfy\s*\d{4}\b", " ", low)
low = re.sub(r"\b20\d{2}\b", " ", low)
low = re.sub(r"\s+", " ", low).strip()
if "fixed loss" in low and "threshold" in low:
return "fixed_loss_threshold"
if "cap policy" in low and "budget neutrality" in low:
return "weight_cap_budget_neutrality"
if "budget neutrality" in low and "drg" in low:
return "ms_drg_budget_neutrality"
if "ucp" in low and "ccr" in low and "ceiling" in low:
return "ucp_ccr_ceiling"
if "operating" in low and "ccr" in low and "ceiling" in low:
return "operating_ccr_ceiling"
if "capital" in low and "ccr" in low and "ceiling" in low:
return "capital_ccr_ceiling"
if "scaling factor" in low:
return "ucp_scaling_factor"
return None
def _cola_area(label: str) -> str | None:
low = label.lower()
if "anchorage" in low:
return "anchorage"
if "fairbanks" in low:
return "fairbanks"
if "juneau" in low:
return "juneau"
if "rest of alaska" in low:
return "rest_of_alaska"
if "honolulu" in low:
return "honolulu"
if "kauai" in low:
return "kauai"
if "maui" in low or "kalawao" in low:
return "maui_kalawao"
if "hawaii" in low and "county" in low:
return "hawaii_county"
return None
def _labor_band(label: str) -> str | None:
low = label.lower()
if "wage index" not in low:
return None
if "greater than 1" in low:
return "gt1"
if "less than or equal" in low:
return "le1"
return None
def _first_text(row: list[object]) -> str | None:
for cell in row:
if cell is None:
continue
text = str(cell).strip()
if text:
return text
return None
def _first_money(row: list[object], start: int = 0) -> float | None:
for cell in row[start:]:
amount = _money(cell)
if amount is not None:
return amount
return None
def _labor_header(row: list[object]) -> tuple[int, int] | None:
labor = None
nonlabor = None
for index, cell in enumerate(row):
norm = _tables()._norm_header(cell)
if norm == "labor share":
labor = index
elif "non-labor" in norm or "nonlabor" in norm:
nonlabor = index
if labor is None or nonlabor is None:
return None
return labor, nonlabor
def _sheet_rows(sheet) -> list[list[object]]:
return [list(row) for row in sheet.iter_rows(values_only=True)]
def _parse_esrd(grid: list[list[object]], year: int) -> list[dict[str, object]]:
"""The rate cell. A 'forthcoming' sentence is not a number."""
parsed: list[dict[str, object]] = []
for row in grid:
amount = _first_money(row)
if amount is None:
continue
parsed.append(
{
"fiscal_year": year,
"factor": "esrd_base_rate",
"area": "",
"amount": amount,
}
)
return parsed
def _parse_factor_sheet(
grid: list[list[object]], year: int, *, sheet_title: str
) -> list[dict[str, object]]:
parsed: list[dict[str, object]] = []
labor_at: tuple[int, int] | None = None
pending: str | None = None
cola_on = "cola" in sheet_title.lower()
def emit(factor: str, amount: float | None, area: str = "") -> None:
if amount is None:
return
parsed.append(
{
"fiscal_year": year,
"factor": factor,
"area": area,
"amount": amount,
}
)
for row in grid:
header = _labor_header(row)
if header is not None:
labor_at = header
pending = None
continue
label = _first_text(row)
if label is None:
continue
low = label.lower()
if "cost of living" in low or "cost-of-living" in low:
cola_on = True
band = _labor_band(label) if labor_at is not None else None
if band is not None:
labor_col, nonlabor_col = labor_at
emit(f"labor_share_{band}", _money(_cell(row, labor_col)))
emit(f"nonlabor_share_{band}", _money(_cell(row, nonlabor_col)))
pending = None
continue
update = _update_factor(label)
if update is not None:
emit(update, _first_money(row, 1))
pending = None
continue
area = _cola_area(label) if cola_on else None
if area is not None:
emit("cola", _first_money(row, 1), area)
pending = None
continue
scalar = _scalar_factor(label)
if scalar is not None:
amount = _first_money(row, 1)
if amount is None:
pending = scalar
else:
emit(scalar, amount)
pending = None
continue
if pending is not None:
emit(pending, _first_money(row, 0))
pending = None
return parsed
def _cell(row: list[object], index: int) -> object:
if index >= len(row):
return None
return row[index]
def parse_mac_xlsx(data: bytes, year: int) -> pd.DataFrame:
"""MAC Implementation File 1 workbook. The xlsx, not the text export."""
book = openpyxl.load_workbook(io.BytesIO(data), data_only=True)
try:
parsed: list[dict[str, object]] = []
for sheet in book.worksheets:
grid = _sheet_rows(sheet)
if "esrd" in sheet.title.lower():
parsed.extend(_parse_esrd(grid, year))
else:
parsed.extend(_parse_factor_sheet(grid, year, sheet_title=sheet.title))
finally:
book.close()
if not parsed:
return _empty(_NATIONAL_COLUMNS)
return pd.DataFrame(parsed, columns=list(_NATIONAL_COLUMNS))
def read_mac_year(zpath: Path, year: int) -> pd.DataFrame:
with zipfile.ZipFile(zpath) as zf:
books = [name for name in zf.namelist() if name.lower().endswith(".xlsx")]
if not books:
return _empty(_NATIONAL_COLUMNS)
books.sort(
key=lambda name: ("factor" not in name.lower(), -zf.getinfo(name).file_size)
)
return parse_mac_xlsx(zf.read(books[0]), year)
def _impact_field(norm: str) -> str | None:
"""Map one impact-file header. Last-year and Puerto Rico columns drop out."""
if "provider number" in norm:
return "provider"
if "puerto" in norm:
return None
if _LY_RE.search(norm) or "last year" in norm:
return None
if "505" in norm or "reclass" in norm:
return None
if "capital" in norm and ("cost of living" in norm or "cola" in norm):
return "capital_cola"
if "cost of living" in norm:
return "cola"
if "wage index" in norm:
return "payment_wage_index"
if norm == "rday" or "resident to average daily census" in norm:
return "resident_to_adc"
if "resident to bed" in norm:
return "resident_to_bed"
if norm == "tchcp" or norm.startswith("tchcp"):
return "capital_ime"
if norm == "tchop" or norm.startswith("tchop"):
return "operating_ime"
if norm == "dshpct" or "disproportionate patient percentage" in norm:
return "dsh_patient_pct"
if norm in {"dshcpg", "dshcpp"}:
return "capital_dsh"
if norm in {"dshopg", "dshopp"}:
return "operating_dsh"
if "per claim" in norm and "ucp" in norm:
return "ucp_per_claim"
if "capital" in norm and "ccr" in norm:
return "capital_ccr"
if "operating" in norm and "ccr" in norm:
return "operating_ccr"
if "gaf" in norm:
return "gaf"
return None
def _impact_columns(header: list[str]) -> dict[str, int]:
found: dict[str, int] = {}
for index, cell in enumerate(header):
key = _impact_field(_tables()._norm_header(cell))
if key and key not in found:
found[key] = index
return found
def _provider(value: object) -> str | None:
text = str(value or "").strip()
if text.isdigit() and 0 < len(text) <= 6:
return text.zfill(6)
return None
def parse_impact_text(text: str, year: int) -> pd.DataFrame:
"""One impact-file reprint. The header row is the one that names the provider."""
rows = _tables()._rows(text)
header_at = None
for index, row in enumerate(rows[:8]):
norms = [_tables()._norm_header(cell) for cell in row]
if any("provider number" in norm for norm in norms):
header_at = index
break
if header_at is None:
return _empty(_PROVIDER_COLUMNS)
columns = _impact_columns(rows[header_at])
if "provider" not in columns:
return _empty(_PROVIDER_COLUMNS)
parsed: list[dict[str, object]] = []
for row in rows[header_at + 1 :]:
if columns["provider"] >= len(row):
continue
provider = _provider(row[columns["provider"]])
if provider is None:
continue
item: dict[str, object] = {column: None for column in _PROVIDER_COLUMNS}
item["fiscal_year"] = year
item["provider"] = provider
for key, index in columns.items():
if key == "provider" or index >= len(row):
continue
item[key] = _money(row[index])
parsed.append(item)
if not parsed:
return _empty(_PROVIDER_COLUMNS)
return pd.DataFrame(parsed, columns=list(_PROVIDER_COLUMNS))
def _reprint_date(path: str) -> tuple[int, int]:
"""Latest month-year named in the path. (0, 0) when the name has none."""
found: list[tuple[int, int]] = []
for match in _MONTH_RE.finditer(path.lower()):
month = _MONTH_NUMBER[match.group(1)]
found.append((int(match.group(2)), month))
if not found:
return (0, 0)
return max(found)
def _path_cn_rank(path: str) -> int:
ranks = [_tables()._cn_rank(part) for part in re.split(r"[\\/]", path)]
return max(ranks) if ranks else 0
def _impact_members(zpath: Path) -> list[tuple[str, str]]:
"""Text reprints, one nested zip deep. Descriptions are not reprints."""
found: list[tuple[str, str]] = []
def take(path: str, raw: bytes) -> None:
low = path.lower()
if not low.endswith(".txt"):
return
if "variable" in low or "description" in low:
return
found.append((path, raw.decode("latin-1", "replace")))
with zipfile.ZipFile(zpath) as outer:
for name in outer.namelist():
if name.lower().endswith(".zip"):
with zipfile.ZipFile(io.BytesIO(outer.read(name))) as inner:
for inner_name in inner.namelist():
take(f"{name}/{inner_name}", inner.read(inner_name))
else:
take(name, outer.read(name))
return found
def choose_impact(zpath: Path, year: int) -> tuple[str, str] | None:
"""Full reprint with the latest named month, else the higher correction rank."""
scored: list[tuple[tuple[int, int], int, str, str, int]] = []
for path, text in _impact_members(zpath):
count = len(parse_impact_text(text, year))
scored.append((_reprint_date(path), _path_cn_rank(path), path, text, count))
if not scored:
return None
best = max(item[4] for item in scored)
if best == 0:
return None
full = [item for item in scored if item[4] >= 0.8 * best]
full.sort(key=lambda item: (item[0], item[1], item[2]))
chosen = full[-1]
return chosen[2], chosen[3]
def read_impact_year(zpath: Path, year: int) -> pd.DataFrame:
chosen = choose_impact(zpath, year)
if chosen is None:
return _empty(_PROVIDER_COLUMNS)
_path, text = chosen
return parse_impact_text(text, year)
def _year_dirs(ipps_dir: Path) -> list[tuple[int, Path]]:
found: list[tuple[int, Path]] = []
for folder in sorted(ipps_dir.glob("fy*")):
if not folder.is_dir():
continue
match = re.fullmatch(r"fy(\d{4})", folder.name)
if match:
found.append((int(match.group(1)), folder))
return found
def read_gaf_dir(ipps_dir: Path) -> pd.DataFrame:
frames: list[pd.DataFrame] = []
for year, folder in _year_dirs(ipps_dir):
zips = sorted(folder.glob("wage--*.zip"))
if not zips:
continue
frame = read_gaf_year(zips[0], year)
print(f" gaf fy{year}: {len(frame)} from {zips[0].name}")
if not frame.empty:
frames.append(frame)
if not frames:
return _empty(_GAF_COLUMNS)
return pd.concat(frames, ignore_index=True)
def _factor_zip(factor_dir: Path, year: int, kind: str) -> Path | None:
matches = sorted(factor_dir.glob(f"fy{year}-{kind}.zip"))
return matches[0] if matches else None
def read_mac_dir(factor_dir: Path) -> pd.DataFrame:
frames: list[pd.DataFrame] = []
for year in range(2014, 2035):
path = _factor_zip(factor_dir, year, "mac1")
if path is None:
continue
frame = read_mac_year(path, year)
print(f" mac fy{year}: {len(frame)} factors")
if not frame.empty:
frames.append(frame)
if not frames:
return _empty(_NATIONAL_COLUMNS)
return pd.concat(frames, ignore_index=True)
def read_impact_dir(factor_dir: Path) -> pd.DataFrame:
frames: list[pd.DataFrame] = []
for year in range(2014, 2035):
path = _factor_zip(factor_dir, year, "impact")
if path is None:
continue
chosen = choose_impact(path, year)
if chosen is None:
print(f" impact fy{year}: no reprint")
continue
name, text = chosen
frame = parse_impact_text(text, year)
print(f" impact fy{year}: {len(frame)} from {name}")
if not frame.empty:
frames.append(frame)
if not frames:
return _empty(_PROVIDER_COLUMNS)
return pd.concat(frames, ignore_index=True)
def read_tables(
ipps_dir: Path | None = None, factor_dir: Path | None = None
) -> dict[str, pd.DataFrame]:
ipps_root = IPPS_DIR if ipps_dir is None else ipps_dir
factor_root = FACTOR_DIR if factor_dir is None else factor_dir
gaf = _require_unique(
read_gaf_dir(ipps_root),
["fiscal_year", "cbsa", "state"],
"ipps.capital_gaf",
)
national = _require_unique(
read_mac_dir(factor_root),
["fiscal_year", "factor", "area"],
"ipps.national_factor",
)
national["area"] = national["area"].fillna("").astype(str)
provider = _require_unique(
read_impact_dir(factor_root),
["fiscal_year", "provider"],
"ipps.provider_factor",
)
return {"gaf": gaf, "national": national, "provider": provider}
def _create_typed(
con, table: str, frame: pd.DataFrame, doubles: set[str], integers: set[str]
) -> None:
con.register("_src", frame)
try:
casts = []
for column in frame.columns:
if column in doubles:
kind = "DOUBLE"
elif column in integers:
kind = "INTEGER"
else:
kind = "VARCHAR"
quoted = '"' + column.replace('"', '""') + '"'
casts.append(f"CAST({quoted} AS {kind}) AS {quoted}")
con.execute(f"DROP TABLE IF EXISTS {table}")
con.execute(f"CREATE TABLE {table} AS SELECT {', '.join(casts)} FROM _src")
finally:
con.unregister("_src")
def load_into(con, tables: dict[str, pd.DataFrame]) -> None:
"""Replace only the three factor tables. Wage, DRG, and amounts stay."""
con.execute("CREATE SCHEMA IF NOT EXISTS ipps")
gaf = tables["gaf"].reindex(columns=list(_GAF_COLUMNS))
national = tables["national"].reindex(columns=list(_NATIONAL_COLUMNS))
national["area"] = national["area"].fillna("").astype(str)
provider = tables["provider"].reindex(columns=list(_PROVIDER_COLUMNS))
_create_typed(
con, "ipps.capital_gaf", gaf, {"gaf", "reclassified_gaf"}, {"fiscal_year"}
)
_create_typed(con, "ipps.national_factor", national, {"amount"}, {"fiscal_year"})
_create_typed(
con,
"ipps.provider_factor",
provider,
_PROVIDER_DOUBLES,
{"fiscal_year"},
)
def main() -> None:
from conf.connect import duckdb_batch, publish_replica
print("Reading published IPPS factors ...")
tables = read_tables()
with duckdb_batch("aco") as con:
load_into(con, tables)
for table in ("capital_gaf", "national_factor", "provider_factor"):
count = con.execute(f"SELECT count(*) FROM ipps.{table}").fetchone()[0]
print(f" ipps.{table}: {count}")
replica = publish_replica("aco")
print("replica", replica)
if __name__ == "__main__":
main()