feat(pay): load the published numbers the fence was missing
Some checks failed
CI / lint (push) Failing after 48s
CI / notebooks-smoke (push) Successful in 2m0s
Deploy / changes (push) Successful in 18s
CI / test (push) Successful in 4m23s
Infra CI / notebooks (push) Successful in 1m5s
Infra CI / zotero (push) Successful in 16s
Infra CI / docs (push) Successful in 19s
Infra CI / api (push) Successful in 1m19s
Infra CI / bcda (push) Successful in 1m48s
Infra CI / mc (push) Successful in 18s
Infra CI / llm (push) Successful in 56s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 2m34s
Lean Pay / pay (push) Successful in 3m22s
Deploy / llm (push) Successful in 1m43s
Deploy / mc (push) Has been skipped
Deploy / bcda (push) Successful in 2m10s
Deploy / report (push) Successful in 16s
Some checks failed
CI / lint (push) Failing after 48s
CI / notebooks-smoke (push) Successful in 2m0s
Deploy / changes (push) Successful in 18s
CI / test (push) Successful in 4m23s
Infra CI / notebooks (push) Successful in 1m5s
Infra CI / zotero (push) Successful in 16s
Infra CI / docs (push) Successful in 19s
Infra CI / api (push) Successful in 1m19s
Infra CI / bcda (push) Successful in 1m48s
Infra CI / mc (push) Successful in 18s
Infra CI / llm (push) Successful in 56s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 2m34s
Lean Pay / pay (push) Successful in 3m22s
Deploy / llm (push) Successful in 1m43s
Deploy / mc (push) Has been skipped
Deploy / bcda (push) Successful in 2m10s
Deploy / report (push) Successful in 16s
The wage index, pass-through rates, the missing APC years, the CY2027 work GPCI, equipment usage, IPPS standardized amounts, MS-DRG weights, and the MSSP and REACH financial results now come from the CMS files. The snapshot records those observed ranges. Maintenance stays the published 5/100. There is still no IPPS or LEAD pricer.
This commit is contained in:
731
dev/scripts/ingest_ipps_tables.py
Normal file
731
dev/scripts/ingest_ipps_tables.py
Normal file
@@ -0,0 +1,731 @@
|
||||
"""Load published IPPS tables and the MSSP / ACO REACH settlement files.
|
||||
|
||||
Wage index: IPPS final-rule Table 3 (CBSA). Calendar year N of OPPS uses
|
||||
fiscal year N. Years whose Table 3 is split into average-hourly-wage
|
||||
files (FY2014 and FY2015) use Table 4A and 4B for the wage index and
|
||||
Table 4C for the reclassified index. A correction-notice file replaces
|
||||
the final-rule file only when it is a full reprint, not a short delta.
|
||||
|
||||
Table 5 is the MS-DRG weight. When CMS publishes both a weight and a
|
||||
10% cap-applied weight, the cap-applied weight is the one stored.
|
||||
|
||||
Table 1A-1E is the national standardized amounts. Each published dollar
|
||||
is one point.
|
||||
|
||||
There is still no claim-level IPPS pricer. IME, DSH, and outliers are
|
||||
not computed here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote
|
||||
|
||||
import openpyxl
|
||||
import pandas as pd
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
IPPS_DIR = ROOT / "data" / "cms" / "ipps"
|
||||
MSSP_DIR = ROOT / "data" / "cms" / "mssp"
|
||||
REACH_DIR = ROOT / "data" / "cms" / "reach"
|
||||
|
||||
_WAGE_KINDS = ("table3", "table4a", "table4b", "table4c")
|
||||
|
||||
|
||||
def _norm_header(value: object) -> str:
|
||||
text = re.sub(r"\s+", " ", str(value or "")).strip().lower()
|
||||
text = text.replace("\xa0", " ")
|
||||
text = re.sub(r"^\d+\s*", "", text)
|
||||
return text.strip(" *")
|
||||
|
||||
|
||||
def _money(value: object) -> float | None:
|
||||
if value is None or (isinstance(value, float) and pd.isna(value)):
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int | float):
|
||||
return float(value)
|
||||
text = str(value).strip().replace("$", "").replace(",", "").replace('"', "")
|
||||
if text.lower() in {"", "not applicable", "n/a", "na", "------", "*"}:
|
||||
return None
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _cn_rank(name: str) -> int:
|
||||
"""Higher wins when two files both reprint the table.
|
||||
|
||||
A numbered correction notice outranks an unnumbered one. The FY2016
|
||||
CAA reprint outranks the correction notice. A short delta still loses
|
||||
on row count before this rank is consulted.
|
||||
"""
|
||||
low = Path(name).name.lower()
|
||||
if "caa" in low:
|
||||
return 3
|
||||
numbered = re.search(r"cn(\d+)", low)
|
||||
if numbered:
|
||||
return int(numbered.group(1))
|
||||
if (
|
||||
"correction" in low
|
||||
or "correcting" in low
|
||||
or re.match(r"ca\b", low)
|
||||
or re.search(r"(^|[^a-z])cn([^a-z]|$)", low)
|
||||
):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _member_kind(name: str) -> str | None:
|
||||
"""Table id from the member name. ``Tables 3`` and ``Table_5`` both count.
|
||||
|
||||
``3`` inside a rule number (``1735``) does not. Table 3A and 3B are
|
||||
average hourly wage, not the index, so a letter or digit after the 3
|
||||
is not Table 3.
|
||||
"""
|
||||
base = Path(name).name.lower()
|
||||
if not base.endswith(".txt"):
|
||||
return None
|
||||
if re.search(r"tables?[_\s-]*3([^0-9ab]|$)", base):
|
||||
return "table3"
|
||||
if re.search(r"tables?[_\s-]*4a([^0-9a-z]|$)", base):
|
||||
return "table4a"
|
||||
if re.search(r"tables?[_\s-]*4b([^0-9a-z]|$)", base):
|
||||
return "table4b"
|
||||
if re.search(r"tables?[_\s-]*4c([^0-9a-z]|$)", base):
|
||||
return "table4c"
|
||||
if re.search(r"tables?[_\s-]*5([^0-9]|$)", base):
|
||||
return "table5"
|
||||
return None
|
||||
|
||||
|
||||
def _read_zip_text(zf: zipfile.ZipFile, name: str) -> str:
|
||||
return zf.read(name).decode("latin-1", "replace")
|
||||
|
||||
|
||||
def _rows(text: str) -> list[list[str]]:
|
||||
return list(csv.reader(io.StringIO(text), delimiter="\t"))
|
||||
|
||||
|
||||
def _header_index(rows: list[list[str]], *needles: str) -> int | None:
|
||||
for index, row in enumerate(rows[:15]):
|
||||
norms = [_norm_header(cell) for cell in row]
|
||||
if any(any(needle in norm for needle in needles) for norm in norms):
|
||||
if any(
|
||||
norm in {"cbsa", "cbsa code", "ms-drg"} or norm.startswith("ms-drg")
|
||||
for norm in norms
|
||||
):
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def _wage_columns(header: list[str], *, reclassified_only: bool) -> dict[str, int]:
|
||||
found: dict[str, int] = {}
|
||||
for index, cell in enumerate(header):
|
||||
norm = _norm_header(cell)
|
||||
if norm in {"cbsa", "cbsa code"} and "cbsa" not in found:
|
||||
found["cbsa"] = index
|
||||
elif norm in {
|
||||
"area name",
|
||||
"urban area",
|
||||
"rural area",
|
||||
"nonurban area",
|
||||
"area",
|
||||
"state or urban area",
|
||||
}:
|
||||
found["cbsa_name"] = index
|
||||
elif norm in {"state", "state*"} or (
|
||||
norm.startswith("state") and "code" not in norm
|
||||
):
|
||||
found.setdefault("state", index)
|
||||
elif _is_reclassified_wage(norm):
|
||||
found["reclassified_wage_index"] = index
|
||||
elif _is_plain_wage_index(norm):
|
||||
if reclassified_only:
|
||||
# Table 4C labels the reclassified index as "Wage Index".
|
||||
# A real reclassified column, when the file has both, wins.
|
||||
found.setdefault("reclassified_wage_index", index)
|
||||
else:
|
||||
found.setdefault("wage_index", index)
|
||||
return found
|
||||
|
||||
|
||||
def _is_reclassified_wage(norm: str) -> bool:
|
||||
"""The CBSA reclassified index, not the Puerto Rico or GAF column."""
|
||||
if "puerto" in norm or "gaf" in norm:
|
||||
return False
|
||||
return "reclassified" in norm and "wage index" in norm
|
||||
|
||||
|
||||
def _is_plain_wage_index(norm: str) -> bool:
|
||||
if "reclassified" in norm or "puerto" in norm or "gaf" in norm:
|
||||
return False
|
||||
return norm == "wage index" or norm.startswith("wage index ")
|
||||
|
||||
|
||||
def parse_wage_text(
|
||||
text: str, year: int, *, reclassified_only: bool = False
|
||||
) -> pd.DataFrame:
|
||||
"""One Table 3 (or Table 4A/4B/4C) text file."""
|
||||
rows = _rows(text)
|
||||
header_at = _header_index(rows, "wage index", "cbsa")
|
||||
if header_at is None:
|
||||
return pd.DataFrame()
|
||||
columns = _wage_columns(rows[header_at], reclassified_only=reclassified_only)
|
||||
if "cbsa" not in columns:
|
||||
return pd.DataFrame()
|
||||
if "wage_index" not in columns and "reclassified_wage_index" not in columns:
|
||||
return pd.DataFrame()
|
||||
parsed: list[dict[str, object]] = []
|
||||
for row in rows[header_at + 1 :]:
|
||||
if columns["cbsa"] >= len(row):
|
||||
continue
|
||||
cbsa = str(row[columns["cbsa"]]).strip()
|
||||
if not cbsa or not re.search(r"\d", cbsa):
|
||||
continue
|
||||
item: dict[str, object] = {
|
||||
"cbsa": cbsa,
|
||||
"cbsa_name": None,
|
||||
"state": None,
|
||||
"wage_index": None,
|
||||
"reclassified_wage_index": None,
|
||||
"year": year,
|
||||
}
|
||||
if "cbsa_name" in columns and columns["cbsa_name"] < len(row):
|
||||
item["cbsa_name"] = str(row[columns["cbsa_name"]]).strip() or None
|
||||
if "state" in columns and columns["state"] < len(row):
|
||||
state = str(row[columns["state"]]).strip()
|
||||
item["state"] = state or None
|
||||
for key in ("wage_index", "reclassified_wage_index"):
|
||||
if key in columns and columns[key] < len(row):
|
||||
item[key] = _money(row[columns[key]])
|
||||
if item["wage_index"] is None and item["reclassified_wage_index"] is None:
|
||||
continue
|
||||
parsed.append(item)
|
||||
return pd.DataFrame(parsed)
|
||||
|
||||
|
||||
def _pick_full_reprint(zf: zipfile.ZipFile, names: list[str], parser) -> str | None:
|
||||
"""A correction notice wins only when it reprints the table, not a delta."""
|
||||
if not names:
|
||||
return None
|
||||
scored: list[tuple[int, int, str]] = []
|
||||
for name in names:
|
||||
frame = parser(_read_zip_text(zf, name))
|
||||
scored.append((len(frame), _cn_rank(name), name))
|
||||
best = max(count for count, _, _ in scored)
|
||||
if best == 0:
|
||||
return None
|
||||
full = [item for item in scored if item[0] >= 0.8 * best]
|
||||
full.sort()
|
||||
return full[-1][2]
|
||||
|
||||
|
||||
def _wage_root(zpath: Path) -> tuple[zipfile.ZipFile, str]:
|
||||
outer = zipfile.ZipFile(zpath)
|
||||
nested = [name for name in outer.namelist() if name.lower().endswith(".zip")]
|
||||
if not nested:
|
||||
return outer, zpath.name
|
||||
new = [name for name in nested if "new" in name.lower()]
|
||||
chosen = new[0] if new else nested[0]
|
||||
inner = zipfile.ZipFile(io.BytesIO(outer.read(chosen)))
|
||||
outer.close()
|
||||
return inner, chosen
|
||||
|
||||
|
||||
def read_wage_year(zpath: Path, year: int) -> pd.DataFrame:
|
||||
"""Wage index rows for one fiscal year. Empty if the zip has no index table."""
|
||||
root, _source = _wage_root(zpath)
|
||||
try:
|
||||
grouped: dict[str, list[str]] = {kind: [] for kind in _WAGE_KINDS}
|
||||
for name in root.namelist():
|
||||
kind = _member_kind(name)
|
||||
if kind in grouped:
|
||||
grouped[kind].append(name)
|
||||
if grouped["table3"]:
|
||||
chosen = _pick_full_reprint(
|
||||
root,
|
||||
grouped["table3"],
|
||||
lambda text: parse_wage_text(text, year),
|
||||
)
|
||||
if chosen is None:
|
||||
return pd.DataFrame()
|
||||
return parse_wage_text(_read_zip_text(root, chosen), year)
|
||||
|
||||
frames: list[pd.DataFrame] = []
|
||||
for kind in ("table4a", "table4b"):
|
||||
chosen = _pick_full_reprint(
|
||||
root,
|
||||
grouped[kind],
|
||||
lambda text: parse_wage_text(text, year),
|
||||
)
|
||||
if chosen:
|
||||
frames.append(parse_wage_text(_read_zip_text(root, chosen), year))
|
||||
reclass_name = _pick_full_reprint(
|
||||
root,
|
||||
grouped["table4c"],
|
||||
lambda text: parse_wage_text(text, year, reclassified_only=True),
|
||||
)
|
||||
base = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
|
||||
if reclass_name is None:
|
||||
return base
|
||||
reclass = parse_wage_text(
|
||||
_read_zip_text(root, reclass_name), year, reclassified_only=True
|
||||
)
|
||||
if base.empty:
|
||||
return reclass
|
||||
reclass = reclass.rename(columns={"reclassified_wage_index": "_reclass"})
|
||||
keep = reclass[["cbsa", "state", "_reclass"]].drop_duplicates()
|
||||
merged = base.merge(keep, on=["cbsa", "state"], how="left")
|
||||
merged["reclassified_wage_index"] = merged["_reclass"].combine_first(
|
||||
merged["reclassified_wage_index"]
|
||||
)
|
||||
return merged.drop(columns=["_reclass"])
|
||||
finally:
|
||||
root.close()
|
||||
|
||||
|
||||
def parse_drg_text(text: str, year: int) -> pd.DataFrame:
|
||||
"""Table 5. The cap-applied weight is the paid weight when both are published."""
|
||||
rows = _rows(text)
|
||||
header_at = None
|
||||
for index, row in enumerate(rows[:12]):
|
||||
norms = [_norm_header(cell) for cell in row]
|
||||
if any(norm.startswith("ms-drg") for norm in norms) and any(
|
||||
"weight" in norm for norm in norms
|
||||
):
|
||||
header_at = index
|
||||
break
|
||||
if header_at is None:
|
||||
return pd.DataFrame()
|
||||
header = [_norm_header(cell) for cell in rows[header_at]]
|
||||
weight_at = next(
|
||||
(
|
||||
i
|
||||
for i, cell in enumerate(header)
|
||||
if "cap applied" in cell or "10% cap" in cell
|
||||
),
|
||||
None,
|
||||
)
|
||||
if weight_at is None:
|
||||
weight_at = next(
|
||||
(
|
||||
i
|
||||
for i, cell in enumerate(header)
|
||||
if cell == "weights" or cell.endswith(" weights")
|
||||
),
|
||||
None,
|
||||
)
|
||||
drg_at = next(
|
||||
(i for i, cell in enumerate(header) if cell.startswith("ms-drg")), None
|
||||
)
|
||||
geo_at = next((i for i, cell in enumerate(header) if "geometric" in cell), None)
|
||||
ari_at = next((i for i, cell in enumerate(header) if "arithmetic" in cell), None)
|
||||
if weight_at is None or drg_at is None:
|
||||
return pd.DataFrame()
|
||||
parsed: list[dict[str, object]] = []
|
||||
for row in rows[header_at + 1 :]:
|
||||
if drg_at >= len(row) or weight_at >= len(row):
|
||||
continue
|
||||
code = str(row[drg_at]).strip()
|
||||
if not code.isdigit():
|
||||
continue
|
||||
weight = _money(row[weight_at])
|
||||
if weight is None:
|
||||
continue
|
||||
parsed.append(
|
||||
{
|
||||
"ms_drg": code.zfill(3),
|
||||
"relative_weight": weight,
|
||||
"geometric_mean_los": _money(row[geo_at])
|
||||
if geo_at is not None and geo_at < len(row)
|
||||
else None,
|
||||
"arithmetic_mean_los": _money(row[ari_at])
|
||||
if ari_at is not None and ari_at < len(row)
|
||||
else None,
|
||||
"fiscal_year": year,
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(parsed)
|
||||
|
||||
|
||||
def read_drg_year(zpath: Path, year: int) -> pd.DataFrame:
|
||||
with zipfile.ZipFile(zpath) as zf:
|
||||
names = [name for name in zf.namelist() if _member_kind(name) == "table5"]
|
||||
chosen = _pick_full_reprint(zf, names, lambda text: parse_drg_text(text, year))
|
||||
if chosen is None:
|
||||
return pd.DataFrame()
|
||||
return parse_drg_text(_read_zip_text(zf, chosen), year)
|
||||
|
||||
|
||||
def _hospital_band(text: str) -> str:
|
||||
low = text.lower()
|
||||
if "greater than 1" in low:
|
||||
wage = "wi_gt_1"
|
||||
elif "less than or equal" in low:
|
||||
wage = "wi_le_1"
|
||||
else:
|
||||
wage = ""
|
||||
if "not submit quality" in low or "did not submit quality" in low:
|
||||
quality = "no_quality"
|
||||
elif "quality" in low:
|
||||
quality = "quality"
|
||||
else:
|
||||
quality = ""
|
||||
if "not a meaningful" in low:
|
||||
ehr = "no_ehr"
|
||||
elif "meaningful" in low or "ehr" in low:
|
||||
ehr = "ehr"
|
||||
else:
|
||||
ehr = ""
|
||||
if not quality and not ehr and not wage:
|
||||
if "full update" in low:
|
||||
return "full_update"
|
||||
if "reduced update" in low:
|
||||
return "reduced_update"
|
||||
return "_".join(part for part in (quality, ehr, wage) if part)
|
||||
|
||||
|
||||
def _share_label(text: str) -> str:
|
||||
low = _norm_header(text)
|
||||
if "nonlabor" in low:
|
||||
return "nonlabor"
|
||||
if low == "labor" or low.startswith("labor"):
|
||||
return "labor"
|
||||
if low.startswith("full update"):
|
||||
return "full_update"
|
||||
if "reduced update" in low:
|
||||
return "reduced_update"
|
||||
if low == "rate":
|
||||
return "rate"
|
||||
return ""
|
||||
|
||||
|
||||
def _row_label(text: str) -> str:
|
||||
low = _norm_header(text)
|
||||
if low.startswith("national"):
|
||||
return "national"
|
||||
if "puerto rico" in low:
|
||||
return "puerto_rico"
|
||||
if "standard federal" in low:
|
||||
return "standard_federal_rate"
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_tables(candidates: list[tuple[int, int, str, pd.DataFrame]]) -> pd.DataFrame:
|
||||
"""One frame per table code. A short correction sheet does not replace a full one."""
|
||||
by_code: dict[str, list[tuple[int, int, str, pd.DataFrame]]] = {}
|
||||
for rank, _count, title, frame in candidates:
|
||||
if frame.empty or "table_code" not in frame.columns:
|
||||
continue
|
||||
for code, part in frame.groupby("table_code", sort=False):
|
||||
by_code.setdefault(str(code), []).append((len(part), rank, title, part))
|
||||
chosen: list[pd.DataFrame] = []
|
||||
for parts in by_code.values():
|
||||
best = max(item[0] for item in parts)
|
||||
full = [item for item in parts if item[0] >= 0.8 * best]
|
||||
full.sort()
|
||||
chosen.append(full[-1][3])
|
||||
if not chosen:
|
||||
return pd.DataFrame()
|
||||
return pd.concat(chosen, ignore_index=True)
|
||||
|
||||
|
||||
def parse_standardized_xlsx(data: bytes, year: int) -> pd.DataFrame:
|
||||
"""Each dollar in Tables 1A-1E. 'Not Applicable' is not a number.
|
||||
|
||||
A workbook often has a final-rule sheet and a correction sheet. The
|
||||
correction sheet is kept only when it reprints that table. A table
|
||||
the correction sheet omits (FY2015 Table 1E) stays from the final rule.
|
||||
"""
|
||||
book = openpyxl.load_workbook(io.BytesIO(data), data_only=True)
|
||||
try:
|
||||
parsed: list[tuple[int, int, str, pd.DataFrame]] = []
|
||||
for sheet in book.worksheets:
|
||||
grid = [[cell for cell in row] for row in sheet.iter_rows(values_only=True)]
|
||||
frame = _parse_standardized_grid(grid, year)
|
||||
parsed.append((_cn_rank(sheet.title), len(frame), sheet.title, frame))
|
||||
finally:
|
||||
book.close()
|
||||
if sum(1 for _rank, count, _title, _frame in parsed if count) > 1:
|
||||
for rank, count, title, _frame in parsed:
|
||||
print(f" sheet {title!r} cn={rank} rows={count}")
|
||||
return _pick_tables(parsed)
|
||||
|
||||
|
||||
def _parse_standardized_grid(grid: list[list[object]], year: int) -> pd.DataFrame:
|
||||
"""One worksheet of Tables 1A-1E."""
|
||||
table = ""
|
||||
parsed: list[dict[str, object]] = []
|
||||
for index, row in enumerate(grid):
|
||||
joined = " ".join(str(cell) for cell in row if cell is not None)
|
||||
found = re.search(r"TABLE\s+1([A-E])\b", joined, flags=re.I)
|
||||
if found:
|
||||
table = f"1{found.group(1).upper()}"
|
||||
continue
|
||||
if not table:
|
||||
continue
|
||||
amounts = [_money(cell) for cell in row]
|
||||
if not any(amount is not None for amount in amounts):
|
||||
continue
|
||||
# The label row is the nearest prior row that names labor/nonlabor/rate/update.
|
||||
share_row = None
|
||||
band_row = None
|
||||
for look in range(index - 1, max(-1, index - 4), -1):
|
||||
labels = [_share_label(str(cell or "")) for cell in grid[look]]
|
||||
if any(labels):
|
||||
share_row = labels
|
||||
if look > 0:
|
||||
band_row = [
|
||||
_hospital_band(str(cell or "")) for cell in grid[look - 1]
|
||||
]
|
||||
break
|
||||
if share_row is None:
|
||||
continue
|
||||
label = ""
|
||||
for cell, amount in zip(row, amounts, strict=False):
|
||||
if amount is None and _row_label(str(cell or "")):
|
||||
label = _row_label(str(cell or ""))
|
||||
break
|
||||
for col, amount in enumerate(amounts):
|
||||
if amount is None:
|
||||
continue
|
||||
share = share_row[col] if col < len(share_row) else ""
|
||||
if not share:
|
||||
for left in range(col, -1, -1):
|
||||
if left < len(share_row) and share_row[left]:
|
||||
share = share_row[left]
|
||||
break
|
||||
band = ""
|
||||
if band_row:
|
||||
for left in range(col, -1, -1):
|
||||
if left < len(band_row) and band_row[left]:
|
||||
band = band_row[left]
|
||||
break
|
||||
parts = [part for part in (label, band, share) if part]
|
||||
if not parts:
|
||||
continue
|
||||
parsed.append(
|
||||
{
|
||||
"fiscal_year": year,
|
||||
"table_code": table,
|
||||
"component": "_".join(parts),
|
||||
"amount": amount,
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(parsed)
|
||||
|
||||
|
||||
def read_standardized_year(zpath: Path, year: int) -> pd.DataFrame:
|
||||
with zipfile.ZipFile(zpath) as zf:
|
||||
books = [name for name in zf.namelist() if name.lower().endswith(".xlsx")]
|
||||
if not books:
|
||||
return pd.DataFrame()
|
||||
books.sort(key=lambda name: (_cn_rank(name), name))
|
||||
return parse_standardized_xlsx(zf.read(books[-1]), year)
|
||||
|
||||
|
||||
def _year_from_dirname(path: Path) -> int | None:
|
||||
match = re.fullmatch(r"fy(\d{4})", path.name)
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def read_ipps_dir(root: Path = IPPS_DIR) -> dict[str, pd.DataFrame]:
|
||||
"""Read every downloaded fiscal year. A missing file is skipped."""
|
||||
wages: list[pd.DataFrame] = []
|
||||
drgs: list[pd.DataFrame] = []
|
||||
amounts: list[pd.DataFrame] = []
|
||||
for year_dir in sorted(root.glob("fy*")):
|
||||
year = _year_from_dirname(year_dir)
|
||||
if year is None:
|
||||
continue
|
||||
wage_zip = next(year_dir.glob("wage--*.zip"), None)
|
||||
drg_zip = next(year_dir.glob("drg--*.zip"), None)
|
||||
amount_zip = next(year_dir.glob("standardized--*.zip"), None)
|
||||
if wage_zip is not None:
|
||||
frame = read_wage_year(wage_zip, year)
|
||||
print(f" FY{year} wage {len(frame)} from {wage_zip.name}")
|
||||
if not frame.empty:
|
||||
wages.append(frame)
|
||||
if drg_zip is not None:
|
||||
frame = read_drg_year(drg_zip, year)
|
||||
print(f" FY{year} drg {len(frame)} from {drg_zip.name}")
|
||||
if not frame.empty:
|
||||
drgs.append(frame)
|
||||
if amount_zip is not None:
|
||||
frame = read_standardized_year(amount_zip, year)
|
||||
print(f" FY{year} amounts {len(frame)} from {amount_zip.name}")
|
||||
if not frame.empty:
|
||||
amounts.append(frame)
|
||||
return {
|
||||
"wage": pd.concat(wages, ignore_index=True) if wages else pd.DataFrame(),
|
||||
"drg": pd.concat(drgs, ignore_index=True) if drgs else pd.DataFrame(),
|
||||
"amount": pd.concat(amounts, ignore_index=True) if amounts else pd.DataFrame(),
|
||||
}
|
||||
|
||||
|
||||
_MSSP_MONEY = (
|
||||
"earnsaveloss",
|
||||
"gensaveloss",
|
||||
"abtotbnchmk",
|
||||
"abtotexp",
|
||||
"updatedbnchmk",
|
||||
"histbnchmk",
|
||||
"sav_rate",
|
||||
"qualscore",
|
||||
"finalsharerate",
|
||||
)
|
||||
_REACH_MONEY = (
|
||||
"shared_savings",
|
||||
"gross_savings",
|
||||
"final_benchmark",
|
||||
"unadj_benchmark",
|
||||
"tot_cost_care",
|
||||
"tot_cost_care_net",
|
||||
"sav_rate",
|
||||
)
|
||||
|
||||
|
||||
def _perf_year_from_filename(name: str) -> int | None:
|
||||
"""First calendar year in the file name. ``%20`` is a space, not 2020."""
|
||||
match = re.search(r"(20\d{2})", unquote(name))
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def read_settlement_dir(
|
||||
directory: Path, money: tuple[str, ...], *, year_column: str
|
||||
) -> pd.DataFrame:
|
||||
"""Every catalog CSV in one directory. The filename's first year is the performance year."""
|
||||
from cms.datacms import sanitize_column_name
|
||||
|
||||
frames: list[pd.DataFrame] = []
|
||||
for path in sorted(directory.glob("*.csv")):
|
||||
frame = pd.read_csv(
|
||||
path, dtype=str, encoding="utf-8", encoding_errors="replace"
|
||||
)
|
||||
frame.columns = [sanitize_column_name(column) for column in frame.columns]
|
||||
year = _perf_year_from_filename(path.name)
|
||||
if year_column in frame.columns:
|
||||
parsed = pd.to_numeric(frame[year_column], errors="coerce")
|
||||
frame[year_column] = parsed.round().astype("Int64").astype(str)
|
||||
frame.loc[parsed.isna(), year_column] = str(year) if year else pd.NA
|
||||
elif year is not None:
|
||||
frame[year_column] = str(year)
|
||||
frame["source_file"] = path.name
|
||||
for column in frame.columns:
|
||||
if column in money:
|
||||
frame[column] = pd.to_numeric(frame[column], errors="coerce")
|
||||
else:
|
||||
frame[column] = frame[column].astype("string")
|
||||
frames.append(frame)
|
||||
print(f" {path.name}: {len(frame)} rows year {year}")
|
||||
if not frames:
|
||||
return pd.DataFrame()
|
||||
return pd.concat(frames, ignore_index=True)
|
||||
|
||||
|
||||
def _create_typed(
|
||||
con, table: str, frame: pd.DataFrame, doubles: set[str], integers: set[str]
|
||||
) -> None:
|
||||
"""Load a frame with explicit types so an id column is not inferred as a measure."""
|
||||
con.register("_src", frame)
|
||||
try:
|
||||
casts = []
|
||||
for column in frame.columns:
|
||||
if column in doubles:
|
||||
kind = "DOUBLE"
|
||||
elif column in integers:
|
||||
kind = "INTEGER"
|
||||
else:
|
||||
kind = "VARCHAR"
|
||||
quoted = '"' + column.replace('"', '""') + '"'
|
||||
casts.append(f"CAST({quoted} AS {kind}) AS {quoted}")
|
||||
con.execute(f"CREATE TABLE {table} AS SELECT {', '.join(casts)} FROM _src")
|
||||
finally:
|
||||
con.unregister("_src")
|
||||
|
||||
|
||||
def load_into(con) -> None:
|
||||
"""Replace the IPPS and settlement tables. Other schemas are left alone."""
|
||||
tables = read_ipps_dir()
|
||||
con.execute("CREATE SCHEMA IF NOT EXISTS opps")
|
||||
con.execute("CREATE SCHEMA IF NOT EXISTS ipps")
|
||||
con.execute("CREATE SCHEMA IF NOT EXISTS cms")
|
||||
if not tables["wage"].empty:
|
||||
con.execute("DROP TABLE IF EXISTS opps.wage_index")
|
||||
_create_typed(
|
||||
con,
|
||||
"opps.wage_index",
|
||||
tables["wage"],
|
||||
{"wage_index", "reclassified_wage_index"},
|
||||
{"year"},
|
||||
)
|
||||
if not tables["drg"].empty:
|
||||
con.execute("DROP TABLE IF EXISTS ipps.drg_weight")
|
||||
_create_typed(
|
||||
con,
|
||||
"ipps.drg_weight",
|
||||
tables["drg"],
|
||||
{"relative_weight", "geometric_mean_los", "arithmetic_mean_los"},
|
||||
{"fiscal_year"},
|
||||
)
|
||||
if not tables["amount"].empty:
|
||||
con.execute("DROP TABLE IF EXISTS ipps.standardized_amount")
|
||||
_create_typed(
|
||||
con,
|
||||
"ipps.standardized_amount",
|
||||
tables["amount"],
|
||||
{"amount"},
|
||||
{"fiscal_year"},
|
||||
)
|
||||
mssp = read_settlement_dir(MSSP_DIR, _MSSP_MONEY, year_column="perf_year")
|
||||
reach = read_settlement_dir(REACH_DIR, _REACH_MONEY, year_column="perf_year")
|
||||
if not mssp.empty:
|
||||
con.execute(
|
||||
"DROP TABLE IF EXISTS cms.performance_year_financial_quality_results"
|
||||
)
|
||||
_create_typed(
|
||||
con,
|
||||
"cms.performance_year_financial_quality_results",
|
||||
mssp,
|
||||
set(_MSSP_MONEY),
|
||||
set(),
|
||||
)
|
||||
if not reach.empty:
|
||||
con.execute("DROP TABLE IF EXISTS cms.aco_reach_financial_quality_results")
|
||||
_create_typed(
|
||||
con,
|
||||
"cms.aco_reach_financial_quality_results",
|
||||
reach,
|
||||
set(_REACH_MONEY),
|
||||
set(),
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
from conf.connect import duckdb_batch, publish_replica
|
||||
|
||||
print("Reading published tables ...")
|
||||
with duckdb_batch("aco") as con:
|
||||
load_into(con)
|
||||
for schema, table in (
|
||||
("opps", "wage_index"),
|
||||
("ipps", "drg_weight"),
|
||||
("ipps", "standardized_amount"),
|
||||
("cms", "performance_year_financial_quality_results"),
|
||||
("cms", "aco_reach_financial_quality_results"),
|
||||
):
|
||||
count = con.execute(f"SELECT count(*) FROM {schema}.{table}").fetchone()[0]
|
||||
print(f" {schema}.{table}: {count}")
|
||||
replica = publish_replica("aco")
|
||||
print("replica", replica)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -136,11 +136,12 @@ def find_data_file(zf: zipfile.ZipFile, pattern: str = "") -> str | None:
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
def _find_header_row(df: pd.DataFrame, col_map: dict[str, str]) -> int:
|
||||
def _find_header_row(df: pd.DataFrame, col_map: dict[str, str]) -> int | None:
|
||||
"""Find the row index containing column headers in a preamble-laden CMS file.
|
||||
|
||||
Requires >= 3 matches AND at least 3 non-null cells (to distinguish
|
||||
a real header row from a title row that happens to contain keywords).
|
||||
None means no row cleared that bar. A header on row 0 is 0, not a miss.
|
||||
"""
|
||||
target_cols = [k for k in col_map if k not in ("", "year")]
|
||||
for i, row in df.iterrows():
|
||||
@@ -151,8 +152,36 @@ def _find_header_row(df: pd.DataFrame, col_map: dict[str, str]) -> int:
|
||||
1 for v in vals for k in target_cols if v == k or (len(k) > 3 and k in v)
|
||||
)
|
||||
if matches >= 3:
|
||||
return i
|
||||
return 0
|
||||
return int(i)
|
||||
return None
|
||||
|
||||
|
||||
def _excel_payment_sheet(
|
||||
data: bytes, col_map: dict[str, str]
|
||||
) -> tuple[str, int] | None:
|
||||
"""First sheet whose opening rows are a payment header.
|
||||
|
||||
Some Addendum A workbooks put an APC-category lookup on sheet 0 and
|
||||
the relative weights on the next sheet. A sheet counts only when one
|
||||
of its first 20 rows matches at least three column-map headers. The
|
||||
first such sheet wins, so a workbook whose first sheet is already the
|
||||
payment sheet stays on that sheet.
|
||||
"""
|
||||
book = pd.ExcelFile(io.BytesIO(data))
|
||||
try:
|
||||
for sheet in book.sheet_names:
|
||||
try:
|
||||
preview = pd.read_excel(
|
||||
book, sheet_name=sheet, dtype=str, header=None, nrows=20
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
hdr = _find_header_row(preview, col_map)
|
||||
if hdr is not None:
|
||||
return str(sheet), hdr
|
||||
finally:
|
||||
book.close()
|
||||
return None
|
||||
|
||||
|
||||
def read_from_zip(
|
||||
@@ -184,12 +213,21 @@ def read_from_zip(
|
||||
buf = "\n".join(lines[hdr_idx:])
|
||||
df = pd.read_csv(io.StringIO(buf), dtype=str, on_bad_lines="skip")
|
||||
else:
|
||||
# Excel: read without header to find preamble extent
|
||||
df_raw = pd.read_excel(
|
||||
io.BytesIO(data), dtype=str, header=None, nrows=20
|
||||
found = _excel_payment_sheet(data, col_map)
|
||||
if found is None:
|
||||
print(f" {zpath.name}/{pattern}: no payment sheet")
|
||||
return pd.DataFrame()
|
||||
sheet, hdr_row = found
|
||||
df = pd.read_excel(
|
||||
io.BytesIO(data),
|
||||
sheet_name=sheet,
|
||||
dtype=str,
|
||||
header=hdr_row,
|
||||
)
|
||||
hdr_row = _find_header_row(df_raw, col_map)
|
||||
df = pd.read_excel(io.BytesIO(data), dtype=str, header=hdr_row)
|
||||
if sheet:
|
||||
print(
|
||||
f" {pattern or target}: sheet {sheet!r} header row {hdr_row}"
|
||||
)
|
||||
|
||||
return normalise_columns(df, col_map)
|
||||
except Exception as e:
|
||||
@@ -344,6 +382,57 @@ def ingest_all(con: duckdb.DuckDBPyConnection, year_filter: str = "") -> None:
|
||||
for yr, si, ct in si_rows:
|
||||
print(f" CY{yr} SI={si}: {ct} codes")
|
||||
|
||||
# Pass-through rates are Addendum B rows with status indicator G
|
||||
# (drugs and biologicals) or H (devices). Addendum D1/D2 are indicator
|
||||
# definitions, not payment rates. Built from the table, so a --year
|
||||
# run keeps every other year.
|
||||
load_pass_through(con)
|
||||
|
||||
|
||||
def load_pass_through(con: duckdb.DuckDBPyConnection) -> int:
|
||||
"""Copy Addendum B status indicators G and H into opps.pass_through.
|
||||
|
||||
``asp_per_unit`` stays null. ``drugs.asp_payment.payment_limit`` is
|
||||
the ASP+6% allowance, and dividing it by 1.06 would invent an ASP.
|
||||
Status indicator K is a separately payable drug, not pass-through.
|
||||
"""
|
||||
exists = con.execute(
|
||||
"SELECT 1 FROM information_schema.tables "
|
||||
"WHERE table_schema = 'opps' AND table_name = 'addendum_b'"
|
||||
).fetchone()
|
||||
if not exists:
|
||||
return 0
|
||||
has_si = con.execute(
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_schema = 'opps' AND table_name = 'addendum_b' "
|
||||
"AND column_name = 'status_indicator'"
|
||||
).fetchone()
|
||||
if not has_si:
|
||||
return 0
|
||||
con.execute("CREATE SCHEMA IF NOT EXISTS opps")
|
||||
con.execute("DROP TABLE IF EXISTS opps.pass_through")
|
||||
con.execute(
|
||||
"""
|
||||
CREATE TABLE opps.pass_through AS
|
||||
SELECT
|
||||
TRIM(CAST(hcpcs AS VARCHAR)) AS hcpcs,
|
||||
short_description,
|
||||
CASE UPPER(TRIM(CAST(status_indicator AS VARCHAR)))
|
||||
WHEN 'G' THEN 'drug'
|
||||
WHEN 'H' THEN 'device'
|
||||
END AS pass_through_type,
|
||||
UPPER(TRIM(CAST(status_indicator AS VARCHAR))) AS status_indicator,
|
||||
payment_rate,
|
||||
CAST(NULL AS DOUBLE) AS asp_per_unit,
|
||||
year
|
||||
FROM opps.addendum_b
|
||||
WHERE UPPER(TRIM(CAST(status_indicator AS VARCHAR))) IN ('G', 'H')
|
||||
"""
|
||||
)
|
||||
n = con.execute("SELECT count(*) FROM opps.pass_through").fetchone()[0]
|
||||
print(f" opps.pass_through: {n} rows (Addendum B SI G and H)")
|
||||
return int(n)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Ingest CMS OPPS files")
|
||||
|
||||
@@ -50,11 +50,18 @@ The free knobs are a tag confidence in [0, 1], a REACH quality score in
|
||||
window is whichever periods the file lists.
|
||||
|
||||
`pay.bounds.allows` rejects a point outside that row. It also rejects
|
||||
proposed NPRM numbers, an unstated null (nothing is imputed), a missing
|
||||
wage index, a missing equipment maintenance factor, and a
|
||||
limiting-charge fee, which is an output of `pfsPayment`. IPPS and LEAD
|
||||
have no pricer. MSSP and REACH settlements stay published tables. The
|
||||
budget-neutrality adjustor is already inside the conversion factor.
|
||||
proposed NPRM numbers, an unstated null (nothing is imputed), and a
|
||||
limiting-charge fee, which is an output of `pfsPayment`. The OPPS wage
|
||||
index is the IPPS final-rule table for that fiscal year (Table 3, or
|
||||
Table 4A and 4B in FY2014 and FY2015). A blank pre-reclassification
|
||||
index stays blank. Equipment maintenance is the published factor 5/100
|
||||
for every item; the straight-line function still fills a missing factor
|
||||
with 0, and that fill is not the CMS factor. There is no IPPS claim
|
||||
pricer and no LEAD pricer. Standardized amounts and MS-DRG weights are
|
||||
published points and ranges. MSSP and REACH settlements stay published
|
||||
tables; the observed dollar columns are separate rows, and there is no
|
||||
savings function to sample. The budget-neutrality adjustor is already
|
||||
inside the conversion factor.
|
||||
|
||||
Rebuild the file from the registries and `data/replica/aco.ro.duckdb`
|
||||
when either changes. A checkout without the replica still checks the
|
||||
|
||||
@@ -71,9 +71,9 @@ CMS Data Files
|
||||
│ │ │ │ minimum unadjusted│
|
||||
│ │ │ │ copayment │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ Addendum D │ Annual │ CSV/XLSX │ HCPCS, pass-thru │
|
||||
│ (Pass-through) │ w/ rule │ │ device/drug/bio, │
|
||||
│ │ │ │ payment amount │
|
||||
│ Addendum B, SI G/H │ Annual │ CSV/XLSX │ HCPCS, pass-thru │
|
||||
│ (Pass-through) │ w/ rule │ │ drug (G) / device │
|
||||
│ │ │ │ (H), payment rate │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ Wage Index │ Annual │ CSV │ CBSA, wage index │
|
||||
│ │ │ │ (pre/post reclss) │
|
||||
|
||||
@@ -4,7 +4,9 @@ Separately payable items that bypass APC packaging. Prior to
|
||||
Jan 2026, skin substitutes were paid as pass-through biologicals
|
||||
at ASP+6%. The reclassification moved them to incident-to supplies.
|
||||
|
||||
Source: Addendum D from CMS OPPS Final Rule downloads.
|
||||
Source: Addendum B rows whose status indicator is G (drugs and
|
||||
biologicals) or H (devices). Addendum D1 and D2 define indicators;
|
||||
they are not the payment-rate file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -13,7 +15,7 @@ from conf.table_base import SQLTable
|
||||
|
||||
|
||||
class PassThrough(SQLTable):
|
||||
"""Pass-through payment items (Addendum D)."""
|
||||
"""Pass-through payment items (Addendum B, status indicator G or H)."""
|
||||
|
||||
__schema__ = "opps"
|
||||
__tablename__ = "pass_through"
|
||||
|
||||
7498
src/pay/bounds.json
7498
src/pay/bounds.json
File diff suppressed because it is too large
Load Diff
@@ -91,7 +91,17 @@ _CONSTANTS = (
|
||||
"maintenanceDefault",
|
||||
0,
|
||||
1,
|
||||
"Fill used when maintenance_factor is absent.",
|
||||
"Fill used by the straight-line equipment function when"
|
||||
" maintenance_factor is absent. This is not the CMS factor."
|
||||
" CMS publishes maintenanceFactor.",
|
||||
),
|
||||
(
|
||||
"maintenanceFactor",
|
||||
5,
|
||||
100,
|
||||
"CMS maintenance factor for every equipment item. The PUF has"
|
||||
" no per-item column. The straight-line function does not apply"
|
||||
" this factor.",
|
||||
),
|
||||
("mpprReduction", 1, 2, "Default PE reduction for rank 2 and later."),
|
||||
("laborShare", 3, 5, "Default OPPS labor share, 0.60."),
|
||||
@@ -159,6 +169,7 @@ SPECS = (
|
||||
"nf_minutes",
|
||||
"f_minutes",
|
||||
"maintenance_factor",
|
||||
"utilization_rate",
|
||||
),
|
||||
"payment",
|
||||
),
|
||||
@@ -220,6 +231,49 @@ SPECS = (
|
||||
_Spec("drugs.asp_payment", ("quarter",), ("payment_limit",), "payment"),
|
||||
_Spec("drugs.asp_payment", ("quarter",), ("coinsurance_pct",), "outside"),
|
||||
_Spec("drugs.asp_noc", ("quarter",), ("payment_limit",), "payment"),
|
||||
_Spec("ipps.drg_weight", ("fiscal_year",), ("relative_weight",), "payment"),
|
||||
_Spec(
|
||||
"ipps.drg_weight",
|
||||
("fiscal_year",),
|
||||
("geometric_mean_los", "arithmetic_mean_los"),
|
||||
"outside",
|
||||
),
|
||||
_Spec(
|
||||
"ipps.standardized_amount",
|
||||
("fiscal_year", "table_code", "component"),
|
||||
("amount",),
|
||||
"payment",
|
||||
),
|
||||
_Spec(
|
||||
"cms.performance_year_financial_quality_results",
|
||||
("perf_year",),
|
||||
(
|
||||
"earnsaveloss",
|
||||
"gensaveloss",
|
||||
"abtotbnchmk",
|
||||
"abtotexp",
|
||||
"updatedbnchmk",
|
||||
"histbnchmk",
|
||||
"sav_rate",
|
||||
"qualscore",
|
||||
"finalsharerate",
|
||||
),
|
||||
"payment",
|
||||
),
|
||||
_Spec(
|
||||
"cms.aco_reach_financial_quality_results",
|
||||
("perf_year",),
|
||||
(
|
||||
"shared_savings",
|
||||
"gross_savings",
|
||||
"final_benchmark",
|
||||
"unadj_benchmark",
|
||||
"tot_cost_care",
|
||||
"tot_cost_care_net",
|
||||
"sav_rate",
|
||||
),
|
||||
"payment",
|
||||
),
|
||||
)
|
||||
|
||||
# Output tables and the derived ASP trajectory. Named so a new numeric
|
||||
@@ -272,16 +326,31 @@ _MISSING_NOTES = {
|
||||
"No opps.wage_index table. oppsAdjusted does not read this column."
|
||||
),
|
||||
("pfs.medical_equipment", "maintenance_factor"): (
|
||||
"Column is not loaded. equipmentCostPerMinute uses 0."
|
||||
"The PUF has no maintenance column. CMS applies 5/100 to every"
|
||||
" item (lean.maintenanceFactor). The straight-line function fills"
|
||||
" a missing factor with 0, which is not that factor."
|
||||
),
|
||||
("opps.pass_through", "payment_rate"): (
|
||||
"Addendum D is not in the replica. Do not invent a pass-through rate."
|
||||
"No opps.pass_through table. Rates are Addendum B status"
|
||||
" indicators G and H. Do not invent a rate."
|
||||
),
|
||||
("opps.pass_through", "asp_per_unit"): (
|
||||
"Addendum D is not in the replica. Do not invent an ASP."
|
||||
"No opps.pass_through table. Do not divide a payment rate by 1.06."
|
||||
),
|
||||
}
|
||||
|
||||
# SBA interest rates CMS reprints for the equipment annuity. Unchanged
|
||||
# since CY2013 (77 FR 68902). The CY2027 final rule uses the same six
|
||||
# rates. Percents, so 7.50% is 750/10000. Price band, then life band.
|
||||
_INTEREST_RATES = (
|
||||
("<25000", "<7", 750, 10000),
|
||||
("25000-50000", "<7", 650, 10000),
|
||||
(">50000", "<7", 550, 10000),
|
||||
("<25000", "7+", 800, 10000),
|
||||
("25000-50000", "7+", 700, 10000),
|
||||
(">50000", "7+", 600, 10000),
|
||||
)
|
||||
|
||||
_NUMERIC_TYPES = frozenset(
|
||||
{
|
||||
"DOUBLE",
|
||||
@@ -364,6 +433,7 @@ def build(con: Any = None) -> dict[str, Any]:
|
||||
rows = [
|
||||
*registry_rows(),
|
||||
*constant_rows(),
|
||||
*interest_rows(),
|
||||
*ceiling_rows(),
|
||||
*hole_rows(),
|
||||
*observed_rows(con),
|
||||
@@ -431,6 +501,31 @@ def constant_rows() -> list[dict[str, Any]]:
|
||||
return rows
|
||||
|
||||
|
||||
def interest_rows() -> list[dict[str, Any]]:
|
||||
"""The six published SBA interest rates. Points, not a sampled range."""
|
||||
rows = []
|
||||
for price, life, numerator, denominator in _INTEREST_RATES:
|
||||
point = num(float(Fraction(numerator, denominator)))
|
||||
rows.append(
|
||||
element(
|
||||
item="pfs.equipment.interest",
|
||||
name="rate",
|
||||
period=f"{price}:{life}",
|
||||
kind="published",
|
||||
role="payment",
|
||||
low=point,
|
||||
high=point,
|
||||
value=f"{numerator}/{denominator}",
|
||||
note=(
|
||||
"SBA interest rate for this purchase-price band and"
|
||||
" useful life. Unchanged since CY2013 (77 FR 68902)."
|
||||
" The straight-line equipment function does not apply it."
|
||||
),
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def ceiling_rows() -> list[dict[str, Any]]:
|
||||
"""The eight Lean methods and the only legal label for each."""
|
||||
return [
|
||||
@@ -513,7 +608,8 @@ def hole_rows() -> list[dict[str, Any]]:
|
||||
kind="published_table",
|
||||
role="payment",
|
||||
note=(
|
||||
"Shared savings are a published CMS table."
|
||||
"Shared savings are the published CMS table"
|
||||
" cms.aco_reach_financial_quality_results."
|
||||
" There is no savings function to sample."
|
||||
),
|
||||
),
|
||||
@@ -523,7 +619,8 @@ def hole_rows() -> list[dict[str, Any]]:
|
||||
kind="published_table",
|
||||
role="payment",
|
||||
note=(
|
||||
"Shared savings are a published CMS table."
|
||||
"Shared savings are the published CMS table"
|
||||
" cms.performance_year_financial_quality_results."
|
||||
" There is no savings function to sample."
|
||||
),
|
||||
),
|
||||
@@ -532,7 +629,11 @@ def hole_rows() -> list[dict[str, Any]]:
|
||||
name="payment",
|
||||
kind="absent",
|
||||
role="payment",
|
||||
note="No IPPS pricer in this repo.",
|
||||
note=(
|
||||
"No IPPS pricer in this repo. Claim-level IME, DSH, and"
|
||||
" outliers are not priced. Published standardized amounts"
|
||||
" and MS-DRG weights are separate rows."
|
||||
),
|
||||
),
|
||||
element(
|
||||
item="lead",
|
||||
@@ -777,6 +878,12 @@ def _observed(
|
||||
) -> dict[str, Any]:
|
||||
seen = int(count or 0)
|
||||
if low is None or high is None or seen == 0:
|
||||
note = "No non-null values in this period. No range is invented."
|
||||
if spec.table == "opps.pass_through" and column == "asp_per_unit":
|
||||
note = (
|
||||
"No ASP is joined. payment_rate is the published"
|
||||
" pass-through amount. Do not divide it by 1.06."
|
||||
)
|
||||
return element(
|
||||
item=spec.table,
|
||||
name=column,
|
||||
@@ -784,7 +891,7 @@ def _observed(
|
||||
kind="unstated",
|
||||
role=spec.role,
|
||||
rows=seen,
|
||||
note="No non-null values in this period. No range is invented.",
|
||||
note=note,
|
||||
)
|
||||
kind = "proposed" if spec.role == "proposed" else "observed"
|
||||
return element(
|
||||
@@ -815,6 +922,37 @@ def _observed_note(spec: _Spec, column: str, kind: str, low: str, period: str) -
|
||||
notes.append("CMS payment allowance (ASP + 6%), not the ASP itself.")
|
||||
elif spec.table == "drugs.asp_noc" and column == "payment_limit":
|
||||
notes.append("CMS payment allowance for a not-otherwise-classified drug.")
|
||||
elif spec.table == "opps.pass_through" and column == "payment_rate":
|
||||
notes.append(
|
||||
"Addendum B status indicator G (drug) or H (device). Not Addendum D."
|
||||
)
|
||||
elif spec.table == "opps.wage_index" and column == "wage_index":
|
||||
notes.append(
|
||||
"IPPS final-rule Table 3 pre-reclassification wage index."
|
||||
" FY2014 and FY2015 use Table 4A and 4B."
|
||||
" Calendar year N uses fiscal year N."
|
||||
)
|
||||
elif spec.table == "ipps.drg_weight" and column == "relative_weight":
|
||||
notes.append(
|
||||
"Paid MS-DRG weight. The 10% cap-applied weight when CMS publishes both."
|
||||
)
|
||||
elif spec.table == "ipps.standardized_amount":
|
||||
notes.append("One published dollar. The row is that point.")
|
||||
if period.split(":")[1:2] == ["1E"]:
|
||||
notes.append(
|
||||
"LTCH PPS standard federal rate. Not an IPPS operating amount."
|
||||
)
|
||||
elif (
|
||||
spec.table == "cms.performance_year_financial_quality_results"
|
||||
and column == "qualscore"
|
||||
):
|
||||
notes.append(
|
||||
"Published score on the file's scale. Not the REACH total_quality knob."
|
||||
)
|
||||
elif spec.table == "pfs.medical_equipment" and column == "utilization_rate":
|
||||
notes.append(
|
||||
"CMS equipment usage rate. The straight-line function does not read it."
|
||||
)
|
||||
if column == "work_gpci" and kind in {"observed", "proposed"}:
|
||||
year = _year(period)
|
||||
floor = PFS_RULES[year].work_gpci_floor if year in PFS_RULES else 1.0
|
||||
|
||||
@@ -164,7 +164,7 @@ _EQUIPMENT_COLUMNS = {
|
||||
"description": "equipment_description",
|
||||
"useful_life": "useful_life",
|
||||
"price": "unit_price",
|
||||
"utilization_rate": "_utilization_rate",
|
||||
"utilization_rate": "utilization_rate",
|
||||
"minutes_per_year": "minutes_per_year",
|
||||
"nf_time": "nf_minutes",
|
||||
"f_time": "f_minutes",
|
||||
@@ -1073,6 +1073,11 @@ def _load_gpci_files(
|
||||
# Normalize column names for non-stacked files
|
||||
if "work_gpci" not in df.columns:
|
||||
rename_map: dict[str, str] = {}
|
||||
# A "without 1.0 floor" work column is the fallback, used
|
||||
# only when the file publishes no other work GPCI. CY2026
|
||||
# publishes both and the with-floor column stays the one
|
||||
# that loads. CY2027 publishes only the without-floor column.
|
||||
work_without: str | None = None
|
||||
yr_str = str(year)
|
||||
for col in df.columns:
|
||||
# Collapse all whitespace (handles "Work GPCI ")
|
||||
@@ -1089,6 +1094,14 @@ def _load_gpci_files(
|
||||
rename_map[col] = "locality_name"
|
||||
elif "LOCALITY" in norm:
|
||||
rename_map[col] = "locality"
|
||||
elif ("PW GPCI" in norm or "WORK GPCI" in norm) and (
|
||||
"WITHOUT" in norm
|
||||
):
|
||||
yr_match = re.match(r"(\d{4})\s", norm)
|
||||
if yr_match and yr_match.group(1) != yr_str:
|
||||
continue
|
||||
if work_without is None:
|
||||
work_without = col
|
||||
elif "WITHOUT" in norm:
|
||||
continue # skip without-floor variants
|
||||
# Year-prefixed: "2020 PW GPCI ..." — only match target year
|
||||
@@ -1111,6 +1124,9 @@ def _load_gpci_files(
|
||||
if "mp_gpci" not in rename_map.values():
|
||||
rename_map[col] = "mp_gpci"
|
||||
|
||||
if "work_gpci" not in rename_map.values() and work_without is not None:
|
||||
rename_map[work_without] = "work_gpci"
|
||||
|
||||
if "locality" not in rename_map.values():
|
||||
log.warning(
|
||||
"Skipping GPCI %s — no locality column",
|
||||
@@ -1382,6 +1398,7 @@ def _load_pe_files(
|
||||
"nf_minutes",
|
||||
"f_minutes",
|
||||
"maintenance_factor",
|
||||
"utilization_rate",
|
||||
]:
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
@@ -1399,6 +1416,7 @@ def _load_pe_files(
|
||||
"useful_life",
|
||||
"minutes_per_year",
|
||||
"maintenance_factor",
|
||||
"utilization_rate",
|
||||
"year",
|
||||
]
|
||||
if c in df.columns
|
||||
|
||||
@@ -43,8 +43,16 @@ class MedicalEquipment(SQLTable):
|
||||
"""Annual utilization minutes (typically derived from
|
||||
practice survey data)."""
|
||||
|
||||
utilization_rate: float | None = None
|
||||
"""CMS equipment usage rate from the direct PE input file.
|
||||
The straight-line cost function does not read it. CMS's
|
||||
annuity formula uses it as usage, with 0.50 for most items
|
||||
and 0.90 for expensive imaging under section 1848(b)(4)(C)."""
|
||||
|
||||
maintenance_factor: float | None = None
|
||||
"""Annual maintenance cost as fraction of purchase price."""
|
||||
"""Annual maintenance cost as a fraction of purchase price.
|
||||
The PUF has no per-item column. CMS publishes 0.05 for every
|
||||
item. The straight-line function fills a missing factor with 0."""
|
||||
|
||||
year: int | None = None
|
||||
"""Calendar year."""
|
||||
|
||||
161
tests/dev/test_ingest_ipps_tables.py
Normal file
161
tests/dev/test_ingest_ipps_tables.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""Published IPPS table readers. Synthetic files, so CI has no CMS dump."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import io
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import openpyxl
|
||||
|
||||
_SCRIPT = (
|
||||
Path(__file__).resolve().parents[2] / "dev" / "scripts" / "ingest_ipps_tables.py"
|
||||
)
|
||||
_spec = importlib.util.spec_from_file_location("ingest_ipps_tables", _SCRIPT)
|
||||
assert _spec is not None and _spec.loader is not None
|
||||
ipps = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["ingest_ipps_tables"] = ipps
|
||||
_spec.loader.exec_module(ipps)
|
||||
|
||||
|
||||
def test_member_kind_is_the_table_id_not_a_digit_in_the_rule_number() -> None:
|
||||
assert ipps._member_kind("CMS-1833-F Tables 3.txt") == "table3"
|
||||
assert ipps._member_kind("CMS-1632-F Table 3.txt") == "table3"
|
||||
assert ipps._member_kind("FY 2014 FR Table 3A.txt") is None
|
||||
assert ipps._member_kind("CMS-1735-CN Table 2.txt") is None
|
||||
assert ipps._member_kind("CMS_1632_CN_Table_5.txt") == "table5"
|
||||
assert ipps._member_kind("CMS-1833-F Tables 4a.txt") == "table4a"
|
||||
|
||||
|
||||
def test_the_reclassified_index_is_not_the_puerto_rico_column() -> None:
|
||||
text = (
|
||||
"CBSA\tArea Name\tState\tWage Index\tReclassified Wage Index\t"
|
||||
"Reclassified Puerto Rico Specific Wage Index\n"
|
||||
"01\tAlabama\tAL\t0.7393\t0.8000\t1.1111\n"
|
||||
)
|
||||
frame = ipps.parse_wage_text(text, 2026)
|
||||
row = frame.iloc[0]
|
||||
assert row["wage_index"] == 0.7393
|
||||
assert row["reclassified_wage_index"] == 0.8
|
||||
|
||||
|
||||
def test_a_short_correction_does_not_replace_the_full_table(tmp_path: Path) -> None:
|
||||
full = _wage_text([("10180", "TX", "0.8464")] * 10)
|
||||
short = _wage_text([("99999", "MA", "0.5000")])
|
||||
path = tmp_path / "wage.zip"
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
zf.writestr("FR Table 3.txt", full)
|
||||
zf.writestr("CN Table 3.txt", short)
|
||||
frame = ipps.read_wage_year(path, 2014)
|
||||
assert len(frame) == 10
|
||||
assert set(frame["wage_index"]) == {0.8464}
|
||||
|
||||
|
||||
def test_a_full_correction_reprint_replaces_the_final_rule(tmp_path: Path) -> None:
|
||||
final = _wage_text([("10180", "TX", "0.8464")] * 10)
|
||||
corrected = _wage_text([("10180", "TX", "0.8500")] * 10)
|
||||
path = tmp_path / "wage.zip"
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
zf.writestr("FR Table 3.txt", final)
|
||||
zf.writestr("CN Table 3.txt", corrected)
|
||||
frame = ipps.read_wage_year(path, 2014)
|
||||
assert set(frame["wage_index"]) == {0.85}
|
||||
|
||||
|
||||
def test_the_cap_applied_weight_is_the_paid_weight() -> None:
|
||||
capped = (
|
||||
"MS-DRG\tWeights\tWeights - 10% Cap Applied\t"
|
||||
"Geometric Mean LOS\tArithmetic Mean LOS\n"
|
||||
"001\t1.5\t1.2\t2.0\t3.0\n"
|
||||
)
|
||||
frame = ipps.parse_drg_text(capped, 2026)
|
||||
assert frame.iloc[0]["relative_weight"] == 1.2
|
||||
plain = "MS-DRG\tWeights\tGeometric Mean LOS\tArithmetic Mean LOS\n001\t1.5\t2\t3\n"
|
||||
assert ipps.parse_drg_text(plain, 2026).iloc[0]["relative_weight"] == 1.5
|
||||
|
||||
|
||||
def test_standardized_amounts_are_the_published_points() -> None:
|
||||
grid = [
|
||||
["TABLE 1A"],
|
||||
[None, "Submitted quality data and is a meaningful EHR user", None],
|
||||
[None, "Labor", "Nonlabor"],
|
||||
["National", 4456.72, "Not Applicable"],
|
||||
["TABLE 1D"],
|
||||
[None, "Rate"],
|
||||
["National", 524.15],
|
||||
]
|
||||
frame = ipps.parse_standardized_xlsx(_workbook({"FY 2026 Table 1A-1E": grid}), 2026)
|
||||
got = {(row.table_code, row.component): row.amount for row in frame.itertuples()}
|
||||
assert got["1A", "national_quality_ehr_labor"] == 4456.72
|
||||
assert got["1D", "national_rate"] == 524.15
|
||||
assert all(amount != 0 for amount in got.values())
|
||||
assert len(got) == 2
|
||||
|
||||
|
||||
def test_a_full_later_reprint_wins_and_a_short_one_does_not() -> None:
|
||||
full = [
|
||||
["TABLE 1A"],
|
||||
[None, "Labor", "Nonlabor"],
|
||||
["National", 10, 11],
|
||||
]
|
||||
short = [
|
||||
["TABLE 1A"],
|
||||
[None, "Labor"],
|
||||
["National", 99],
|
||||
]
|
||||
kept = ipps.parse_standardized_xlsx(
|
||||
_workbook({"FR Table 1A-1E": full, "CN Table 1A-1E": short}),
|
||||
2016,
|
||||
)
|
||||
assert set(kept["amount"]) == {10, 11}
|
||||
|
||||
corrected = [
|
||||
["TABLE 1A"],
|
||||
[None, "Labor", "Nonlabor"],
|
||||
["National", 20, 21],
|
||||
]
|
||||
kept = ipps.parse_standardized_xlsx(
|
||||
_workbook({"FR Table 1A-1E": full, "CN Table 1A-1E": corrected}),
|
||||
2016,
|
||||
)
|
||||
assert set(kept["amount"]) == {20, 21}
|
||||
|
||||
caa = [
|
||||
["TABLE 1A"],
|
||||
[None, "Labor", "Nonlabor"],
|
||||
["National", 30, 31],
|
||||
]
|
||||
kept = ipps.parse_standardized_xlsx(
|
||||
_workbook({"CN Table 1A-1E": corrected, "CAA Table 1A-1E": caa}),
|
||||
2016,
|
||||
)
|
||||
assert set(kept["amount"]) == {30, 31}
|
||||
|
||||
|
||||
def test_a_percent_encoded_space_is_not_the_performance_year() -> None:
|
||||
assert ipps._perf_year_from_filename("PY%202023%20ACO%20Results%20PUF.csv") == 2023
|
||||
name = "PY_Financial_and_Quality_Results_2024_revised%202026_07_17.csv"
|
||||
assert ipps._perf_year_from_filename(name) == 2024
|
||||
|
||||
|
||||
def _wage_text(rows: list[tuple[str, str, str]]) -> str:
|
||||
lines = ["CBSA\tArea Name\tState\tWage Index"]
|
||||
lines.extend(f"{cbsa}\tArea\t{state}\t{index}" for cbsa, state, index in rows)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _workbook(sheets: dict[str, list[list[object]]]) -> bytes:
|
||||
book = openpyxl.Workbook()
|
||||
first = True
|
||||
for title, rows in sheets.items():
|
||||
sheet = book.active if first else book.create_sheet()
|
||||
first = False
|
||||
sheet.title = title
|
||||
for row in rows:
|
||||
sheet.append(row)
|
||||
buf = io.BytesIO()
|
||||
book.save(buf)
|
||||
book.close()
|
||||
return buf.getvalue()
|
||||
122
tests/dev/test_ingest_opps_sheet.py
Normal file
122
tests/dev/test_ingest_opps_sheet.py
Normal file
@@ -0,0 +1,122 @@
|
||||
"""Addendum A workbooks sometimes hide the payment sheet behind a lookup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import io
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
_SCRIPT = Path(__file__).resolve().parents[2] / "dev" / "scripts" / "ingest_opps.py"
|
||||
_spec = importlib.util.spec_from_file_location("ingest_opps", _SCRIPT)
|
||||
assert _spec is not None and _spec.loader is not None
|
||||
ingest_opps = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["ingest_opps"] = ingest_opps
|
||||
_spec.loader.exec_module(ingest_opps)
|
||||
|
||||
|
||||
def _zip_with(dest: Path, sheets: dict[str, pd.DataFrame]) -> Path:
|
||||
buf = io.BytesIO()
|
||||
with pd.ExcelWriter(buf, engine="openpyxl") as writer:
|
||||
for name, frame in sheets.items():
|
||||
frame.to_excel(writer, sheet_name=name, index=False, header=False)
|
||||
with zipfile.ZipFile(dest, "w") as zf:
|
||||
zf.writestr("Addendum A.xlsx", buf.getvalue())
|
||||
return dest
|
||||
|
||||
|
||||
def test_payment_sheet_wins_over_the_category_lookup(tmp_path: Path) -> None:
|
||||
category = pd.DataFrame(
|
||||
[
|
||||
["APC Category Names", None],
|
||||
["APC", "Description"],
|
||||
["0001", "Level 1"],
|
||||
]
|
||||
)
|
||||
payment = pd.DataFrame(
|
||||
[
|
||||
["CY 2015 Addendum A", None, None, None, None, None, None],
|
||||
[
|
||||
"APC",
|
||||
"Group Title",
|
||||
"SI",
|
||||
"Relative Weight",
|
||||
"Payment Rate",
|
||||
"National Unadjusted Copayment",
|
||||
"Minimum Unadjusted Copayment",
|
||||
],
|
||||
["0701", "Sr89 strontium", "K", "2045.702", "100.00", "20", "10"],
|
||||
]
|
||||
)
|
||||
zpath = _zip_with(
|
||||
tmp_path / "addenda.zip",
|
||||
{"APC Category Names New": category, "2015 FR Addendum A": payment},
|
||||
)
|
||||
df = ingest_opps.read_from_zip(
|
||||
zpath, ingest_opps.ADDENDUM_A_COL_MAP, pattern="addendum a"
|
||||
)
|
||||
assert list(df["apc"]) == ["0701"]
|
||||
assert list(df["relative_weight"]) == ["2045.702"]
|
||||
|
||||
|
||||
def test_the_first_sheet_is_used_when_it_is_the_payment_sheet(tmp_path: Path) -> None:
|
||||
payment = pd.DataFrame(
|
||||
[
|
||||
[
|
||||
"APC",
|
||||
"Group Title",
|
||||
"SI",
|
||||
"Relative Weight",
|
||||
"Payment Rate",
|
||||
"Minimum Unadjusted Copayment",
|
||||
],
|
||||
["5071", "Level 1", "T", "1.234", "80.00", "16"],
|
||||
]
|
||||
)
|
||||
zpath = _zip_with(tmp_path / "addenda.zip", {"2018 FR Addendum A": payment})
|
||||
df = ingest_opps.read_from_zip(
|
||||
zpath, ingest_opps.ADDENDUM_A_COL_MAP, pattern="addendum a"
|
||||
)
|
||||
assert list(df["apc"]) == ["5071"]
|
||||
assert list(df["payment_rate"]) == ["80.00"]
|
||||
|
||||
|
||||
def test_pass_through_copies_status_indicators_g_and_h() -> None:
|
||||
import duckdb
|
||||
|
||||
con = duckdb.connect()
|
||||
con.execute("CREATE SCHEMA opps")
|
||||
con.execute(
|
||||
"""
|
||||
CREATE TABLE opps.addendum_b (
|
||||
hcpcs VARCHAR,
|
||||
short_description VARCHAR,
|
||||
status_indicator VARCHAR,
|
||||
payment_rate DOUBLE,
|
||||
year INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
con.execute(
|
||||
"""
|
||||
INSERT INTO opps.addendum_b VALUES
|
||||
('J0120', 'drug', 'G', 12.5, 2024),
|
||||
('C1747', 'device', 'H', NULL, 2024),
|
||||
('J9999', 'separately payable', 'K', 40.0, 2024),
|
||||
('99213', 'visit', 'J1', 80.0, 2024)
|
||||
"""
|
||||
)
|
||||
n = ingest_opps.load_pass_through(con)
|
||||
assert n == 2
|
||||
rows = con.execute(
|
||||
"SELECT hcpcs, pass_through_type, status_indicator, payment_rate, "
|
||||
"asp_per_unit FROM opps.pass_through ORDER BY hcpcs"
|
||||
).fetchall()
|
||||
assert rows == [
|
||||
("C1747", "device", "H", None, None),
|
||||
("J0120", "drug", "G", 12.5, None),
|
||||
]
|
||||
con.close()
|
||||
@@ -163,6 +163,8 @@ def test_lean_constants_match_the_definitions() -> None:
|
||||
assert Fraction(by_name["passThroughDrugs"]["value"]) == 1 + Fraction(6, 100)
|
||||
assert by_name["missingWageIndex"]["value"] == "1/1"
|
||||
assert by_name["maintenanceDefault"]["value"] == "0/1"
|
||||
assert by_name["maintenanceFactor"]["value"] == "5/100"
|
||||
assert by_name["maintenanceFactor"]["low"] == bounds.num(0.05)
|
||||
assert by_name["mpprReduction"]["value"] == "1/2"
|
||||
assert by_name["paymentYearOffset"]["value"] == "2/1"
|
||||
assert bounds.num(PAR_REDUCTION) == by_name["parReduction"]["low"]
|
||||
@@ -400,17 +402,27 @@ def test_a_work_gpci_below_the_floor_stays_in_the_catalog() -> None:
|
||||
and float(row["low"]) < 1
|
||||
]
|
||||
assert below
|
||||
for row in below:
|
||||
assert row["kind"] == "proposed"
|
||||
proposed = [row for row in below if row["kind"] == "proposed"]
|
||||
observed = [row for row in below if row["item"] == "pfs.gpci"]
|
||||
assert proposed
|
||||
assert observed
|
||||
for row in proposed:
|
||||
assert row["role"] == "proposed"
|
||||
assert "below" in row["note"]
|
||||
assert not bounds.allows(
|
||||
row["item"], row["name"], float(row["low"]), period=row["period"]
|
||||
)
|
||||
for row in observed:
|
||||
assert row["kind"] == "observed"
|
||||
assert row["role"] == "payment"
|
||||
assert "floor" in row["note"]
|
||||
assert bounds.allows(
|
||||
row["item"], row["name"], float(row["low"]), period=row["period"]
|
||||
)
|
||||
final = index(bounds.load())["pfs.gpci", "work_gpci", "2027"]
|
||||
assert final["kind"] == "unstated"
|
||||
assert final["rows"] == 0
|
||||
assert not bounds.allows("pfs.gpci", "work_gpci", 1, period=2027)
|
||||
assert final["kind"] == "observed"
|
||||
assert final["low"] == "0.948"
|
||||
assert bounds.allows("pfs.gpci", "work_gpci", 0.948, period="2027")
|
||||
|
||||
|
||||
def test_outside_columns_are_recorded_and_rejected() -> None:
|
||||
@@ -598,7 +610,9 @@ def test_a_small_table_sets_the_period_fence() -> None:
|
||||
assert (
|
||||
got["pfs.medical_equipment", "maintenance_factor", ""]["kind"] == "unobserved"
|
||||
)
|
||||
assert "uses 0" in got["pfs.medical_equipment", "maintenance_factor", ""]["note"]
|
||||
maint_note = got["pfs.medical_equipment", "maintenance_factor", ""]["note"]
|
||||
assert "no maintenance column" in maint_note
|
||||
assert "not that factor" in maint_note
|
||||
assert got["opps.wage_index", "wage_index", ""]["kind"] == "unobserved"
|
||||
assert got["drugs.asp_noc", "payment_limit", ""]["kind"] == "unstated"
|
||||
assert got["drugs.asp_noc", "payment_limit", ""]["rows"] == 0
|
||||
@@ -646,7 +660,14 @@ def test_snapshot_matches_a_fresh_read_of_the_replica() -> None:
|
||||
|
||||
@pytest.mark.skipif(not bounds.REPLICA.is_file(), reason="replica not on disk")
|
||||
def test_every_numeric_column_of_a_fenced_table_is_an_element() -> None:
|
||||
period = {"year", "quarter", "gpci_year", "cms_rule_id"}
|
||||
period = {
|
||||
"year",
|
||||
"quarter",
|
||||
"gpci_year",
|
||||
"cms_rule_id",
|
||||
"fiscal_year",
|
||||
"perf_year",
|
||||
}
|
||||
tables = {spec.table for spec in bounds.SPECS} | {
|
||||
item[0] for item in bounds.DERIVED
|
||||
}
|
||||
|
||||
@@ -209,6 +209,27 @@ class TestLoadGpciFilesEdgeCases:
|
||||
with patch("pfs.pipe._is_stacked_header_gpci", return_value=False):
|
||||
result = _load_gpci_files([f], con)
|
||||
assert result["rows"] == 1
|
||||
df = con.execute("SELECT work_gpci FROM pfs.gpci").pl()
|
||||
assert df["work_gpci"][0] == pytest.approx(1.024)
|
||||
|
||||
def test_gpci_without_floor_is_used_when_it_is_the_only_work_column(
|
||||
self, con, tmp_path
|
||||
):
|
||||
"""CY2027 publishes only the without-floor work GPCI. Load that number."""
|
||||
p = tmp_path / "gpci_2027.txt"
|
||||
p.write_text(
|
||||
"MAC\tLOCALITY\tLOCALITY NAME\tSTATE\t"
|
||||
"2027 PW GPCI (without 1.0 Floor)\t2027 PE GPCI\t2027 MP GPCI\n"
|
||||
"01112\t01\tBirmingham\tAL\t0.975\t0.880\t0.557\n"
|
||||
)
|
||||
f = _f("Addendum_E_GPCI_2027.txt", 2027, ".txt", path=str(p))
|
||||
with patch("pfs.pipe._is_stacked_header_gpci", return_value=False):
|
||||
result = _load_gpci_files([f], con)
|
||||
assert result["rows"] == 1
|
||||
df = con.execute("SELECT work_gpci, pe_gpci, mp_gpci FROM pfs.gpci").pl()
|
||||
assert df["work_gpci"][0] == pytest.approx(0.975)
|
||||
assert df["pe_gpci"][0] == pytest.approx(0.880)
|
||||
assert df["mp_gpci"][0] == pytest.approx(0.557)
|
||||
|
||||
def test_gpci_year_prefix_mismatch_skip(self, con, tmp_path):
|
||||
"""Lines 952-953, 958-959, 964-965: year-prefixed cols for wrong year skipped."""
|
||||
|
||||
@@ -798,6 +798,7 @@ class TestLoadPeFiles:
|
||||
"description": ["Exam table"],
|
||||
"useful_life": ["10"],
|
||||
"price": ["5000"],
|
||||
"utilization_rate": ["0.5"],
|
||||
"nf_time": ["30"],
|
||||
"f_time": ["15"],
|
||||
}
|
||||
@@ -812,6 +813,10 @@ class TestLoadPeFiles:
|
||||
]
|
||||
result = _load_pe_files(files, con)
|
||||
assert result["medical_equipment"]["rows"] == 1
|
||||
rate = con.execute(
|
||||
'SELECT utilization_rate FROM "pfs"."medical_equipment"'
|
||||
).fetchone()[0]
|
||||
assert rate == 0.5
|
||||
|
||||
@patch("pfs.pipe._read_tabular")
|
||||
def test_work_time_hcpcs_int_padding(self, mock_read, con):
|
||||
|
||||
@@ -180,6 +180,7 @@ _TABLE_SPECS = [
|
||||
"unit_price",
|
||||
"useful_life",
|
||||
"minutes_per_year",
|
||||
"utilization_rate",
|
||||
"maintenance_factor",
|
||||
"year",
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user