794 lines
30 KiB
Python
794 lines
30 KiB
Python
"""Supplementary tests for pfs.pipe — covers remaining uncovered lines."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import types
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import duckdb
|
|
import polars as pl
|
|
import pytest
|
|
|
|
from pfs.pipe import (
|
|
_insert_into,
|
|
_is_stacked_header_gpci,
|
|
_load_carrier_files,
|
|
_load_gpci_files,
|
|
_load_pe_files,
|
|
_load_zip_carrier_files,
|
|
_parse_stacked_gpci,
|
|
_read_multi_year_gpci,
|
|
)
|
|
|
|
# ── Fixtures ──────────────────────────────────────────────────
|
|
|
|
|
|
@pytest.fixture()
|
|
def con():
|
|
c = duckdb.connect(":memory:")
|
|
c.execute('CREATE SCHEMA IF NOT EXISTS "pfs"')
|
|
yield c
|
|
c.close()
|
|
|
|
|
|
def _f(
|
|
filename: str,
|
|
year: int,
|
|
ext: str,
|
|
title: str = "CY 2024 PFS Final Rule",
|
|
path: str = "/tmp/fake",
|
|
) -> dict:
|
|
return {
|
|
"filename": filename,
|
|
"year": year,
|
|
"ext": ext,
|
|
"title": title,
|
|
"path": path,
|
|
"item_key": "ABCD",
|
|
}
|
|
|
|
|
|
# ── _is_stacked_header_gpci xlsx path (line 695) ─────────────
|
|
|
|
|
|
class TestIsStackedHeaderGpciXlsx:
|
|
"""Cover the xlsx branch with >5 rows (line 695: break)."""
|
|
|
|
def test_xlsx_stacked_true(self):
|
|
fake_openpyxl = types.ModuleType("openpyxl")
|
|
rows = [
|
|
("Title row",),
|
|
("Carrier", "Locality", "2017", "", "", "2018", "", ""),
|
|
("", "", "PW GPCI", "PE GPCI", "MP GPCI", "PW GPCI", "PE GPCI", "MP GPCI"),
|
|
("01", "01", "1.0", "0.9", "0.8", "1.1", "0.95", "0.85"),
|
|
]
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = rows
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
fake_openpyxl.load_workbook = MagicMock(return_value=mock_wb)
|
|
|
|
with patch.dict("sys.modules", {"openpyxl": fake_openpyxl}):
|
|
result = _is_stacked_header_gpci("/fake/file.xlsx", ".xlsx")
|
|
assert result is True
|
|
|
|
def test_xlsx_many_rows_hits_break(self):
|
|
"""Ensure i > 5 break is hit (line 695)."""
|
|
fake_openpyxl = types.ModuleType("openpyxl")
|
|
rows = [(f"row{i}",) for i in range(10)] # 10 rows, no year patterns
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = rows
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
fake_openpyxl.load_workbook = MagicMock(return_value=mock_wb)
|
|
|
|
with patch.dict("sys.modules", {"openpyxl": fake_openpyxl}):
|
|
result = _is_stacked_header_gpci("/fake/file.xlsx", ".xlsx")
|
|
assert result is False
|
|
|
|
|
|
# ── _parse_stacked_gpci edge cases (lines 752, 757) ──────────
|
|
|
|
|
|
class TestParseStackedGpciEdges:
|
|
def test_sub_label_out_of_bounds(self):
|
|
"""Line 752: _sub_label returns '' when index is out of bounds."""
|
|
year_parts = ["Carrier", "Locality", "Name", "2020"]
|
|
sub_parts = ["PW GPCI"] # too short — only covers 1 data col
|
|
data_lines = [
|
|
["01", "01", "Test", "1.0", "0.9", "0.8"],
|
|
]
|
|
result = _parse_stacked_gpci(year_parts, sub_parts, data_lines, 2020)
|
|
# Work GPCI found from sub_parts[0], PE and MP are "" (out of bounds)
|
|
assert result is not None
|
|
assert len(result) == 1
|
|
|
|
def test_short_data_line_skipped(self):
|
|
"""Line 756-757: data line with < n_id_cols + 3 is skipped."""
|
|
year_parts = ["Carrier", "Locality", "Name", "2020"]
|
|
sub_parts = ["", "", "", "PW GPCI", "PE GPCI", "MP GPCI"]
|
|
data_lines = [
|
|
["01", "01"], # too short (2 < 3+3=6)
|
|
["01", "01", "Test", "1.0", "0.9", "0.8"],
|
|
]
|
|
result = _parse_stacked_gpci(year_parts, sub_parts, data_lines, 2020)
|
|
assert result is not None
|
|
assert len(result) == 1 # only second line parsed
|
|
|
|
|
|
# ── _read_multi_year_gpci xlsx path (line 857) ───────────────
|
|
|
|
|
|
class TestReadMultiYearGpciXlsx:
|
|
def test_xlsx_successful_parse(self):
|
|
"""Cover the xlsx branch in _read_multi_year_gpci (line 845-864)."""
|
|
fake_openpyxl = types.ModuleType("openpyxl")
|
|
all_rows = [
|
|
("Carrier", "Locality", "Name", "2020", "", ""),
|
|
("", "", "", "PW GPCI", "PE GPCI", "MP GPCI"),
|
|
("01", "01", "TestLoc", "1.0", "0.95", "0.8"),
|
|
]
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = all_rows
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
fake_openpyxl.load_workbook = MagicMock(return_value=mock_wb)
|
|
|
|
with patch.dict("sys.modules", {"openpyxl": fake_openpyxl}):
|
|
result = _read_multi_year_gpci("/fake/file.xlsx", ".xlsx", 2020)
|
|
assert result is not None
|
|
assert len(result) == 1
|
|
|
|
def test_xlsx_no_header_returns_none(self):
|
|
"""xlsx path but no stacked header found → None."""
|
|
fake_openpyxl = types.ModuleType("openpyxl")
|
|
all_rows = [("just", "data"), ("more", "data")]
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = all_rows
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
fake_openpyxl.load_workbook = MagicMock(return_value=mock_wb)
|
|
|
|
with patch.dict("sys.modules", {"openpyxl": fake_openpyxl}):
|
|
result = _read_multi_year_gpci("/fake/file.xlsx", ".xlsx", 2020)
|
|
assert result is None
|
|
|
|
|
|
# ── _load_gpci_files complex branches (lines 915-998) ────────
|
|
|
|
|
|
class TestLoadGpciFilesEdgeCases:
|
|
def test_stacked_returns_empty_df(self, con, tmp_path):
|
|
"""Line 914-920: stacked GPCI returns None → skip with warning."""
|
|
p = tmp_path / "gpci_stacked.txt"
|
|
# Write a file that looks stacked but yields no data for target year
|
|
p.write_text(
|
|
"Header\nCarrier\tLocality\t2018\t\t\n\t\tPW GPCI\tPE GPCI\tMP GPCI\n"
|
|
)
|
|
f = _f("GPCI_stacked.txt", 2024, ".txt", path=str(p))
|
|
with (
|
|
patch("pfs.pipe._is_stacked_header_gpci", return_value=True),
|
|
patch("pfs.pipe._read_multi_year_gpci", return_value=None),
|
|
):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
def test_gpci_txt_non_stacked(self, con, tmp_path):
|
|
"""Line 921-922: non-stacked .txt GPCI file goes through _read_tsv."""
|
|
p = tmp_path / "gpci_simple.txt"
|
|
p.write_text(
|
|
"MAC\tLocality\tLocality Name\tState\tWork GPCI\tPE GPCI\tMP GPCI\n"
|
|
"01\t01\tTest Area\tNY\t1.024\t0.988\t0.574\n"
|
|
)
|
|
f = _f("GPCI_simple.txt", 2024, ".txt", path=str(p))
|
|
with patch("pfs.pipe._is_stacked_header_gpci", return_value=False):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 1
|
|
|
|
def test_gpci_csv_non_stacked(self, con, tmp_path):
|
|
"""Line 923-924: non-stacked .csv GPCI file."""
|
|
p = tmp_path / "gpci.csv"
|
|
p.write_text(
|
|
"MAC,Locality,Locality Name,State,Work GPCI,PE GPCI,MP GPCI\n"
|
|
"01,01,Test,NY,1.024,0.988,0.574\n"
|
|
)
|
|
f = _f("GPCI_2024.csv", 2024, ".csv", path=str(p))
|
|
with patch("pfs.pipe._is_stacked_header_gpci", return_value=False):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 1
|
|
|
|
def test_gpci_rename_carrier_state_without_branches(self, con, tmp_path):
|
|
"""Lines 939-940 (CARRIER→mac), 941-942 (STATE→state), 947-948 (WITHOUT skip)."""
|
|
p = tmp_path / "gpci_old.txt"
|
|
p.write_text(
|
|
"CARRIER\tSTATE\tLOCALITY\tLOCALITY NAME\t"
|
|
"WITHOUT FLOOR PW GPCI\t2024 PW GPCI\t2024 PE GPCI\t2024 MP GPCI\n"
|
|
"01\tNY\t01\tTest Area\t0.999\t1.024\t0.988\t0.574\n"
|
|
)
|
|
f = _f("GPCI_old.txt", 2024, ".txt", path=str(p))
|
|
with patch("pfs.pipe._is_stacked_header_gpci", return_value=False):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 1
|
|
|
|
def test_gpci_year_prefix_mismatch_skip(self, con, tmp_path):
|
|
"""Lines 952-953, 958-959, 964-965: year-prefixed cols for wrong year skipped."""
|
|
p = tmp_path / "gpci_multi.txt"
|
|
p.write_text(
|
|
"MAC\tLOCALITY\tLOCALITY NAME\t"
|
|
"2023 PW GPCI\t2023 PE GPCI\t2023 MP GPCI\t"
|
|
"2024 PW GPCI\t2024 PE GPCI\t2024 MP GPCI\n"
|
|
"01\t01\tTest\t0.9\t0.8\t0.7\t1.024\t0.988\t0.574\n"
|
|
)
|
|
f = _f("GPCI_multi_year.txt", 2024, ".txt", path=str(p))
|
|
with patch("pfs.pipe._is_stacked_header_gpci", return_value=False):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 1
|
|
df = con.execute("SELECT work_gpci FROM pfs.gpci").pl()
|
|
assert df["work_gpci"][0] == pytest.approx(1.024)
|
|
|
|
def test_gpci_exception_caught(self, con):
|
|
"""Lines 997-998: exception in GPCI loading is caught."""
|
|
f = _f("GPCI_bad.txt", 2024, ".txt", path="/nonexistent/path.txt")
|
|
with patch("pfs.pipe._is_stacked_header_gpci", return_value=False):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
def test_gpci_gap_fill_txt_path(self, con, tmp_path):
|
|
"""Lines 1005-1014: gap filling from txt multi-year file."""
|
|
p = tmp_path / "gpci_stacked.txt"
|
|
p.write_text(
|
|
"Carrier\tLocality\tName\t2016\t\t\t2017\t\t\n"
|
|
"\t\t\tPW GPCI\tPE GPCI\tMP GPCI\tPW GPCI\tPE GPCI\tMP GPCI\n"
|
|
"01\t01\tTest\t1.0\t0.9\t0.8\t1.1\t0.95\t0.85\n"
|
|
)
|
|
f = _f(
|
|
"GPCI_stacked.txt",
|
|
2017,
|
|
".txt",
|
|
title="CY 2017 PFS Final Rule",
|
|
path=str(p),
|
|
)
|
|
with patch("pfs.pipe._is_stacked_header_gpci", return_value=True):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["files"] >= 1
|
|
|
|
def test_gpci_gap_fill_csv_skipped(self, con):
|
|
"""Line 1002-1003: .csv multi-year file skipped in gap fill."""
|
|
f = _f("GPCI.csv", 2024, ".csv")
|
|
# Mark as stacked so it enters multi_year_files
|
|
with (
|
|
patch("pfs.pipe._is_stacked_header_gpci", return_value=True),
|
|
patch("pfs.pipe._read_multi_year_gpci", return_value=None),
|
|
):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
def test_gpci_gap_fill_xlsx_path(self, con):
|
|
"""Lines 1015-1029: gap fill reads available years from xlsx file."""
|
|
fake_openpyxl = types.ModuleType("openpyxl")
|
|
# Mock the xlsx workbook used in both _is_stacked_header_gpci and gap-fill
|
|
rows_data = [
|
|
("Carrier", "Locality", "Name", "2020", "", ""),
|
|
("", "", "", "PW GPCI", "PE GPCI", "MP GPCI"),
|
|
("01", "01", "TestLoc", "1.0", "0.95", "0.8"),
|
|
]
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = rows_data
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
fake_openpyxl.load_workbook = MagicMock(return_value=mock_wb)
|
|
|
|
f = _f("GPCI_multi.xlsx", 2020, ".xlsx", path="/fake/gpci.xlsx")
|
|
|
|
with (
|
|
patch.dict("sys.modules", {"openpyxl": fake_openpyxl}),
|
|
patch("pfs.pipe._is_stacked_header_gpci", return_value=True),
|
|
patch("pfs.pipe._read_multi_year_gpci", return_value=None),
|
|
):
|
|
result = _load_gpci_files([f], con)
|
|
# Primary load returns None (mocked), gap fill finds year 2020
|
|
# but it's already loaded_years, so no extra rows
|
|
assert result["rows"] == 0
|
|
|
|
def test_gpci_gap_fill_exception(self, con, tmp_path):
|
|
"""Lines 1060-1061: exception during gap fill is caught."""
|
|
p = tmp_path / "gpci_gap.txt"
|
|
p.write_text(
|
|
"Carrier\tLocality\t2015\t\t\n"
|
|
"\t\tPW GPCI\tPE GPCI\tMP GPCI\n"
|
|
"01\t01\t1.0\t0.9\t0.8\n"
|
|
)
|
|
f = _f("GPCI_gap.txt", 2015, ".txt", path=str(p))
|
|
with (
|
|
patch("pfs.pipe._is_stacked_header_gpci", return_value=True),
|
|
patch("pfs.pipe._read_multi_year_gpci") as mock_read,
|
|
):
|
|
# First call (primary load) returns None
|
|
# Second call (gap fill) raises
|
|
mock_read.side_effect = [None, RuntimeError("gap boom")]
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
def test_gpci_gap_fill_empty_df(self, con, tmp_path):
|
|
"""Line 1041-1042: gap fill returns None/empty → continue."""
|
|
p = tmp_path / "gpci_gap2.txt"
|
|
p.write_text(
|
|
"Carrier\tLocality\t2015\t\t\n"
|
|
"\t\tPW GPCI\tPE GPCI\tMP GPCI\n"
|
|
"01\t01\t1.0\t0.9\t0.8\n"
|
|
)
|
|
f = _f("GPCI_gap2.txt", 2015, ".txt", path=str(p))
|
|
with (
|
|
patch("pfs.pipe._is_stacked_header_gpci", return_value=True),
|
|
patch("pfs.pipe._read_multi_year_gpci", return_value=None),
|
|
):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
def test_gpci_gap_fill_txt_no_years(self, con, tmp_path):
|
|
"""Line 1014: txt file with no year patterns → avail_years = []."""
|
|
p = tmp_path / "gpci_noyears.txt"
|
|
p.write_text("just some text\nno years here\n")
|
|
f = _f("GPCI_noyears.txt", 2024, ".txt", path=str(p))
|
|
with (
|
|
patch("pfs.pipe._is_stacked_header_gpci", return_value=True),
|
|
patch("pfs.pipe._read_multi_year_gpci", return_value=None),
|
|
):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
def test_gpci_gap_fill_xlsx_many_rows(self, con):
|
|
"""Line 1023: xlsx with >5 rows, no years found → break at i>5."""
|
|
fake_openpyxl = types.ModuleType("openpyxl")
|
|
rows_data = [("row",)] * 10 # 10 rows, no year patterns
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = rows_data
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
fake_openpyxl.load_workbook = MagicMock(return_value=mock_wb)
|
|
|
|
f = _f("GPCI_many.xlsx", 2024, ".xlsx", path="/fake/gpci.xlsx")
|
|
|
|
with (
|
|
patch.dict("sys.modules", {"openpyxl": fake_openpyxl}),
|
|
patch("pfs.pipe._is_stacked_header_gpci", return_value=True),
|
|
patch("pfs.pipe._read_multi_year_gpci", return_value=None),
|
|
):
|
|
result = _load_gpci_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
|
|
# ── _load_pe_files edge cases ────────────────────────────────
|
|
|
|
|
|
class TestLoadPeFilesEdges:
|
|
def test_pe_ext_not_ok_skipped(self, con):
|
|
"""Line 1101-1102: file with unsupported ext is skipped."""
|
|
f = _f("PUF_LABOR_2024.pdf", 2024, ".pdf")
|
|
result = _load_pe_files([f], con)
|
|
assert result["clinical_labor"]["files"] == 0
|
|
|
|
def test_labor_nf_minutes_computed(self, con, tmp_path):
|
|
"""Lines 1146-1157: nf_minutes and f_minutes computed from sub-columns.
|
|
|
|
_normalize_columns maps nf_pre_svc→_nf_pre_svc (internal, dropped).
|
|
To exercise lines 1146-1157, mock _normalize_columns to return a df
|
|
with nf_* sub-columns still present (simulating a different column map).
|
|
"""
|
|
p = tmp_path / "PUF_LABOR_2024.csv"
|
|
p.write_text("hcpcs,x\n99213,1\n")
|
|
f = _f(
|
|
"PUF_LABOR_2024.csv",
|
|
2024,
|
|
".csv",
|
|
title="CY 2024 PFS Final Rule",
|
|
path=str(p),
|
|
)
|
|
|
|
mock_df = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213"],
|
|
"nf_pre_svc": ["5.0"],
|
|
"nf_svc": ["10.0"],
|
|
"nf_post_svc": ["3.0"],
|
|
"f_pre_svc": ["2.0"],
|
|
"f_svc": ["8.0"],
|
|
"f_post_svc": ["1.0"],
|
|
"rate_per_minute": ["0.50"],
|
|
}
|
|
)
|
|
|
|
with patch("pfs.pipe._normalize_columns", return_value=mock_df):
|
|
result = _load_pe_files([f], con)
|
|
assert result["clinical_labor"]["rows"] == 1
|
|
df = con.execute("SELECT nf_minutes, f_minutes FROM pfs.clinical_labor").pl()
|
|
assert df["nf_minutes"][0] == pytest.approx(18.0)
|
|
assert df["f_minutes"][0] == pytest.approx(11.0)
|
|
|
|
def test_labor_exception_caught(self, con):
|
|
"""Lines 1181-1182: labor exception caught."""
|
|
f = _f("PUF_LABOR_2024.csv", 2024, ".csv", path="/nonexistent.csv")
|
|
result = _load_pe_files([f], con)
|
|
assert result["clinical_labor"]["files"] == 0
|
|
|
|
def test_supply_no_hcpcs_skipped(self, con, tmp_path):
|
|
"""Lines 1192-1193: supply with no hcpcs column skipped."""
|
|
p = tmp_path / "PUF_SUPPLY_2024.csv"
|
|
p.write_text("code,description,price\nS001,Gauze,1.50\n")
|
|
f = _f(
|
|
"PUF_SUPPLY_2024.csv",
|
|
2024,
|
|
".csv",
|
|
title="CY 2024 PFS Final Rule",
|
|
path=str(p),
|
|
)
|
|
result = _load_pe_files([f], con)
|
|
assert result["medical_supply"]["files"] == 0
|
|
|
|
def test_supply_exception_caught(self, con):
|
|
"""Lines 1219-1220: supply exception caught."""
|
|
f = _f("PUF_SUPPLY_2024.csv", 2024, ".csv", path="/nonexistent.csv")
|
|
result = _load_pe_files([f], con)
|
|
assert result["medical_supply"]["files"] == 0
|
|
|
|
def test_equipment_no_hcpcs_skipped(self, con, tmp_path):
|
|
"""Lines 1230-1231: equipment with no hcpcs column skipped."""
|
|
p = tmp_path / "PUF_EQUIP_2024.csv"
|
|
p.write_text("code,desc,price\nEQ001,Monitor,500\n")
|
|
f = _f(
|
|
"PUF_EQUIP_2024.csv",
|
|
2024,
|
|
".csv",
|
|
title="CY 2024 PFS Final Rule",
|
|
path=str(p),
|
|
)
|
|
result = _load_pe_files([f], con)
|
|
assert result["medical_equipment"]["files"] == 0
|
|
|
|
def test_equipment_exception_caught(self, con):
|
|
"""Lines 1267-1268: equipment exception caught."""
|
|
f = _f("PUF_EQUIP_2024.csv", 2024, ".csv", path="/nonexistent.csv")
|
|
result = _load_pe_files([f], con)
|
|
assert result["medical_equipment"]["files"] == 0
|
|
|
|
def test_work_time_no_hcpcs_skipped(self, con, tmp_path):
|
|
"""Lines 1278-1279: work_time with no hcpcs skipped."""
|
|
p = tmp_path / "WORK_TIME_2024.csv"
|
|
p.write_text("code,time\nA,100\n")
|
|
f = _f(
|
|
"WORK_TIME_2024.csv",
|
|
2024,
|
|
".csv",
|
|
title="CY 2024 PFS Final Rule",
|
|
path=str(p),
|
|
)
|
|
result = _load_pe_files([f], con)
|
|
assert result["physician_work_time"]["files"] == 0
|
|
|
|
def test_work_time_exception_caught(self, con):
|
|
"""Lines 1342-1343: work_time exception caught."""
|
|
f = _f("WORK_TIME_2024.csv", 2024, ".csv", path="/nonexistent.csv")
|
|
result = _load_pe_files([f], con)
|
|
assert result["physician_work_time"]["files"] == 0
|
|
|
|
|
|
# ── _load_zip_carrier_files edges ─────────────────────────────
|
|
|
|
|
|
class TestLoadZipCarrierEdges:
|
|
def test_no_year_quarter_column(self, con, tmp_path):
|
|
"""Line 1425-1429: data without year_quarter gets literal year.
|
|
|
|
The fixed-width reader always creates year_quarter from positions 75-80.
|
|
To exercise the else branch, mock the DataFrame to not have year_quarter.
|
|
"""
|
|
p = tmp_path / "ZIP5_2024.txt"
|
|
# Standard fixed-width line — will produce year_quarter column
|
|
line = "NY" + "10001" + "01234" + "01" + " " + " " + " " + "0" + " " + "B"
|
|
line = line.ljust(80) + "\n"
|
|
p.write_text(line)
|
|
f = _f("ZIP5_2024.txt", 2024, ".txt", path=str(p))
|
|
|
|
# Patch pl.DataFrame to drop year_quarter before the if-check.
|
|
#
|
|
# IMPORTANT: patch the module *binding* pfs.pipe.pl, not the
|
|
# attribute pfs.pipe.pl.DataFrame. `pl` inside pfs.pipe is the
|
|
# real `polars` module object, so `patch("pfs.pipe.pl.DataFrame")`
|
|
# would replace `polars.DataFrame` globally (process-wide) for
|
|
# the duration of the `with` block. If duckdb resolves its
|
|
# Python replacement-scan type for polars DataFrames for the
|
|
# first time while that global is a MagicMock, it caches the
|
|
# MagicMock as "the polars DataFrame type" for the rest of the
|
|
# process, and every later `SELECT * FROM df` scan anywhere in
|
|
# the suite fails with
|
|
# "TypeError: isinstance() arg 2 must be a type, a tuple of
|
|
# types, or a union". Using a standalone proxy module for the
|
|
# `pl` name (with the real polars attributes copied in) keeps
|
|
# the real `polars.DataFrame` class untouched.
|
|
_orig_df = pl.DataFrame
|
|
|
|
def _df_no_yq(rows):
|
|
df = _orig_df(rows)
|
|
if "year_quarter" in df.columns:
|
|
df = df.drop("year_quarter")
|
|
return df
|
|
|
|
fake_pl = types.ModuleType("fake_polars_proxy")
|
|
fake_pl.__dict__.update(pl.__dict__)
|
|
fake_pl.DataFrame = _df_no_yq
|
|
|
|
with patch("pfs.pipe.pl", fake_pl):
|
|
result = _load_zip_carrier_files([f], con)
|
|
assert result["rows"] >= 1
|
|
|
|
def test_zip_carrier_exception_caught(self, con):
|
|
"""Lines 1441-1442: exception during zip carrier loading caught."""
|
|
f = _f("ZIP5_2024.txt", 2024, ".txt", path="/nonexistent.txt")
|
|
result = _load_zip_carrier_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
|
|
# ── _load_carrier_files edges ─────────────────────────────────
|
|
|
|
|
|
class TestLoadCarrierEdges:
|
|
def test_carrier_read_csv_exception(self, con):
|
|
"""Lines 1525-1527: read_csv failure skips file."""
|
|
f = _f("PFALL26.TXT", 2026, ".txt", path="/nonexistent/PFALL26.TXT")
|
|
with patch("pfs.pipe.pl.read_csv", side_effect=Exception("read fail")):
|
|
result = _load_carrier_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
def test_carrier_too_few_columns(self, con, tmp_path):
|
|
"""Lines 1535-1536: file with <8 columns skipped."""
|
|
p = tmp_path / "PFALL26.TXT"
|
|
p.write_text("a,b,c\n1,2,3\n")
|
|
f = _f("PFALL26.TXT", 2026, ".txt", path=str(p))
|
|
result = _load_carrier_files([f], con)
|
|
assert result["rows"] == 0
|
|
|
|
def test_carrier_empty_year_frames(self, con, tmp_path):
|
|
"""Line 1583-1584: no valid frames for a year → continue."""
|
|
p = tmp_path / "PFALL26.TXT"
|
|
p.write_text("a,b\n1,2\n") # only 2 cols
|
|
f = _f("PFALL26.TXT", 2026, ".txt", path=str(p))
|
|
result = _load_carrier_files([f], con)
|
|
assert result["files"] == 0
|
|
|
|
|
|
# ── _insert_into add column exception (lines 1646-1647) ──────
|
|
|
|
|
|
# ── _load_rvu_files CF scanning and multi-release selection ──────
|
|
|
|
|
|
class TestLoadRvuFilesCfMatching:
|
|
"""Lines 663-677: _scan_cf function; lines 698-714: multi-release matching."""
|
|
|
|
def test_scan_cf_returns_value(self, con):
|
|
"""Lines 663-674: _scan_cf reads conv_factor from PPRRVU file."""
|
|
from pfs.pipe import _load_rvu_files
|
|
|
|
pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213", "99214"],
|
|
"mod": ["", ""],
|
|
"conv_factor": [35.0, 35.0],
|
|
}
|
|
)
|
|
|
|
# Two PPRRVU files for same year — forces multi-release branch (line 697)
|
|
f1 = {
|
|
"filename": "PPRRVU26A.xlsx",
|
|
"year": 2026,
|
|
"ext": ".xlsx",
|
|
"title": "CY 2026 PFS Final Rule",
|
|
"path": "/tmp/fake1.xlsx",
|
|
"item_key": "A1",
|
|
"release": "q1",
|
|
}
|
|
f2 = {
|
|
"filename": "PPRRVU26B.xlsx",
|
|
"year": 2026,
|
|
"ext": ".xlsx",
|
|
"title": "CY 2026 PFS Final Rule",
|
|
"path": "/tmp/fake2.xlsx",
|
|
"item_key": "A2",
|
|
"release": "q2",
|
|
}
|
|
|
|
# Mock _read_pprrvu to return a df with conv_factor matching RULES[2026]
|
|
from pfs.rules import RULES
|
|
|
|
target_cf = RULES[2026].conversion_factor
|
|
mock_df_match = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213"],
|
|
"mod": [""],
|
|
"conv_factor": [target_cf],
|
|
"work_rvu": [0.97],
|
|
"non_fac_pe_rvu": [1.04],
|
|
"fac_pe_rvu": [0.41],
|
|
"mp_rvu": [0.07],
|
|
"status_code": ["A"],
|
|
"description": ["x"],
|
|
}
|
|
)
|
|
mock_df_nomatch = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213"],
|
|
"mod": [""],
|
|
"conv_factor": [99.99],
|
|
"work_rvu": [0.97],
|
|
"non_fac_pe_rvu": [1.04],
|
|
"fac_pe_rvu": [0.41],
|
|
"mp_rvu": [0.07],
|
|
"status_code": ["A"],
|
|
"description": ["x"],
|
|
}
|
|
)
|
|
|
|
def _mock_read(path):
|
|
if "fake1" in path:
|
|
return mock_df_nomatch
|
|
return mock_df_match
|
|
|
|
with (
|
|
patch("pfs.pipe._read_pprrvu", side_effect=_mock_read),
|
|
patch("pfs.pipe._normalize_columns", side_effect=lambda df, _: df),
|
|
):
|
|
result = _load_rvu_files([f1, f2], con)
|
|
assert result["files"] >= 1
|
|
|
|
def test_scan_cf_exception_returns_none(self, con):
|
|
"""Lines 675-677: _scan_cf catches exceptions and returns None."""
|
|
from pfs.pipe import _load_rvu_files
|
|
|
|
f1 = {
|
|
"filename": "PPRRVU26A.xlsx",
|
|
"year": 2026,
|
|
"ext": ".xlsx",
|
|
"title": "CY 2026 PFS Final Rule",
|
|
"path": "/tmp/fake1.xlsx",
|
|
"item_key": "A1",
|
|
"release": "q1",
|
|
}
|
|
f2 = {
|
|
"filename": "PPRRVU26B.xlsx",
|
|
"year": 2026,
|
|
"ext": ".xlsx",
|
|
"title": "CY 2026 PFS Final Rule",
|
|
"path": "/tmp/fake2.xlsx",
|
|
"item_key": "A2",
|
|
"release": "q2",
|
|
}
|
|
|
|
# Both reads fail during CF scanning → falls back to earliest PPRRVU
|
|
mock_df = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213"],
|
|
"mod": [""],
|
|
"work_rvu": [0.97],
|
|
"non_fac_pe_rvu": [1.04],
|
|
"fac_pe_rvu": [0.41],
|
|
"mp_rvu": [0.07],
|
|
"status_code": ["A"],
|
|
"description": ["x"],
|
|
}
|
|
)
|
|
|
|
call_count = [0]
|
|
|
|
def _mock_read(path):
|
|
call_count[0] += 1
|
|
if call_count[0] <= 2:
|
|
raise RuntimeError("CF scan fail")
|
|
return mock_df
|
|
|
|
with (
|
|
patch("pfs.pipe._read_pprrvu", side_effect=_mock_read),
|
|
patch("pfs.pipe._normalize_columns", side_effect=lambda df, _: df),
|
|
):
|
|
result = _load_rvu_files([f1, f2], con)
|
|
# Should still load (fallback to earliest PPRRVU)
|
|
assert result["files"] >= 1
|
|
|
|
def test_no_pprrvu_falls_to_addendum_b(self, con):
|
|
"""Lines 725-731: no PPRRVU → falls to Addendum B."""
|
|
from pfs.pipe import _load_rvu_files
|
|
|
|
f1 = {
|
|
"filename": "Addendum_B_2026.xlsx",
|
|
"year": 2026,
|
|
"ext": ".xlsx",
|
|
"title": "CY 2026 PFS Final Rule",
|
|
"path": "/tmp/fake_addb.xlsx",
|
|
"item_key": "A1",
|
|
}
|
|
|
|
mock_df = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213"],
|
|
"mod": [""],
|
|
"work_rvu": [0.97],
|
|
"non_fac_pe_rvu": [1.04],
|
|
"fac_pe_rvu": [0.41],
|
|
"mp_rvu": [0.07],
|
|
"status_code": ["A"],
|
|
"description": ["x"],
|
|
}
|
|
)
|
|
|
|
with (
|
|
patch("pfs.pipe._read_excel", return_value=mock_df),
|
|
patch("pfs.pipe._normalize_columns", side_effect=lambda df, _: df),
|
|
):
|
|
result = _load_rvu_files([f1], con)
|
|
assert result["files"] >= 1
|
|
|
|
def test_pprrvu_31_columns(self):
|
|
"""Line 415: PPRRVU with exactly 31 columns uses _EXPECTED_31 layout."""
|
|
import sys
|
|
import types
|
|
|
|
from pfs.pipe import _read_pprrvu
|
|
|
|
fake_openpyxl = types.ModuleType("openpyxl")
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = [
|
|
("HCPCS", "MOD", "DESCRIPTION"),
|
|
]
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
fake_openpyxl.load_workbook = MagicMock(return_value=mock_wb)
|
|
|
|
# 31 columns → _EXPECTED_31 layout
|
|
col_names = [f"col{i}" for i in range(31)]
|
|
data = {c: ["val"] for c in col_names}
|
|
fake_df = pl.DataFrame(data)
|
|
|
|
prev = sys.modules.get("openpyxl")
|
|
sys.modules["openpyxl"] = fake_openpyxl
|
|
try:
|
|
with patch("polars.read_excel", return_value=fake_df):
|
|
result = _read_pprrvu("fake.xlsx")
|
|
# Should use _EXPECTED_31 (no _pric_ind column)
|
|
assert "hcpcs" in result.columns
|
|
assert "_pric_ind" not in result.columns
|
|
finally:
|
|
if prev is None:
|
|
sys.modules.pop("openpyxl", None)
|
|
else:
|
|
sys.modules["openpyxl"] = prev
|
|
|
|
|
|
class TestInsertIntoAlterException:
|
|
def test_add_new_column(self, con):
|
|
"""Lines 1636-1645: ALTER TABLE ADD COLUMN succeeds for new column."""
|
|
df1 = pl.DataFrame({"a": ["x"]})
|
|
_insert_into(con, "pfs", "alter_test", df1)
|
|
|
|
# Insert with extra column "b" — triggers ALTER TABLE ADD
|
|
df2 = pl.DataFrame({"a": ["y"], "b": ["z"]})
|
|
_insert_into(con, "pfs", "alter_test", df2)
|
|
result = con.execute("SELECT COUNT(*) FROM pfs.alter_test").fetchone()
|
|
assert result[0] == 2
|
|
|
|
def test_add_column_different_dtypes(self, con):
|
|
"""Lines 1638-1645: ALTER TABLE ADD for Float64, Int64, Utf8 dtypes."""
|
|
df1 = pl.DataFrame({"a": ["x"]})
|
|
_insert_into(con, "pfs", "alter_dtype", df1)
|
|
|
|
# Insert df with Float64, Int64, and Utf8 extra columns
|
|
df2 = pl.DataFrame(
|
|
{
|
|
"a": ["y"],
|
|
"b_float": [1.5],
|
|
"c_int": [42],
|
|
"d_str": ["hello"],
|
|
}
|
|
)
|
|
_insert_into(con, "pfs", "alter_dtype", df2)
|
|
count = con.execute("SELECT COUNT(*) FROM pfs.alter_dtype").fetchone()[0]
|
|
assert count == 2
|