1417 lines
52 KiB
Python
1417 lines
52 KiB
Python
"""Tests for pfs.pipe table loader functions and DuckDB insertion (lines 550-1755).
|
|
|
|
Covers: _is_stacked_header_gpci, _parse_stacked_gpci, _find_stacked_header,
|
|
_read_multi_year_gpci, _load_rvu_files, _load_gpci_files, _read_tabular,
|
|
_load_pe_files, _load_zip_carrier_files, _load_carrier_files, _insert_into,
|
|
and load_all.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import duckdb
|
|
import polars as pl
|
|
import pytest
|
|
|
|
from pfs.pipe import (
|
|
_find_stacked_header,
|
|
_insert_into,
|
|
_is_stacked_header_gpci,
|
|
_load_carrier_files,
|
|
_load_gpci_files,
|
|
_load_pe_files,
|
|
_load_rvu_files,
|
|
_load_zip_carrier_files,
|
|
_parse_stacked_gpci,
|
|
_read_multi_year_gpci,
|
|
_read_tabular,
|
|
load_all,
|
|
)
|
|
|
|
# ── Fixtures ──────────────────────────────────────────────────
|
|
|
|
|
|
@pytest.fixture()
|
|
def con():
|
|
"""In-memory DuckDB connection for testing."""
|
|
c = duckdb.connect(":memory:")
|
|
c.execute('CREATE SCHEMA IF NOT EXISTS "pfs"')
|
|
yield c
|
|
c.close()
|
|
|
|
|
|
def _make_file(
|
|
filename: str,
|
|
year: int,
|
|
ext: str,
|
|
title: str = "CY 2024 PFS Final Rule",
|
|
path: str = "/tmp/fake",
|
|
) -> dict:
|
|
return {
|
|
"filename": filename,
|
|
"year": year,
|
|
"ext": ext,
|
|
"title": title,
|
|
"path": path,
|
|
"item_key": "ABCD",
|
|
}
|
|
|
|
|
|
# ── _is_stacked_header_gpci ──────────────────────────────────
|
|
|
|
|
|
class TestIsStackedHeaderGpci:
|
|
"""Detect stacked 2-row header GPCI files."""
|
|
|
|
def test_txt_stacked_returns_true(self, tmp_path):
|
|
"""A .txt file with bare year row followed by GPCI sub-header."""
|
|
content = (
|
|
"Carrier\tLocality\tName\t2016\t\t\t2017\n"
|
|
"\t\t\tPW GPCI\tPE GPCI\tMP GPCI\tPW GPCI\n"
|
|
"01\t00\tALABAMA\t1.000\t0.900\t0.800\t1.010\n"
|
|
)
|
|
f = tmp_path / "gpci.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
assert _is_stacked_header_gpci(str(f), ".txt") is True
|
|
|
|
def test_txt_no_gpci_sub_header_returns_false(self, tmp_path):
|
|
"""A .txt file with year row but no matching sub-header."""
|
|
content = (
|
|
"Carrier\tLocality\tName\t2016\nSome other header without the keyword\n"
|
|
)
|
|
f = tmp_path / "gpci_no.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
assert _is_stacked_header_gpci(str(f), ".txt") is False
|
|
|
|
def test_txt_no_year_row_returns_false(self, tmp_path):
|
|
"""A .txt file without any bare year values."""
|
|
content = "Carrier\tLocality\tName\tWork GPCI\tPE GPCI\n"
|
|
f = tmp_path / "gpci_single.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
assert _is_stacked_header_gpci(str(f), ".txt") is False
|
|
|
|
def test_xlsx_stacked_returns_true(self):
|
|
"""An .xlsx file with stacked header, mocking openpyxl."""
|
|
rows = [
|
|
("Carrier", "Locality", "Name", 2016, None, None, 2017),
|
|
(None, None, None, "PW GPCI", "PE GPCI", "MP GPCI", "PW GPCI"),
|
|
("01", "00", "ALABAMA", 1.0, 0.9, 0.8, 1.01),
|
|
]
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = iter(rows)
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
mock_openpyxl = MagicMock()
|
|
mock_openpyxl.load_workbook.return_value = mock_wb
|
|
with patch.dict("sys.modules", {"openpyxl": mock_openpyxl}):
|
|
assert _is_stacked_header_gpci("/fake/path.xlsx", ".xlsx") is True
|
|
|
|
def test_xlsx_no_match_returns_false(self):
|
|
"""An .xlsx without stacked header returns False."""
|
|
rows = [
|
|
("MAC", "Locality", "Name", "2020 PW GPCI", "2020 PE GPCI"),
|
|
]
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = iter(rows)
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
mock_openpyxl = MagicMock()
|
|
mock_openpyxl.load_workbook.return_value = mock_wb
|
|
with patch.dict("sys.modules", {"openpyxl": mock_openpyxl}):
|
|
assert _is_stacked_header_gpci("/fake/path.xlsx", ".xlsx") is False
|
|
|
|
def test_unsupported_ext_returns_false(self, tmp_path):
|
|
"""Extension not .txt or .xlsx returns False."""
|
|
assert _is_stacked_header_gpci("/fake.csv", ".csv") is False
|
|
|
|
|
|
# ── _parse_stacked_gpci ──────────────────────────────────────
|
|
|
|
|
|
class TestParseStackedGpci:
|
|
"""Parse stacked header GPCI data."""
|
|
|
|
def test_successful_parse(self):
|
|
year_parts = ["Carrier", "Locality", "Name", "2016", "", "", "2017"]
|
|
sub_parts = ["", "", "", "PW GPCI", "PE GPCI", "MP GPCI", "PW GPCI"]
|
|
data_lines = [
|
|
["01", "00", "ALABAMA", "1.000", "0.900", "0.800", "1.010"],
|
|
]
|
|
result = _parse_stacked_gpci(year_parts, sub_parts, data_lines, 2016)
|
|
assert result is not None
|
|
assert len(result) == 1
|
|
assert result["work_gpci"][0] == "1.000"
|
|
assert result["pe_gpci"][0] == "0.900"
|
|
assert result["mp_gpci"][0] == "0.800"
|
|
assert result["mac"][0] == "01"
|
|
assert result["locality"][0] == "00"
|
|
|
|
def test_no_first_year_idx_returns_none(self):
|
|
"""No bare year in year_parts returns None."""
|
|
year_parts = ["Carrier", "Locality", "Name"]
|
|
sub_parts = ["PW GPCI", "PE GPCI", "MP GPCI"]
|
|
data_lines = [["01", "00", "ALABAMA", "1.0", "0.9", "0.8"]]
|
|
assert _parse_stacked_gpci(year_parts, sub_parts, data_lines, 2016) is None
|
|
|
|
def test_empty_locality_skipped(self):
|
|
"""Rows with empty locality are skipped."""
|
|
year_parts = ["Carrier", "Locality", "Name", "2016", "", ""]
|
|
sub_parts = ["", "", "", "PW GPCI", "PE GPCI", "MP GPCI"]
|
|
data_lines = [
|
|
["01", "", "ALABAMA", "1.000", "0.900", "0.800"],
|
|
]
|
|
result = _parse_stacked_gpci(year_parts, sub_parts, data_lines, 2016)
|
|
# Empty locality -> row skipped -> no data
|
|
assert result is None
|
|
|
|
def test_no_work_value_skipped(self):
|
|
"""Rows without a work GPCI value are skipped."""
|
|
year_parts = ["Carrier", "Locality", "Name", "2016", "", ""]
|
|
sub_parts = ["", "", "", "PE GPCI", "MP GPCI", "SOMETHING"]
|
|
data_lines = [
|
|
["01", "00", "ALABAMA", "0.900", "0.800", "X"],
|
|
]
|
|
result = _parse_stacked_gpci(year_parts, sub_parts, data_lines, 2016)
|
|
# No "PW GPCI" or "WORK" in sub-labels so work is None -> skipped
|
|
assert result is None
|
|
|
|
def test_single_year_format_sub_offset(self):
|
|
"""Single-year format: sub_parts shorter than year_parts -> offset applied."""
|
|
# sub_parts omits identifier columns entirely
|
|
year_parts = ["Carrier", "Locality", "Name", "2018", "", ""]
|
|
sub_parts = ["PW GPCI", "PE GPCI", "MP GPCI"] # shorter -> sub_offset=3
|
|
data_lines = [
|
|
["01", "00", "ALABAMA", "1.000", "0.900", "0.800"],
|
|
]
|
|
result = _parse_stacked_gpci(year_parts, sub_parts, data_lines, 2018)
|
|
assert result is not None
|
|
assert len(result) == 1
|
|
assert result["work_gpci"][0] == "1.000"
|
|
|
|
def test_target_year_filter(self):
|
|
"""Only the target year is extracted from multi-year data."""
|
|
year_parts = ["Carrier", "Locality", "Name", "2016", "", "", "2017", "", ""]
|
|
sub_parts = [
|
|
"",
|
|
"",
|
|
"",
|
|
"PW GPCI",
|
|
"PE GPCI",
|
|
"MP GPCI",
|
|
"PW GPCI",
|
|
"PE GPCI",
|
|
"MP GPCI",
|
|
]
|
|
data_lines = [
|
|
[
|
|
"01",
|
|
"00",
|
|
"ALABAMA",
|
|
"1.000",
|
|
"0.900",
|
|
"0.800",
|
|
"1.010",
|
|
"0.910",
|
|
"0.810",
|
|
],
|
|
]
|
|
r2016 = _parse_stacked_gpci(year_parts, sub_parts, data_lines, 2016)
|
|
r2017 = _parse_stacked_gpci(year_parts, sub_parts, data_lines, 2017)
|
|
assert r2016 is not None
|
|
assert r2017 is not None
|
|
assert r2016["work_gpci"][0] == "1.000"
|
|
assert r2017["work_gpci"][0] == "1.010"
|
|
|
|
|
|
# ── _find_stacked_header ─────────────────────────────────────
|
|
|
|
|
|
class TestFindStackedHeader:
|
|
"""Find year-header and sub-header rows."""
|
|
|
|
def test_text_mode_found(self):
|
|
lines = [
|
|
"Title line\n",
|
|
"Carrier\tLocality\tName\t2016\t\t\t2017\n",
|
|
"\t\t\tPW GPCI\tPE GPCI\tMP GPCI\tPW GPCI\n",
|
|
"01\t00\tALABAMA\t1.0\t0.9\t0.8\t1.01\n",
|
|
]
|
|
yr_idx, sub_idx, yp, sp = _find_stacked_header(lines, is_text=True)
|
|
assert yr_idx == 1
|
|
assert sub_idx == 2
|
|
assert "2016" in yp
|
|
assert any("GPCI" in p for p in sp)
|
|
|
|
def test_text_mode_not_found(self):
|
|
lines = [
|
|
"MAC\tLocality\tName\t2020 PW GPCI\t2020 PE GPCI\n",
|
|
]
|
|
yr_idx, sub_idx, yp, sp = _find_stacked_header(lines, is_text=True)
|
|
assert yr_idx is None
|
|
assert sub_idx is None
|
|
assert yp == []
|
|
assert sp == []
|
|
|
|
def test_row_mode_found(self):
|
|
rows = [
|
|
("Carrier", "Locality", "Name", 2016, None, None, 2017),
|
|
(None, None, None, "PW GPCI", "PE GPCI", "MP GPCI", "PW GPCI"),
|
|
]
|
|
yr_idx, sub_idx, yp, sp = _find_stacked_header(rows, is_text=False)
|
|
assert yr_idx == 0
|
|
assert sub_idx == 1
|
|
|
|
def test_row_mode_not_found(self):
|
|
rows = [
|
|
("MAC", "Locality", "2020 PW GPCI", "2020 PE GPCI"),
|
|
]
|
|
yr_idx, sub_idx, yp, sp = _find_stacked_header(rows, is_text=False)
|
|
assert yr_idx is None
|
|
assert sub_idx is None
|
|
|
|
def test_year_only_no_gpci_returns_none(self):
|
|
"""Year row exists but no subsequent GPCI row."""
|
|
lines = [
|
|
"Carrier\tLocality\t2016\n",
|
|
"Some\tOther\tStuff\n",
|
|
]
|
|
yr_idx, sub_idx, yp, sp = _find_stacked_header(lines, is_text=True)
|
|
assert yr_idx is None
|
|
assert sub_idx is None
|
|
|
|
|
|
# ── _read_multi_year_gpci ─────────────────────────────────────
|
|
|
|
|
|
class TestReadMultiYearGpci:
|
|
"""Read stacked GPCI files (txt and xlsx paths)."""
|
|
|
|
def test_txt_path(self, tmp_path):
|
|
content = (
|
|
"Title line\n"
|
|
"Carrier\tLocality\tName\t2016\t\t\t2017\n"
|
|
"\t\t\tPW GPCI\tPE GPCI\tMP GPCI\tPW GPCI\tPE GPCI\tMP GPCI\n"
|
|
"01\t00\tALABAMA\t1.000\t0.900\t0.800\t1.010\t0.910\t0.810\n"
|
|
)
|
|
f = tmp_path / "gpci.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
result = _read_multi_year_gpci(str(f), ".txt", 2016)
|
|
assert result is not None
|
|
assert len(result) == 1
|
|
assert result["work_gpci"][0] == "1.000"
|
|
|
|
def test_txt_returns_none_when_no_header(self, tmp_path):
|
|
content = "Just some text without years\n"
|
|
f = tmp_path / "gpci_bad.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
result = _read_multi_year_gpci(str(f), ".txt", 2016)
|
|
assert result is None
|
|
|
|
def test_xlsx_path(self):
|
|
"""Mock openpyxl for xlsx reading."""
|
|
all_rows = [
|
|
("Title", None, None, None),
|
|
("Carrier", "Locality", "Name", 2016, None, None, 2017),
|
|
(None, None, None, "PW GPCI", "PE GPCI", "MP GPCI", "PW GPCI"),
|
|
("01", "00", "ALABAMA", "1.000", "0.900", "0.800", "1.010"),
|
|
]
|
|
mock_ws = MagicMock()
|
|
mock_ws.iter_rows.return_value = iter(all_rows)
|
|
mock_wb = MagicMock()
|
|
mock_wb.active = mock_ws
|
|
mock_openpyxl = MagicMock()
|
|
mock_openpyxl.load_workbook.return_value = mock_wb
|
|
with patch.dict("sys.modules", {"openpyxl": mock_openpyxl}):
|
|
result = _read_multi_year_gpci("/fake.xlsx", ".xlsx", 2016)
|
|
assert result is not None
|
|
assert len(result) == 1
|
|
|
|
def test_unsupported_ext_returns_none(self):
|
|
assert _read_multi_year_gpci("/fake.csv", ".csv", 2016) is None
|
|
|
|
|
|
# ── _load_rvu_files ───────────────────────────────────────────
|
|
|
|
|
|
class TestLoadRvuFiles:
|
|
"""Load RVU files into DuckDB."""
|
|
|
|
def _make_rvu_df(self):
|
|
return pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213", "99214"],
|
|
"mod": ["", ""],
|
|
"description": ["Office visit", "Office visit"],
|
|
"work_rvu": ["1.3", "1.92"],
|
|
"non_fac_pe_rvu": ["1.51", "2.12"],
|
|
"fac_pe_rvu": ["0.82", "1.12"],
|
|
"mp_rvu": ["0.09", "0.13"],
|
|
}
|
|
)
|
|
|
|
@patch("pfs.pipe._read_excel")
|
|
def test_xlsx_addendum_b(self, mock_read, con):
|
|
"""xlsx Addendum B file is loaded successfully."""
|
|
mock_read.return_value = self._make_rvu_df()
|
|
files = [
|
|
_make_file(
|
|
"Addendum_B_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/addb.xlsx",
|
|
)
|
|
]
|
|
result = _load_rvu_files(files, con)
|
|
assert result["rows"] == 2
|
|
assert result["files"] == 1
|
|
rows = con.execute('SELECT * FROM "pfs"."rvu"').fetchall()
|
|
assert len(rows) == 2
|
|
|
|
@patch("pfs.pipe._read_pprrvu")
|
|
def test_pprrvu_xlsx(self, mock_read, con):
|
|
"""PPRRVU xlsx loads via the _read_pprrvu reader."""
|
|
mock_read.return_value = self._make_rvu_df()
|
|
files = [
|
|
_make_file(
|
|
"PPRRVU24A.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
title="CY 2024 PFS Proposed Rule",
|
|
path="/fake/pprrvu.xlsx",
|
|
)
|
|
]
|
|
result = _load_rvu_files(files, con)
|
|
assert result["rows"] == 2
|
|
mock_read.assert_called_once()
|
|
|
|
@patch("pfs.pipe._read_csv_with_header")
|
|
def test_csv_fallback(self, mock_read, con):
|
|
"""CSV used when no xlsx exists for same year."""
|
|
mock_read.return_value = self._make_rvu_df()
|
|
files = [
|
|
_make_file(
|
|
"Addendum_B_2024.csv",
|
|
2024,
|
|
".csv",
|
|
path="/fake/addb.csv",
|
|
)
|
|
]
|
|
result = _load_rvu_files(files, con)
|
|
assert result["rows"] == 2
|
|
mock_read.assert_called_once()
|
|
|
|
@patch("pfs.pipe._read_excel")
|
|
@patch("pfs.pipe._read_csv_with_header")
|
|
def test_csv_skipped_when_xlsx_exists(self, mock_csv, mock_xlsx, con):
|
|
"""CSV is skipped when an xlsx for the same year+type exists."""
|
|
mock_xlsx.return_value = self._make_rvu_df()
|
|
mock_csv.return_value = self._make_rvu_df()
|
|
files = [
|
|
_make_file(
|
|
"Addendum_B_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/addb.xlsx",
|
|
),
|
|
_make_file(
|
|
"Addendum_B_2024.csv",
|
|
2024,
|
|
".csv",
|
|
path="/fake/addb.csv",
|
|
),
|
|
]
|
|
result = _load_rvu_files(files, con)
|
|
assert result["files"] == 1
|
|
mock_csv.assert_not_called()
|
|
|
|
@patch("pfs.pipe._read_excel")
|
|
def test_final_preferred_over_proposed(self, mock_read, con):
|
|
"""Final rule has higher priority than proposed."""
|
|
mock_read.return_value = self._make_rvu_df()
|
|
files = [
|
|
_make_file(
|
|
"Addendum_B_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
title="CY 2024 PFS Proposed Rule",
|
|
path="/fake/proposed.xlsx",
|
|
),
|
|
_make_file(
|
|
"Addendum_B_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
title="CY 2024 PFS Final Rule",
|
|
path="/fake/final.xlsx",
|
|
),
|
|
]
|
|
result = _load_rvu_files(files, con)
|
|
assert result["files"] == 1
|
|
# Should have called with the final rule path
|
|
mock_read.assert_called_once_with("/fake/final.xlsx")
|
|
|
|
@patch("pfs.pipe._read_excel")
|
|
def test_normalize_columns(self, mock_read, con):
|
|
"""Columns matching _RVU_COLUMNS map are normalized."""
|
|
df = pl.DataFrame(
|
|
{
|
|
"HCPCS": ["99213"],
|
|
"WORK RVU": ["1.3"],
|
|
"NON-FACILITY PE RVU": ["1.51"],
|
|
"FACILITY PE RVU": ["0.82"],
|
|
"MALPRACTICE RVU": ["0.09"],
|
|
}
|
|
)
|
|
mock_read.return_value = df
|
|
files = [
|
|
_make_file(
|
|
"Addendum_B_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/addb.xlsx",
|
|
)
|
|
]
|
|
result = _load_rvu_files(files, con)
|
|
assert result["rows"] == 1
|
|
cols = [
|
|
r[0]
|
|
for r in con.execute(
|
|
"SELECT column_name FROM information_schema.columns "
|
|
"WHERE table_schema = 'pfs' AND table_name = 'rvu'"
|
|
).fetchall()
|
|
]
|
|
assert "hcpcs" in cols
|
|
assert "work_rvu" in cols
|
|
|
|
@patch("pfs.pipe._read_excel")
|
|
def test_skip_file_with_no_hcpcs(self, mock_read, con):
|
|
"""File with no hcpcs column is skipped."""
|
|
df = pl.DataFrame({"unknown_col": ["val1"]})
|
|
mock_read.return_value = df
|
|
files = [
|
|
_make_file(
|
|
"Addendum_B_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/addb.xlsx",
|
|
)
|
|
]
|
|
result = _load_rvu_files(files, con)
|
|
assert result["rows"] == 0
|
|
assert result["files"] == 0
|
|
|
|
@patch("pfs.pipe._read_excel", side_effect=Exception("parse error"))
|
|
def test_log_error_on_exception(self, mock_read, con):
|
|
"""Exception during file reading is caught and logged."""
|
|
files = [
|
|
_make_file(
|
|
"Addendum_B_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/addb.xlsx",
|
|
)
|
|
]
|
|
result = _load_rvu_files(files, con)
|
|
assert result["rows"] == 0
|
|
assert result["files"] == 0
|
|
|
|
@patch("pfs.pipe._read_excel")
|
|
def test_empty_hcpcs_rows_filtered(self, mock_read, con):
|
|
"""Rows with null or empty hcpcs are filtered out."""
|
|
df = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213", "", None],
|
|
"work_rvu": ["1.3", "0", "0"],
|
|
}
|
|
)
|
|
mock_read.return_value = df
|
|
files = [
|
|
_make_file(
|
|
"Addendum_B_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/addb.xlsx",
|
|
)
|
|
]
|
|
result = _load_rvu_files(files, con)
|
|
assert result["rows"] == 1
|
|
|
|
|
|
# ── _load_gpci_files ──────────────────────────────────────────
|
|
|
|
|
|
class TestLoadGpciFiles:
|
|
"""Load GPCI files into DuckDB."""
|
|
|
|
def _make_gpci_df(self):
|
|
return pl.DataFrame(
|
|
{
|
|
"MAC": ["01102"],
|
|
"LOCALITY": ["00"],
|
|
"LOCALITY NAME": ["ALABAMA"],
|
|
"2024 PW GPCI": ["1.000"],
|
|
"2024 PE GPCI": ["0.900"],
|
|
"2024 MP GPCI": ["0.800"],
|
|
}
|
|
)
|
|
|
|
@patch("pfs.pipe._is_stacked_header_gpci", return_value=False)
|
|
@patch("pfs.pipe._read_excel")
|
|
def test_standard_file(self, mock_read, mock_stacked, con):
|
|
"""Standard (non-stacked) GPCI file loads correctly."""
|
|
mock_read.return_value = self._make_gpci_df()
|
|
files = [
|
|
_make_file(
|
|
"GPCI2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/gpci.xlsx",
|
|
)
|
|
]
|
|
result = _load_gpci_files(files, con)
|
|
assert result["rows"] == 1
|
|
assert result["files"] == 1
|
|
rows = con.execute('SELECT * FROM "pfs"."gpci"').fetchall()
|
|
assert len(rows) == 1
|
|
|
|
@patch("pfs.pipe._is_stacked_header_gpci", return_value=False)
|
|
@patch("pfs.pipe._read_excel")
|
|
def test_skip_no_locality(self, mock_read, mock_stacked, con):
|
|
"""File without a locality column is skipped."""
|
|
mock_read.return_value = pl.DataFrame({"SOME_COL": ["val"], "PW GPCI": ["1.0"]})
|
|
files = [
|
|
_make_file(
|
|
"GPCI2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/gpci.xlsx",
|
|
)
|
|
]
|
|
result = _load_gpci_files(files, con)
|
|
assert result["rows"] == 0
|
|
|
|
@patch("pfs.pipe._read_multi_year_gpci")
|
|
@patch("pfs.pipe._is_stacked_header_gpci", return_value=True)
|
|
def test_stacked_header(self, mock_stacked, mock_read, con):
|
|
"""Stacked header file goes through _read_multi_year_gpci."""
|
|
# _read_multi_year_gpci is called once for primary load;
|
|
# then gap fill also calls it for years found in the file.
|
|
# Since _is_stacked_header_gpci returns True, the gap fill
|
|
# code tries to detect available years. We use a .txt extension
|
|
# so it uses open() rather than openpyxl.
|
|
mock_read.return_value = pl.DataFrame(
|
|
{
|
|
"mac": ["01"],
|
|
"locality": ["00"],
|
|
"locality_name": ["ALABAMA"],
|
|
"work_gpci": ["1.0"],
|
|
"pe_gpci": ["0.9"],
|
|
"mp_gpci": ["0.8"],
|
|
}
|
|
)
|
|
# Use a real temp file so the gap fill code can read it
|
|
import tempfile
|
|
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="w", suffix=".txt", delete=False, encoding="utf-8"
|
|
) as f:
|
|
f.write(
|
|
"Carrier\tLocality\tName\t2017\n"
|
|
"\t\t\tPW GPCI\tPE GPCI\tMP GPCI\n"
|
|
"01\t00\tALABAMA\t1.0\t0.9\t0.8\n"
|
|
)
|
|
tmppath = f.name
|
|
|
|
files = [
|
|
_make_file(
|
|
"GPCI2017.txt",
|
|
2017,
|
|
".txt",
|
|
path=tmppath,
|
|
)
|
|
]
|
|
result = _load_gpci_files(files, con)
|
|
assert result["rows"] >= 1
|
|
|
|
import os
|
|
|
|
os.unlink(tmppath)
|
|
|
|
@patch("pfs.pipe._is_stacked_header_gpci", return_value=False)
|
|
@patch("pfs.pipe._read_excel")
|
|
def test_dedup_by_year(self, mock_read, mock_stacked, con):
|
|
"""Only one file per year is loaded (highest priority)."""
|
|
mock_read.return_value = self._make_gpci_df()
|
|
files = [
|
|
_make_file(
|
|
"GPCI2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
title="CY 2024 PFS Final Rule",
|
|
path="/fake/final.xlsx",
|
|
),
|
|
_make_file(
|
|
"Addendum_E_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
title="CY 2024 PFS Proposed Rule",
|
|
path="/fake/proposed.xlsx",
|
|
),
|
|
]
|
|
result = _load_gpci_files(files, con)
|
|
assert result["files"] == 1
|
|
|
|
@patch("pfs.pipe._read_multi_year_gpci")
|
|
@patch("pfs.pipe._is_stacked_header_gpci")
|
|
def test_multi_year_gap_filling(self, mock_stacked, mock_read, con, tmp_path):
|
|
"""Multi-year files fill gap years not already loaded."""
|
|
# Create a real .txt file for the multi-year gap fill detection
|
|
content = (
|
|
"Carrier\tLocality\tName\t2015\t\t\t2016\n"
|
|
"\t\t\tPW GPCI\tPE GPCI\tMP GPCI\tPW GPCI\n"
|
|
"01\t00\tALABAMA\t1.0\t0.9\t0.8\t1.01\n"
|
|
)
|
|
f = tmp_path / "gpci_multi.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
|
|
# The file is for year 2016, but contains 2015 too
|
|
def stacked_side_effect(path, ext):
|
|
return str(f) in str(path)
|
|
|
|
mock_stacked.side_effect = stacked_side_effect
|
|
|
|
mock_read.return_value = pl.DataFrame(
|
|
{
|
|
"mac": ["01"],
|
|
"locality": ["00"],
|
|
"locality_name": ["ALABAMA"],
|
|
"work_gpci": ["1.0"],
|
|
"pe_gpci": ["0.9"],
|
|
"mp_gpci": ["0.8"],
|
|
}
|
|
)
|
|
|
|
files = [
|
|
_make_file(
|
|
"GPCI2016_multi.txt",
|
|
2016,
|
|
".txt",
|
|
path=str(f),
|
|
)
|
|
]
|
|
result = _load_gpci_files(files, con)
|
|
# Should load 2016 (primary) + 2015 (gap fill) = 2 files
|
|
assert result["files"] == 2
|
|
|
|
|
|
# ── _read_tabular ─────────────────────────────────────────────
|
|
|
|
|
|
class TestReadTabular:
|
|
"""Dispatch by extension."""
|
|
|
|
@patch("pfs.pipe._read_tsv")
|
|
def test_txt_path(self, mock_tsv):
|
|
mock_tsv.return_value = pl.DataFrame({"a": [1]})
|
|
result = _read_tabular("/fake.txt", ".txt")
|
|
mock_tsv.assert_called_once_with("/fake.txt")
|
|
assert len(result) == 1
|
|
|
|
@patch("pfs.pipe._read_csv_with_header")
|
|
def test_csv_path(self, mock_csv):
|
|
mock_csv.return_value = pl.DataFrame({"a": [1]})
|
|
result = _read_tabular("/fake.csv", ".csv")
|
|
mock_csv.assert_called_once_with("/fake.csv")
|
|
assert len(result) == 1
|
|
|
|
@patch("pfs.pipe._read_excel")
|
|
def test_xlsx_path(self, mock_xlsx):
|
|
mock_xlsx.return_value = pl.DataFrame({"a": [1]})
|
|
result = _read_tabular("/fake.xlsx", ".xlsx")
|
|
mock_xlsx.assert_called_once_with("/fake.xlsx")
|
|
assert len(result) == 1
|
|
|
|
|
|
# ── _load_pe_files ────────────────────────────────────────────
|
|
|
|
|
|
class TestLoadPeFiles:
|
|
"""Load labor/supply/equipment/work_time files."""
|
|
|
|
@patch("pfs.pipe._read_tabular")
|
|
def test_labor_nf_minutes_computation(self, mock_read, con):
|
|
"""nf_minutes computed from nf_* sub-columns when not present."""
|
|
mock_read.return_value = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213"],
|
|
"labor_code": ["L1"],
|
|
"nf_pre_svc": ["5"],
|
|
"nf_svc": ["10"],
|
|
"nf_post_svc": ["3"],
|
|
"rate_per_minute": ["0.5"],
|
|
}
|
|
)
|
|
files = [
|
|
_make_file(
|
|
"PUF_LABOR_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/labor.xlsx",
|
|
)
|
|
]
|
|
result = _load_pe_files(files, con)
|
|
assert result["clinical_labor"]["rows"] == 1
|
|
rows = con.execute('SELECT * FROM "pfs"."clinical_labor"').fetchall()
|
|
assert len(rows) == 1
|
|
|
|
@patch("pfs.pipe._read_tabular")
|
|
def test_supply(self, mock_read, con):
|
|
"""Supply file loads correctly."""
|
|
mock_read.return_value = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213"],
|
|
"cms_code": ["S1"],
|
|
"description": ["Gauze"],
|
|
"price": ["1.50"],
|
|
"nf_quantity": ["2"],
|
|
"f_quantity": ["1"],
|
|
}
|
|
)
|
|
files = [
|
|
_make_file(
|
|
"PUF_SUPPLY_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/supply.xlsx",
|
|
)
|
|
]
|
|
result = _load_pe_files(files, con)
|
|
assert result["medical_supply"]["rows"] == 1
|
|
|
|
@patch("pfs.pipe._read_tabular")
|
|
def test_equipment(self, mock_read, con):
|
|
"""Equipment file loads correctly."""
|
|
mock_read.return_value = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213"],
|
|
"cms_code": ["EQ1"],
|
|
"description": ["Exam table"],
|
|
"useful_life": ["10"],
|
|
"price": ["5000"],
|
|
"nf_time": ["30"],
|
|
"f_time": ["15"],
|
|
}
|
|
)
|
|
files = [
|
|
_make_file(
|
|
"PUF_EQUIPMENT_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/equip.xlsx",
|
|
)
|
|
]
|
|
result = _load_pe_files(files, con)
|
|
assert result["medical_equipment"]["rows"] == 1
|
|
|
|
@patch("pfs.pipe._read_tabular")
|
|
def test_work_time_hcpcs_int_padding(self, mock_read, con):
|
|
"""Integer hcpcs values are zero-padded to 5 chars."""
|
|
mock_read.return_value = pl.DataFrame(
|
|
{
|
|
"cpt_code": [100, 99213],
|
|
"Pre_Evaluation_Time": [5, 10],
|
|
"Median_Intra_Service_Time": [15, 20],
|
|
}
|
|
)
|
|
files = [
|
|
_make_file(
|
|
"WORK_TIME_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/wt.xlsx",
|
|
)
|
|
]
|
|
result = _load_pe_files(files, con)
|
|
assert result["physician_work_time"]["rows"] == 2
|
|
rows = con.execute(
|
|
'SELECT hcpcs FROM "pfs"."physician_work_time" ORDER BY hcpcs'
|
|
).fetchall()
|
|
assert rows[0][0] == "00100"
|
|
assert rows[1][0] == "99213"
|
|
|
|
@patch("pfs.pipe._read_tabular")
|
|
def test_work_time_total_time_computation(self, mock_read, con):
|
|
"""total_time computed from sub-columns when not present."""
|
|
mock_read.return_value = pl.DataFrame(
|
|
{
|
|
"cpt_code": ["99213"],
|
|
"Pre_Evaluation_Time": ["5"],
|
|
"Median_Intra_Service_Time": ["15"],
|
|
"Immediate_post_Service_time": ["3"],
|
|
}
|
|
)
|
|
files = [
|
|
_make_file(
|
|
"WORK_TIME_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/wt.xlsx",
|
|
)
|
|
]
|
|
result = _load_pe_files(files, con)
|
|
assert result["physician_work_time"]["rows"] == 1
|
|
row = con.execute(
|
|
'SELECT total_time FROM "pfs"."physician_work_time"'
|
|
).fetchone()
|
|
# 5 + 15 + 3 = 23
|
|
assert row[0] == 23.0
|
|
|
|
@patch("pfs.pipe._read_tabular")
|
|
def test_skip_labor_no_hcpcs(self, mock_read, con):
|
|
"""Labor file without hcpcs after normalization is skipped."""
|
|
mock_read.return_value = pl.DataFrame({"unknown_col": ["val"], "rate": ["1.0"]})
|
|
files = [
|
|
_make_file(
|
|
"PUF_LABOR_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
path="/fake/labor.xlsx",
|
|
)
|
|
]
|
|
result = _load_pe_files(files, con)
|
|
assert result["clinical_labor"]["rows"] == 0
|
|
|
|
@patch("pfs.pipe._read_tabular")
|
|
def test_final_preferred_over_proposed(self, mock_read, con):
|
|
"""Final rule file is preferred over proposed for same year."""
|
|
mock_read.return_value = pl.DataFrame(
|
|
{
|
|
"hcpcs": ["99213"],
|
|
"cms_code": ["S1"],
|
|
"description": ["Gauze"],
|
|
"price": ["1.50"],
|
|
}
|
|
)
|
|
files = [
|
|
_make_file(
|
|
"PUF_SUPPLY_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
title="CY 2024 PFS Proposed Rule",
|
|
path="/fake/proposed.xlsx",
|
|
),
|
|
_make_file(
|
|
"PUF_SUPPLY_2024.xlsx",
|
|
2024,
|
|
".xlsx",
|
|
title="CY 2024 PFS Final Rule",
|
|
path="/fake/final.xlsx",
|
|
),
|
|
]
|
|
result = _load_pe_files(files, con)
|
|
assert result["medical_supply"]["files"] == 1
|
|
mock_read.assert_called_once_with("/fake/final.xlsx", ".xlsx")
|
|
|
|
|
|
# ── _load_zip_carrier_files ───────────────────────────────────
|
|
|
|
|
|
class TestLoadZipCarrierFiles:
|
|
"""Load fixed-width ZIP5 carrier locality files."""
|
|
|
|
def _make_zip_line(
|
|
self,
|
|
state="AL",
|
|
zip_code="35004",
|
|
carrier="01102",
|
|
locality="00",
|
|
rural=" ",
|
|
plus_four=" ",
|
|
part_b=" ",
|
|
year_quarter="20241",
|
|
):
|
|
"""Build a fixed-width ZIP5 line per _ZIP5_COLUMNS spec."""
|
|
# Positions: state(0-2), zip(2-7), carrier(7-12), locality(12-14),
|
|
# rural(14-15), 15-20 filler, plus_four(20-21), filler(21-22),
|
|
# part_b(22-23), 23-75 filler, year_quarter(75-80)
|
|
line = state.ljust(2) # 0-2
|
|
line += zip_code.ljust(5) # 2-7
|
|
line += carrier.ljust(5) # 7-12
|
|
line += locality.ljust(2) # 12-14
|
|
line += rural # 14-15
|
|
line += " " * 5 # 15-20 filler
|
|
line += plus_four # 20-21
|
|
line += " " # 21-22 filler
|
|
line += part_b # 22-23
|
|
line += " " * 52 # 23-75 filler
|
|
line += year_quarter.ljust(5) # 75-80
|
|
return line + "\n"
|
|
|
|
def test_successful_load(self, con, tmp_path):
|
|
content = self._make_zip_line()
|
|
f = tmp_path / "ZIP5_2024.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
files = [_make_file("ZIP5_2024.txt", 2024, ".txt", path=str(f))]
|
|
result = _load_zip_carrier_files(files, con)
|
|
assert result["rows"] == 1
|
|
assert result["files"] == 1
|
|
rows = con.execute('SELECT * FROM "pfs"."zip_carrier_locality"').fetchall()
|
|
assert len(rows) == 1
|
|
|
|
def test_short_line_skipped(self, con, tmp_path):
|
|
"""Lines shorter than 14 chars are skipped."""
|
|
content = "short line\n" + self._make_zip_line()
|
|
f = tmp_path / "ZIP5_2024.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
files = [_make_file("ZIP5_2024.txt", 2024, ".txt", path=str(f))]
|
|
result = _load_zip_carrier_files(files, con)
|
|
assert result["rows"] == 1
|
|
|
|
def test_year_quarter_parsing(self, con, tmp_path):
|
|
"""year_quarter field is split into year and quarter columns."""
|
|
content = self._make_zip_line(year_quarter="20243")
|
|
f = tmp_path / "ZIP5_2024.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
files = [_make_file("ZIP5_2024.txt", 2024, ".txt", path=str(f))]
|
|
_load_zip_carrier_files(files, con)
|
|
row = con.execute(
|
|
'SELECT year, quarter FROM "pfs"."zip_carrier_locality"'
|
|
).fetchone()
|
|
assert row[0] == 2024
|
|
assert row[1] == 3
|
|
|
|
def test_no_year_quarter_fallback(self, con, tmp_path):
|
|
"""When year_quarter column is absent, year from metadata is used."""
|
|
# Build a very short line that has no year_quarter (position 75-80)
|
|
# but still >= 14 chars
|
|
content = self._make_zip_line(year_quarter=" ")
|
|
f = tmp_path / "ZIP5_2024.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
files = [_make_file("ZIP5_2024.txt", 2024, ".txt", path=str(f))]
|
|
_load_zip_carrier_files(files, con)
|
|
row = con.execute('SELECT year FROM "pfs"."zip_carrier_locality"').fetchone()
|
|
# year_quarter col exists but is blank -> cast to null
|
|
assert row[0] is None or row[0] == 2024
|
|
|
|
def test_empty_rows_filtered(self, con, tmp_path):
|
|
"""Rows with empty zip_code are filtered out."""
|
|
line = self._make_zip_line(zip_code=" ")
|
|
content = line + self._make_zip_line(zip_code="35004")
|
|
f = tmp_path / "ZIP5_2024.txt"
|
|
f.write_text(content, encoding="utf-8")
|
|
files = [_make_file("ZIP5_2024.txt", 2024, ".txt", path=str(f))]
|
|
result = _load_zip_carrier_files(files, con)
|
|
assert result["rows"] == 1
|
|
|
|
def test_empty_file(self, con, tmp_path):
|
|
"""Empty file results in zero rows."""
|
|
f = tmp_path / "ZIP5_2024.txt"
|
|
f.write_text("", encoding="utf-8")
|
|
files = [_make_file("ZIP5_2024.txt", 2024, ".txt", path=str(f))]
|
|
result = _load_zip_carrier_files(files, con)
|
|
assert result["rows"] == 0
|
|
|
|
def test_non_txt_skipped(self, con):
|
|
"""Non-.txt files are ignored."""
|
|
files = [_make_file("ZIP5_2024.xlsx", 2024, ".xlsx", path="/fake.xlsx")]
|
|
result = _load_zip_carrier_files(files, con)
|
|
assert result["rows"] == 0
|
|
|
|
|
|
# ── _load_carrier_files ───────────────────────────────────────
|
|
|
|
|
|
class TestLoadCarrierFiles:
|
|
"""Load carrier fee schedule files."""
|
|
|
|
def _write_carrier_16(self, tmp_path, filename="PFALL24.TXT"):
|
|
"""Write a 16-column carrier file."""
|
|
# year,mac,locality,hcpcs,mod,nf_fee,f_fee,filler,pctc,status,...
|
|
line = '"2024","01102","00","99213","","45.00","30.00",'
|
|
line += '"","","","","","","","",""\n'
|
|
f = tmp_path / filename
|
|
f.write_text(line, encoding="utf-8")
|
|
return str(f)
|
|
|
|
def _write_carrier_8(self, tmp_path, filename="PFALL09.TXT"):
|
|
"""Write an 8-column carrier file."""
|
|
line = '"2009","01102","00","99213","","45.00","30.00",""\n'
|
|
f = tmp_path / filename
|
|
f.write_text(line, encoding="utf-8")
|
|
return str(f)
|
|
|
|
def test_16_col_format(self, con, tmp_path):
|
|
path = self._write_carrier_16(tmp_path)
|
|
files = [_make_file("PFALL24.TXT", 2024, ".txt", path=path)]
|
|
result = _load_carrier_files(files, con)
|
|
assert result["rows"] == 1
|
|
assert result["files"] == 1
|
|
rows = con.execute('SELECT * FROM "pfs"."carrier_locality"').fetchall()
|
|
assert len(rows) == 1
|
|
|
|
def test_8_col_format(self, con, tmp_path):
|
|
path = self._write_carrier_8(tmp_path)
|
|
files = [_make_file("PFALL09.TXT", 2009, ".txt", path=path)]
|
|
result = _load_carrier_files(files, con)
|
|
assert result["rows"] == 1
|
|
|
|
def test_nonqp_preference(self, con, tmp_path):
|
|
"""NonQP files preferred when both exist for same year."""
|
|
path_qp = self._write_carrier_16(tmp_path, "PFALL26_QP.TXT")
|
|
path_nonqp = self._write_carrier_16(tmp_path, "PFALL26_NONQP.TXT")
|
|
files = [
|
|
_make_file(
|
|
"PFALL26_QP.TXT",
|
|
2026,
|
|
".txt",
|
|
title="CY 2026 PFS QP",
|
|
path=path_qp,
|
|
),
|
|
_make_file(
|
|
"PFALL26_NONQP.TXT",
|
|
2026,
|
|
".txt",
|
|
title="CY 2026 PFS Nonqp",
|
|
path=path_nonqp,
|
|
),
|
|
]
|
|
result = _load_carrier_files(files, con)
|
|
assert result["files"] == 1
|
|
|
|
def test_internal_col_drop(self, con, tmp_path):
|
|
"""Internal columns starting with _ are dropped."""
|
|
path = self._write_carrier_16(tmp_path)
|
|
files = [_make_file("PFALL24.TXT", 2024, ".txt", path=path)]
|
|
_load_carrier_files(files, con)
|
|
cols = [
|
|
r[0]
|
|
for r in con.execute(
|
|
"SELECT column_name FROM information_schema.columns "
|
|
"WHERE table_schema = 'pfs' AND table_name = 'carrier_locality'"
|
|
).fetchall()
|
|
]
|
|
assert not any(c.startswith("_") for c in cols)
|
|
|
|
def test_fee_casting(self, con, tmp_path):
|
|
"""Fee columns are cast to Float64."""
|
|
path = self._write_carrier_16(tmp_path)
|
|
files = [_make_file("PFALL24.TXT", 2024, ".txt", path=path)]
|
|
_load_carrier_files(files, con)
|
|
row = con.execute(
|
|
'SELECT non_fac_fee, fac_fee FROM "pfs"."carrier_locality"'
|
|
).fetchone()
|
|
assert row[0] == 45.0
|
|
assert row[1] == 30.0
|
|
|
|
def test_limiting_charge_computation(self, con, tmp_path):
|
|
"""Limiting charge = fee * 1.0925, rounded to 2 decimals."""
|
|
path = self._write_carrier_16(tmp_path)
|
|
files = [_make_file("PFALL24.TXT", 2024, ".txt", path=path)]
|
|
_load_carrier_files(files, con)
|
|
row = con.execute(
|
|
"SELECT non_fac_limiting_charge, fac_limiting_charge "
|
|
'FROM "pfs"."carrier_locality"'
|
|
).fetchone()
|
|
assert row[0] == pytest.approx(45.0 * 1.0925, abs=0.01)
|
|
assert row[1] == pytest.approx(30.0 * 1.0925, abs=0.01)
|
|
|
|
def test_concat_year_frames(self, con, tmp_path):
|
|
"""Multiple files for same year are concatenated."""
|
|
path1 = self._write_carrier_16(tmp_path, "PFAL24.TXT")
|
|
path2 = self._write_carrier_16(tmp_path, "PFNY24.TXT")
|
|
files = [
|
|
_make_file("PFAL24.TXT", 2024, ".txt", path=path1),
|
|
_make_file("PFNY24.TXT", 2024, ".txt", path=path2),
|
|
]
|
|
result = _load_carrier_files(files, con)
|
|
assert result["rows"] == 2
|
|
assert result["files"] == 1 # 1 year = 1 "file" insert
|
|
|
|
def test_non_txt_skipped(self, con):
|
|
"""Non-.txt carrier files are ignored."""
|
|
files = [_make_file("PFALL24.xlsx", 2024, ".xlsx", path="/fake.xlsx")]
|
|
result = _load_carrier_files(files, con)
|
|
assert result["rows"] == 0
|
|
|
|
|
|
# ── _insert_into ──────────────────────────────────────────────
|
|
|
|
|
|
class TestInsertInto:
|
|
"""DuckDB insertion: create schema, table, insert with shared cols."""
|
|
|
|
def test_create_table_from_df(self, con):
|
|
"""Table doesn't exist -> creates from df structure then inserts."""
|
|
df = pl.DataFrame({"hcpcs": ["99213"], "work_rvu": [1.3]})
|
|
_insert_into(con, "pfs", "test_tbl", df)
|
|
rows = con.execute('SELECT * FROM "pfs"."test_tbl"').fetchall()
|
|
assert len(rows) == 1
|
|
|
|
def test_insert_with_overlapping_columns(self, con):
|
|
"""Table exists with overlapping columns -> INSERT INTO."""
|
|
df1 = pl.DataFrame({"hcpcs": ["99213"], "work_rvu": [1.3]})
|
|
_insert_into(con, "pfs", "test_tbl", df1)
|
|
df2 = pl.DataFrame({"hcpcs": ["99214"], "work_rvu": [1.92]})
|
|
_insert_into(con, "pfs", "test_tbl", df2)
|
|
rows = con.execute('SELECT * FROM "pfs"."test_tbl"').fetchall()
|
|
assert len(rows) == 2
|
|
|
|
def test_no_shared_columns_recreate(self, con):
|
|
"""No shared columns -> drops and recreates table."""
|
|
df1 = pl.DataFrame({"hcpcs": ["99213"], "work_rvu": [1.3]})
|
|
_insert_into(con, "pfs", "test_tbl", df1)
|
|
# Completely different columns
|
|
df2 = pl.DataFrame({"mac": ["01102"], "locality": ["00"]})
|
|
_insert_into(con, "pfs", "test_tbl", df2)
|
|
rows = con.execute('SELECT * FROM "pfs"."test_tbl"').fetchall()
|
|
assert len(rows) == 1
|
|
cols = [
|
|
r[0]
|
|
for r in con.execute(
|
|
"SELECT column_name FROM information_schema.columns "
|
|
"WHERE table_schema = 'pfs' AND table_name = 'test_tbl'"
|
|
).fetchall()
|
|
]
|
|
assert "mac" in cols
|
|
assert "hcpcs" not in cols
|
|
|
|
def test_missing_float_column_added_as_double(self, con):
|
|
"""Missing Float64 column -> ALTER TABLE ADD COLUMN DOUBLE."""
|
|
df1 = pl.DataFrame({"hcpcs": ["99213"]})
|
|
_insert_into(con, "pfs", "test_tbl", df1)
|
|
df2 = pl.DataFrame(
|
|
{"hcpcs": ["99214"], "work_rvu": pl.Series([1.92], dtype=pl.Float64)}
|
|
)
|
|
_insert_into(con, "pfs", "test_tbl", df2)
|
|
cols = con.execute(
|
|
"SELECT column_name, data_type FROM information_schema.columns "
|
|
"WHERE table_schema = 'pfs' AND table_name = 'test_tbl'"
|
|
).fetchall()
|
|
col_types = {name: dtype for name, dtype in cols}
|
|
assert "work_rvu" in col_types
|
|
assert col_types["work_rvu"] == "DOUBLE"
|
|
|
|
def test_missing_int_column_added_as_bigint(self, con):
|
|
"""Missing Int64 column -> ALTER TABLE ADD COLUMN BIGINT."""
|
|
df1 = pl.DataFrame({"hcpcs": ["99213"]})
|
|
_insert_into(con, "pfs", "test_tbl", df1)
|
|
df2 = pl.DataFrame(
|
|
{"hcpcs": ["99214"], "year": pl.Series([2024], dtype=pl.Int64)}
|
|
)
|
|
_insert_into(con, "pfs", "test_tbl", df2)
|
|
cols = con.execute(
|
|
"SELECT column_name, data_type FROM information_schema.columns "
|
|
"WHERE table_schema = 'pfs' AND table_name = 'test_tbl'"
|
|
).fetchall()
|
|
col_types = {name: dtype for name, dtype in cols}
|
|
assert "year" in col_types
|
|
assert col_types["year"] == "BIGINT"
|
|
|
|
def test_missing_string_column_added_as_varchar(self, con):
|
|
"""Missing Utf8 column -> ALTER TABLE ADD COLUMN VARCHAR."""
|
|
df1 = pl.DataFrame({"hcpcs": ["99213"]})
|
|
_insert_into(con, "pfs", "test_tbl", df1)
|
|
df2 = pl.DataFrame({"hcpcs": ["99214"], "description": ["Office visit"]})
|
|
_insert_into(con, "pfs", "test_tbl", df2)
|
|
cols = con.execute(
|
|
"SELECT column_name, data_type FROM information_schema.columns "
|
|
"WHERE table_schema = 'pfs' AND table_name = 'test_tbl'"
|
|
).fetchall()
|
|
col_types = {name: dtype for name, dtype in cols}
|
|
assert "description" in col_types
|
|
assert col_types["description"] == "VARCHAR"
|
|
|
|
def test_schema_created_if_not_exists(self):
|
|
"""Schema is created if it doesn't exist."""
|
|
c = duckdb.connect(":memory:")
|
|
df = pl.DataFrame({"x": [1]})
|
|
_insert_into(c, "new_schema", "tbl", df)
|
|
rows = c.execute('SELECT * FROM "new_schema"."tbl"').fetchall()
|
|
assert len(rows) == 1
|
|
c.close()
|
|
|
|
|
|
# ── load_all ──────────────────────────────────────────────────
|
|
|
|
|
|
class TestLoadAll:
|
|
"""Main entry point orchestration."""
|
|
|
|
@patch("pfs.pipe._load_zip_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch(
|
|
"pfs.pipe._load_pe_files",
|
|
return_value={
|
|
"clinical_labor": {"rows": 0, "files": 0},
|
|
"medical_supply": {"rows": 0, "files": 0},
|
|
"medical_equipment": {"rows": 0, "files": 0},
|
|
"physician_work_time": {"rows": 0, "files": 0},
|
|
},
|
|
)
|
|
@patch("pfs.pipe._load_gpci_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_rvu_files", return_value={"rows": 10, "files": 2})
|
|
@patch("pfs.pipe._zotero_pfs_files")
|
|
def test_orchestration(
|
|
self, mock_zotero, mock_rvu, mock_gpci, mock_pe, mock_carrier, mock_zip, con
|
|
):
|
|
"""load_all discovers files and calls all loaders."""
|
|
mock_zotero.return_value = [
|
|
_make_file("Addendum_B_2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("GPCI2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("PUF_LABOR_2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("PFALL24.TXT", 2024, ".txt"),
|
|
_make_file("ZIP5_2024.txt", 2024, ".txt"),
|
|
]
|
|
result = load_all(con)
|
|
assert "pfs.rvu" in result
|
|
assert result["pfs.rvu"]["rows"] == 10
|
|
mock_rvu.assert_called_once()
|
|
mock_gpci.assert_called_once()
|
|
mock_pe.assert_called_once()
|
|
mock_carrier.assert_called_once()
|
|
mock_zip.assert_called_once()
|
|
|
|
@patch("pfs.pipe._load_zip_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch(
|
|
"pfs.pipe._load_pe_files",
|
|
return_value={
|
|
"clinical_labor": {"rows": 0, "files": 0},
|
|
"medical_supply": {"rows": 0, "files": 0},
|
|
"medical_equipment": {"rows": 0, "files": 0},
|
|
"physician_work_time": {"rows": 0, "files": 0},
|
|
},
|
|
)
|
|
@patch("pfs.pipe._load_gpci_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_rvu_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._zotero_pfs_files")
|
|
def test_year_range_filter(
|
|
self, mock_zotero, mock_rvu, mock_gpci, mock_pe, mock_carrier, mock_zip, con
|
|
):
|
|
"""Year range filter excludes files outside range."""
|
|
mock_zotero.return_value = [
|
|
_make_file("Addendum_B_2023.xlsx", 2023, ".xlsx"),
|
|
_make_file("Addendum_B_2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("Addendum_B_2025.xlsx", 2025, ".xlsx"),
|
|
]
|
|
load_all(con, years=(2024, 2024))
|
|
# Should only pass 2024 files to loaders
|
|
rvu_files = mock_rvu.call_args[0][0]
|
|
assert len(rvu_files) == 1
|
|
assert rvu_files[0]["year"] == 2024
|
|
|
|
@patch("pfs.pipe._load_zip_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch(
|
|
"pfs.pipe._load_pe_files",
|
|
return_value={
|
|
"clinical_labor": {"rows": 0, "files": 0},
|
|
"medical_supply": {"rows": 0, "files": 0},
|
|
"medical_equipment": {"rows": 0, "files": 0},
|
|
"physician_work_time": {"rows": 0, "files": 0},
|
|
},
|
|
)
|
|
@patch("pfs.pipe._load_gpci_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_rvu_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._zotero_pfs_files")
|
|
def test_schema_creation_and_table_drop(
|
|
self, mock_zotero, mock_rvu, mock_gpci, mock_pe, mock_carrier, mock_zip, con
|
|
):
|
|
"""Verifies schema is created and tables are dropped for clean load."""
|
|
# Pre-create a table that should be dropped
|
|
con.execute('CREATE TABLE "pfs"."rvu" (x INTEGER)')
|
|
con.execute('INSERT INTO "pfs"."rvu" VALUES (1)')
|
|
mock_zotero.return_value = []
|
|
load_all(con)
|
|
# Table should have been dropped (no longer exists or is empty)
|
|
try:
|
|
rows = con.execute('SELECT * FROM "pfs"."rvu"').fetchall()
|
|
# If table exists from loader, it should be empty since no files
|
|
assert len(rows) == 0
|
|
except duckdb.CatalogException:
|
|
pass # Table was dropped, that's fine
|
|
|
|
@patch("pfs.pipe._load_zip_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch(
|
|
"pfs.pipe._load_pe_files",
|
|
return_value={
|
|
"clinical_labor": {"rows": 0, "files": 0},
|
|
"medical_supply": {"rows": 0, "files": 0},
|
|
"medical_equipment": {"rows": 0, "files": 0},
|
|
"physician_work_time": {"rows": 0, "files": 0},
|
|
},
|
|
)
|
|
@patch("pfs.pipe._load_gpci_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_rvu_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._zotero_pfs_files")
|
|
def test_file_classification(
|
|
self, mock_zotero, mock_rvu, mock_gpci, mock_pe, mock_carrier, mock_zip, con
|
|
):
|
|
"""Files are classified correctly to each loader."""
|
|
mock_zotero.return_value = [
|
|
_make_file("Addendum_B_2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("PPRRVU24A.xlsx", 2024, ".xlsx"),
|
|
_make_file("GPCI2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("Addendum E 2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("PUF_LABOR_2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("PUF_SUPPLY_2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("PUF_EQUIPMENT_2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("WORK_TIME_2024.xlsx", 2024, ".xlsx"),
|
|
_make_file("PFALL24.TXT", 2024, ".txt"),
|
|
_make_file("ZIP5_2024.txt", 2024, ".txt"),
|
|
]
|
|
load_all(con)
|
|
|
|
# Check RVU got Addendum B + PPRRVU
|
|
rvu_files = mock_rvu.call_args[0][0]
|
|
assert len(rvu_files) == 2
|
|
|
|
# Check GPCI got GPCI + Addendum E
|
|
gpci_files = mock_gpci.call_args[0][0]
|
|
assert len(gpci_files) == 2
|
|
|
|
# Check PE got labor + supply + equipment + work_time
|
|
pe_files = mock_pe.call_args[0][0]
|
|
assert len(pe_files) == 4
|
|
|
|
# Check carrier
|
|
carrier_files = mock_carrier.call_args[0][0]
|
|
assert len(carrier_files) == 1
|
|
|
|
# Check zip carrier
|
|
zip_files = mock_zip.call_args[0][0]
|
|
assert len(zip_files) == 1
|
|
|
|
@patch("pfs.pipe._load_zip_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_carrier_files", return_value={"rows": 0, "files": 0})
|
|
@patch(
|
|
"pfs.pipe._load_pe_files",
|
|
return_value={
|
|
"clinical_labor": {"rows": 0, "files": 0},
|
|
"medical_supply": {"rows": 0, "files": 0},
|
|
"medical_equipment": {"rows": 0, "files": 0},
|
|
"physician_work_time": {"rows": 0, "files": 0},
|
|
},
|
|
)
|
|
@patch("pfs.pipe._load_gpci_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._load_rvu_files", return_value={"rows": 0, "files": 0})
|
|
@patch("pfs.pipe._zotero_pfs_files")
|
|
def test_returns_all_stat_keys(
|
|
self, mock_zotero, mock_rvu, mock_gpci, mock_pe, mock_carrier, mock_zip, con
|
|
):
|
|
"""Return dict includes all table keys."""
|
|
mock_zotero.return_value = []
|
|
result = load_all(con)
|
|
expected = {
|
|
"pfs.rvu",
|
|
"pfs.gpci",
|
|
"pfs.clinical_labor",
|
|
"pfs.medical_supply",
|
|
"pfs.medical_equipment",
|
|
"pfs.physician_work_time",
|
|
"pfs.carrier_locality",
|
|
"pfs.zip_carrier_locality",
|
|
}
|
|
assert set(result.keys()) == expected
|