stack/tests/aco/test_load_cclf.py

"""Tests for CCLF file loading."""

from __future__ import annotations

import zipfile
from datetime import date
from pathlib import Path

import pytest

from aco.load.cclf import (
    _extract_zip,
    _parse_value,
    _polars_type,
    discover_cclf_files,
    load_cclf_directory,
    parse_cclf_file,
)
from aco.table.cclf_layout import LAYOUTS


class TestParseValue:
    def test_empty_string(self) -> None:
        assert _parse_value("", "X(11)") is None

    def test_whitespace_only(self) -> None:
        assert _parse_value("   ", "X(11)") is None

    def test_string_field(self) -> None:
        assert _parse_value("1234567890", "X(10)") == "1234567890"

    def test_date_field(self) -> None:
        assert _parse_value("2025-07-16", "YYYY-MM-DD") == date(2025, 7, 16)

    def test_date_invalid(self) -> None:
        assert _parse_value("0000-00-00", "YYYY-MM-DD") is None

    def test_numeric_field(self) -> None:
        assert _parse_value("123.45", "-9(13).99") == 123.45

    def test_numeric_invalid(self) -> None:
        assert _parse_value("N/A", "-9(13).99") is None

    def test_numeric_9_format(self) -> None:
        assert _parse_value("42", "9(13)") == "42"


class TestParseCclfFile:
    def test_parse_cclf9(self) -> None:
        """CCLF9 is the simplest file: 6 fields, 55 bytes per line."""
        # Build a realistic CCLF9 line: positions 1-55
        line = (
            "N"  # hicn_mbi_xref_ind (1-1)
            "1AN0Y00AA04"  # crnt_num (2-12)
            "2AN0Y00AA05"  # prvs_num (13-23)
            "2024-01-15"  # prvs_id_efctv_dt (24-33)
            "2025-06-30"  # prvs_id_obslt_dt (34-43)
            "RRB123456789"  # bene_rrb_num (44-55)
        )
        df = parse_cclf_file([line], "cclf9")
        assert len(df) == 1
        assert df["hicn_mbi_xref_ind"][0] == "N"
        assert df["crnt_num"][0] == "1AN0Y00AA04"
        assert df["prvs_num"][0] == "2AN0Y00AA05"
        assert df["prvs_id_efctv_dt"][0] == date(2024, 1, 15)
        assert df["prvs_id_obslt_dt"][0] == date(2025, 6, 30)
        assert df["bene_rrb_num"][0] == "RRB123456789"

    def test_parse_empty_lines(self) -> None:
        df = parse_cclf_file(["", "  ", ""], "cclf9")
        assert df.is_empty()

    def test_parse_cclf8_date_and_string(self) -> None:
        """Check a CCLF8 line with date, numeric, and string fields."""
        # Build CCLF8 line (549 bytes)
        line = " " * 549
        parts = list(line)
        # bene_mbi_id (1-11)
        for i, c in enumerate("1AN0Y00AA04"):
            parts[i] = c
        # bene_dob (33-42)
        for i, c in enumerate("1945-03-22"):
            parts[32 + i] = c
        # bene_sex_cd (43)
        parts[42] = "1"
        # bene_race_cd (44)
        parts[43] = "2"
        line = "".join(parts)
        df = parse_cclf_file([line], "cclf8")
        assert len(df) == 1
        assert df["bene_mbi_id"][0] == "1AN0Y00AA04"
        assert df["bene_dob"][0] == date(1945, 3, 22)
        assert df["bene_sex_cd"][0] == "1"
        assert df["bene_race_cd"][0] == "2"

    def test_empty_file_returns_schema(self) -> None:
        df = parse_cclf_file([], "cclf9")
        assert df.is_empty()
        assert "crnt_num" in df.columns


class TestDiscoverCclfFiles:
    def test_discover_plain_files(self, tmp_path: Path) -> None:
        # Create a couple of fake CCLF files
        (tmp_path / "P.A1234.ACO.ZC1Y25.D250716.T1234567").write_text("data\n")
        (tmp_path / "P.A1234.ACO.ZC8Y25.D250716.T1234567").write_text("data\n")
        (tmp_path / "random.txt").write_text("noise\n")

        found = discover_cclf_files(tmp_path)
        assert "cclf1" in found
        assert "cclf8" in found
        assert len(found) == 2

    def test_discover_skips_hidden(self, tmp_path: Path) -> None:
        (tmp_path / ".extracted").mkdir()
        (tmp_path / "P.A1234.ACO.ZC1Y25.D250716.T1234567").write_text("data\n")
        found = discover_cclf_files(tmp_path)
        assert "cclf1" in found


class TestPolarsType:
    def test_date_format_returns_date(self) -> None:
        assert _polars_type("YYYY-MM-DD") is date

    def test_mm_dd_format_returns_date(self) -> None:
        assert _polars_type("MM-DD-YYYY") is date

    def test_v9_format_returns_float(self) -> None:
        assert _polars_type("9(13)V9(2)") is float

    def test_dot99_format_returns_float(self) -> None:
        assert _polars_type("-9(13).99") is float

    def test_dot9999_format_returns_float(self) -> None:
        assert _polars_type("9(13).9999") is float

    def test_default_returns_str(self) -> None:
        assert _polars_type("X(11)") is str


class TestLayouts:
    def test_all_pipeline_inputs_have_layouts(self) -> None:
        """Every CCLF table referenced by the pipeline must have a layout."""
        needed = {
            "cclf1",
            "cclf2",
            "cclf3",
            "cclf4",
            "cclf5",
            "cclf6",
            "cclf7",
            "cclf8",
            "cclf9",
        }
        assert needed.issubset(LAYOUTS.keys())

    def test_positions_are_sequential(self) -> None:
        """Field positions should be non-overlapping and ordered."""
        for table, fields in LAYOUTS.items():
            if table == "cclf0":
                continue  # CCLF0 has two record types sharing positions
            for name, start, end, fmt in fields:
                if name in ("blank", "delimiter", "filler"):
                    continue
                assert start <= end, f"{table}.{name}: start {start} > end {end}"
                assert start >= 1, f"{table}.{name}: start must be >= 1"


class TestExtractZip:
    def test_extracts_cclf_files_from_zip(self, tmp_path: Path) -> None:
        """ZIP file containing CCLF9 file is extracted."""
        # Create a CCLF9 file content
        cclf9_name = "P.A1234.ACO.ZC9Y25.D250716.T1234567"
        cclf9_content = b"N1AN0Y00AA042AN0Y00AA052024-01-152025-06-30RRB123456789\n"

        zip_path = tmp_path / "cclf.zip"
        with zipfile.ZipFile(zip_path, "w") as zf:
            zf.writestr(cclf9_name, cclf9_content)
            zf.writestr("README.txt", b"not a cclf file")

        dest = tmp_path / "extracted"
        dest.mkdir()

        extracted = _extract_zip(zip_path, dest)
        assert len(extracted) == 1
        assert extracted[0].name == cclf9_name

    def test_skips_non_cclf_files_in_zip(self, tmp_path: Path) -> None:
        """Non-CCLF files in a ZIP are not extracted."""
        zip_path = tmp_path / "cclf.zip"
        with zipfile.ZipFile(zip_path, "w") as zf:
            zf.writestr("README.txt", b"not cclf")
            zf.writestr("data.csv", b"col1,col2")

        dest = tmp_path / "extracted"
        dest.mkdir()

        extracted = _extract_zip(zip_path, dest)
        assert extracted == []


class TestDiscoverCclfFilesWithZip:
    def test_discovers_zip_extension_and_extracts(self, tmp_path: Path) -> None:
        """ZIP files with .zip extension + is_zip classify result are extracted."""
        from unittest.mock import patch

        from aco.table.cclf_filenames import CclfFilename

        cclf9_name = "P.A1234.ACO.ZC9Y25.D250716.T1234567"
        cclf9_content = b"N1AN0Y00AA042AN0Y00AA052024-01-152025-06-30RRB123456789\n"

        zip_path = tmp_path / "bundle.zip"
        with zipfile.ZipFile(zip_path, "w") as zf:
            zf.writestr(cclf9_name, cclf9_content)

        zip_info = CclfFilename(
            program="sssp",
            aco_id="1234",
            entity="ACO",
            file_id="",
            cclf_table="",
            run_type="Y",
            performance_year=2025,
            delivery_date="250716",
            delivery_time="1234567",
            is_zip=True,
        )
        cclf9_info = CclfFilename(
            program="sssp",
            aco_id="1234",
            entity="ACO",
            file_id="9",
            cclf_table="cclf9",
            run_type="Y",
            performance_year=2025,
            delivery_date="250716",
            delivery_time="1234567",
            is_zip=False,
        )

        def classify_mock(name):
            if name == "bundle.zip":
                return zip_info
            if name == cclf9_name:
                return cclf9_info
            return None

        with patch("aco.load.cclf.classify", side_effect=classify_mock):
            found = discover_cclf_files(tmp_path)

        assert "cclf9" in found


class TestLoadCclfDirectory:
    def test_no_files_raises(self, tmp_path: Path) -> None:
        with pytest.raises(FileNotFoundError):
            load_cclf_directory(tmp_path, run_pipeline=False)

    def test_load_raw_no_pipeline(self, tmp_path: Path) -> None:
        """Load a single CCLF9 file into a temp DuckDB (no pipeline)."""
        import duckdb

        # Create a fake CCLF9 file
        line = "N1AN0Y00AA042AN0Y00AA052024-01-152025-06-30RRB123456789"
        cclf_file = tmp_path / "P.A1234.ACO.ZC9Y25.D250716.T1234567"
        cclf_file.write_text(line + "\n")

        db_path = str(tmp_path / "test.duckdb")
        stats = load_cclf_directory(tmp_path, database=db_path, run_pipeline=False)
        assert stats["cclf9"] == 1

        # Verify it's in DuckDB
        con = duckdb.connect(db_path, read_only=True)
        rows = con.execute("SELECT * FROM cclf.cclf9").fetchall()
        assert len(rows) == 1
        assert rows[0][1] == "1AN0Y00AA04"  # crnt_num
        con.close()

    def test_skips_file_with_no_lines(self, tmp_path: Path) -> None:
        """Files with only whitespace/empty lines are skipped."""
        cclf_file = tmp_path / "P.A1234.ACO.ZC9Y25.D250716.T1234567"
        cclf_file.write_text("\n\n\n")  # Empty lines only

        db_path = str(tmp_path / "test.duckdb")
        stats = load_cclf_directory(tmp_path, database=db_path, run_pipeline=False)
        assert "cclf9" not in stats

    def test_skips_empty_dataframe(self, tmp_path: Path) -> None:
        """If parsing yields an empty DataFrame, the table is skipped."""
        from unittest.mock import patch

        cclf_file = tmp_path / "P.A1234.ACO.ZC9Y25.D250716.T1234567"
        cclf_file.write_text("some line\n")

        db_path = str(tmp_path / "test.duckdb")
        import polars as pl

        with patch("aco.load.cclf.parse_cclf_file", return_value=pl.DataFrame()):
            stats = load_cclf_directory(tmp_path, database=db_path, run_pipeline=False)
        assert "cclf9" not in stats