stack/tests/bcda/test_ui_service.py

"""bcda.ui.service — the manager's operations, with mock transports."""

from __future__ import annotations

import json
from datetime import datetime, timezone
from pathlib import Path

import duckdb
import httpx
import polars as pl
import pytest

from bcda.client import SANDBOX, Client
from bcda.store import Store
from bcda.ui import service as svc

ROOT = Path(__file__).resolve().parents[2]


def _token() -> httpx.Response:
    return httpx.Response(200, json={"access_token": "t" * 40, "expires_in": 1200})


def _client(handler, version: str = "v3", tmp_path: Path | None = None) -> Client:
    c = Client(
        "cid", "csec", version=version, output_dir=tmp_path or Path("/tmp/bcda-ui-test")
    )
    c._http = httpx.Client(transport=httpx.MockTransport(handler))
    return c


def _task(job_id: str, status: str, request: str, start: str, end: str) -> dict:
    return {
        "resourceType": "Task",
        "identifier": [{"value": job_id}],
        "status": status,
        "input": [{"valueString": request}],
        "executionPeriod": {"start": start, "end": end},
    }


# ── connections ──────────────────────────────────────────────


class TestConnections:
    def test_presets_are_public_sandbox_entities(self) -> None:
        assert set(svc.PRESETS) >= {"Large Model Entity", "Small Model Entity"}
        large = svc.PRESETS["Large Model Entity"]
        assert (
            large.base_url == SANDBOX
            and large.environment == "sandbox"
            and large.version == "v3"
        )
        assert svc.PRESETS["Legacy Large REACH ACO (v2)"].version == "v2"
        assert large.masked_secret().startswith("96ac") and "…" in large.masked_secret()
        assert svc.Connection("x", SANDBOX, "a", "short").masked_secret() == "…"
        assert svc.Connection("x", svc.PRODUCTION, "a", "b").environment == "production"

    def test_connection_from_env(self) -> None:
        assert svc.connection_from_env({}) is None
        assert svc.connection_from_env({"BCDA_CLIENT_ID": "a"}) is None
        c = svc.connection_from_env({"BCDA_CLIENT_ID": "a", "BCDA_CLIENT_SECRET": "b"})
        assert c is not None and c.base_url == SANDBOX and c.version == "v3"
        c = svc.connection_from_env(
            {
                "BCDA_CLIENT_ID": "a",
                "BCDA_CLIENT_SECRET": "b",
                "BCDA_BASE_URL": svc.PRODUCTION,
                "BCDA_API_VERSION": "v2",
            }
        )
        assert c.environment == "production" and c.version == "v2"

    def test_connection_from_real_env(self, monkeypatch) -> None:
        monkeypatch.setenv("BCDA_CLIENT_ID", "id")
        monkeypatch.setenv("BCDA_CLIENT_SECRET", "secret")
        assert svc.connection_from_env().client_id == "id"

    def test_make_client_and_store(self, tmp_path) -> None:
        c = svc.make_client(svc.PRESETS["Small Model Entity"], output_dir=tmp_path)
        assert c.version == "v3" and c.base_url == SANDBOX and c.output_dir == tmp_path
        c.close()
        s = svc.make_store(tmp_path)
        assert str(tmp_path) in s._root
        assert svc.store_path().name == "bcda"


# ── server side ──────────────────────────────────────────────


class TestServerSide:
    def test_preflight_ok(self) -> None:
        def handler(req: httpx.Request) -> httpx.Response:
            url = str(req.url)
            if url.endswith("/metadata"):
                return httpx.Response(
                    200,
                    json={
                        "fhirVersion": "4.0.1",
                        "software": {"version": "r302", "releaseDate": "2026-10-25"},
                        "rest": [
                            {
                                "resource": [
                                    {"type": "Patient"},
                                    {"type": "ExplanationOfBenefit"},
                                ]
                            }
                        ],
                    },
                )
            if url.endswith("/_version"):
                return httpx.Response(200, json={"version": "r302"})
            return httpx.Response(200, json={"database": "ok"})

        out = svc.preflight(_client(handler))
        assert (
            out["ok"] and out["fhir_version"] == "4.0.1" and out["software"] == "r302"
        )
        assert out["resources"] == ["Patient", "ExplanationOfBenefit"]
        assert out["server_version"] == {"version": "r302"} and out["health"] == {
            "database": "ok"
        }

    def test_preflight_degrades(self) -> None:
        def handler(req: httpx.Request) -> httpx.Response:
            url = str(req.url)
            if url.endswith("/metadata"):
                return httpx.Response(200, json={"fhirVersion": "4.0.1"})
            return httpx.Response(
                404,
                json={
                    "resourceType": "OperationOutcome",
                    "issue": [{"diagnostics": "no"}],
                },
            )

        out = svc.preflight(_client(handler))
        assert (
            out["ok"] and "error" in out["server_version"] and "error" in out["health"]
        )

        def down(req: httpx.Request) -> httpx.Response:
            raise httpx.ConnectError("boom")

        out = svc.preflight(_client(down))
        assert out["ok"] is False and "boom" in out["error"]

    def test_auth_check(self) -> None:
        def handler(req: httpx.Request) -> httpx.Response:
            if "/auth/token" in str(req.url):
                return _token()
            return httpx.Response(
                200,
                json={
                    "ingestion_dates": [
                        {"type": "last_attribution_update", "timestamp": "A"}
                    ],
                    "expiration_dates": [
                        {"type": "attribution_access_expiration", "timestamp": "E"}
                    ],
                },
            )

        out = svc.auth_check(_client(handler))
        assert out["token_chars"] == 40 and out["last_attribution_update"] == "A"
        assert (
            out["attribution_access_expiration"] == "E"
            and out["last_runout_update"] is None
        )

    def test_server_jobs(self) -> None:
        tasks = [
            _task(
                "1",
                "completed",
                f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Patient",
                "2026-10-25T10:00:00Z",
                "2026-10-25T10:30:00Z",
            ),
            _task(
                "2",
                "completed",
                f"GET {SANDBOX}/api/v3/Group/runout/$export",
                "2026-10-20T10:00:00Z",
                "2026-10-20T11:00:00Z",
            ),
            _task(
                "3",
                "in-progress",
                f"GET {SANDBOX}/api/v3/Patient/$export?_type=Coverage",
                "2026-10-25T11:00:00Z",
                "",
            ),
            _task(
                "4",
                "completed",
                f"GET {SANDBOX}/api/v2/Group/all/$export",
                "2026-10-25T12:00:00Z",
                "2026-10-25T12:30:00Z",
            ),
            _task(
                "5",
                "cancelled",
                f"GET {SANDBOX}/api/v3/Group/all/$export",
                "2026-10-25T09:00:00Z",
                "bad-instant",
            ),
        ]

        def handler(req: httpx.Request) -> httpx.Response:
            if "/auth/token" in str(req.url):
                return _token()
            return httpx.Response(
                200,
                json={
                    "resourceType": "Bundle",
                    "entry": [{"resource": t} for t in tasks],
                },
            )

        now = datetime(2026, 10, 25, 12, 0, tzinfo=timezone.utc)
        rows = svc.server_jobs(_client(handler), now=now)
        assert [r["job_id"] for r in rows] == [
            "3",
            "1",
            "5",
            "2",
        ]  # newest first, v2 dropped
        by = {r["job_id"]: r for r in rows}
        assert (
            by["1"]["reusable"] is True
            and by["1"]["types"] == "Patient"
            and by["1"]["endpoint"] == "Group/all"
        )
        assert (
            by["2"]["reusable"] is False
            and by["2"]["types"] == "(all)"
            and by["2"]["endpoint"] == "Group/runout"
        )
        assert by["3"]["endpoint"] == "Patient" and by["3"]["reusable"] is False
        assert by["5"]["reusable"] is False
        assert by["1"]["job_url"].endswith("/api/v3/jobs/1")
        assert len(svc.server_jobs(_client(handler), limit=2, now=now)) == 2
        assert svc._endpoint_from_request("GET https://x/api/v3/Other/$export") == "?"

    def test_build_type_filter(self) -> None:
        assert svc.build_type_filter() is None
        assert (
            svc.build_type_filter(system_types=["DDPS"])
            == "ExplanationOfBenefit?_tag=https://bluebutton.cms.gov/fhir/CodeSystem/System-Type|DDPS"
        )
        tf = svc.build_type_filter(
            system_types=["SharedSystem", "NationalClaimsHistory"],
            outcome="partial",
            service_date_from="2026-01-01",
        )
        assert (
            tf.startswith("ExplanationOfBenefit?_tag=")
            and "&outcome=partial&service-date=gt2026-01-01" in tf
        )
        with pytest.raises(ValueError, match="System-Type"):
            svc.build_type_filter(system_types=["Nope"])
        with pytest.raises(ValueError, match="outcome"):
            svc.build_type_filter(outcome="maybe")

    def _export_handler(self, calls: list[str]):  # noqa: ANN202
        def handler(req: httpx.Request) -> httpx.Response:
            url = str(req.url)
            calls.append(url)
            if "/auth/token" in url:
                return _token()
            if "$export" in url:
                n = sum(1 for c in calls if "$export" in c)
                return httpx.Response(
                    202, headers={"Content-Location": f"{SANDBOX}/api/v3/jobs/{n}"}
                )
            if "/jobs/" in url and req.method == "GET":
                jid = url.rsplit("/", 1)[1]
                return httpx.Response(
                    200,
                    json={
                        "output": [
                            {"type": "Patient", "url": f"{SANDBOX}/data/{jid}/a.ndjson"}
                        ]
                    },
                )
            if req.method == "DELETE":
                return httpx.Response(202)
            return httpx.Response(200, content=b'{"resourceType":"Patient"}\n')

        return handler

    def test_start_export_and_per_type(self, tmp_path) -> None:
        calls: list[str] = []
        client = _client(self._export_handler(calls), tmp_path=tmp_path)
        store = Store(path=str(tmp_path))
        # an empty store has no baseline to extend: the _since is refused …
        with pytest.raises(svc.SinceBlocked, match="no_baseline"):
            svc.start_export(
                store,
                client,
                endpoint="Patient",
                types=["Patient"],
                since="2026-01-01T00:00:00+00:00",
                poll_interval=0,
            )
        with pytest.raises(svc.SinceBlocked, match="no_baseline"):
            svc.start_export(
                store, client, endpoint="Patient", types=["Patient"], since="auto"
            )
        # … unless forced
        key = svc.start_export(
            store,
            client,
            endpoint="Patient",
            types=["Patient"],
            since="2026-01-01T00:00:00+00:00",
            type_filter="ExplanationOfBenefit?outcome=complete",
            poll_interval=0,
            force=True,
        )
        assert store.get_job(key)["status"] == "complete"
        assert any(
            "/api/v3/Patient/$export" in c and "_since=" in c and "_typeFilter=" in c
            for c in calls
        )
        with pytest.raises(ValueError, match="endpoint"):
            svc.start_export(store, client, endpoint="Nope")
        keys = svc.export_per_type(
            store,
            client,
            type_filter="ExplanationOfBenefit?outcome=partial",
            poll_interval=0,
        )
        assert set(keys) == set(svc.RESOURCE_TYPES)
        exports = [c for c in calls if "$export" in c]
        # the EOB job carries the _typeFilter, the Patient/Coverage jobs do not
        assert sum(1 for c in exports if "_typeFilter" in c) == 2  # first call + EOB
        assert all(store.get_job(k)["status"] == "complete" for k in keys.values())
        client.close()

    def test_adopt_and_cancel(self, tmp_path) -> None:
        calls: list[str] = []
        client = _client(self._export_handler(calls), tmp_path=tmp_path)
        store = Store(path=str(tmp_path))
        key = svc.adopt_job(
            store, client, f"{SANDBOX}/api/v3/jobs/77", types=["Patient"]
        )
        assert store.get_job(key)["job_url"].endswith("/jobs/77")
        svc.cancel_job(client, f"{SANDBOX}/api/v3/jobs/77")
        assert any(c.endswith("/jobs/77") for c in calls)
        assert svc.resume_job(store, client, key) == key
        assert store.manifest(key).get("output") is not None
        client.close()


# ── local side ───────────────────────────────────────────────


class TestLocalSide:
    def test_store_jobs_files_and_dirs(self, tmp_path) -> None:
        store = Store(path=str(tmp_path))
        assert svc.store_jobs(store) == []
        store._append_job(
            key="K1",
            status="complete",
            endpoint="Group/all",
            types="Patient",
            job_url="u",
        )
        store._append_file(
            job_key="K1",
            resource_type="Patient",
            url="u1",
            storage_path="p",
            file_size=10,
            status="complete",
        )
        store._append_job(key="K2", status="failed", error="boom")
        rows = svc.store_jobs(store)
        assert [r["key"] for r in rows] == ["K2", "K1"]
        k1 = next(r for r in rows if r["key"] == "K1")
        assert k1["files"] == 1 and k1["bytes"] == 10
        assert next(r for r in rows if r["key"] == "K2")["error"] == "boom"
        assert svc.store_files(store, "K1")[0]["storage_path"] == "p"
        assert svc.export_dirs(tmp_path) == []
        d = tmp_path / "exports" / "K1"
        d.mkdir(parents=True)
        (d / "Patient.ndjson").write_text("{}\n")
        (d / "Patient.1.ndjson").write_text("{}\n")
        (tmp_path / "exports" / "stray.txt").write_text("x")
        dirs = svc.export_dirs(tmp_path)
        assert (
            dirs[0]["key"] == "K1"
            and dirs[0]["types"] == ["Patient"]
            and dirs[0]["files"] == 2
            and dirs[0]["bytes"] == 6
        )

    def test_load_and_publish(self, monkeypatch) -> None:
        calls = {}

        def fake_load(**kw):  # noqa: ANN003, ANN202
            calls["load"] = kw
            return {"bcda.patient": 1}

        monkeypatch.setattr("aco.load.bcda.load_bcda", fake_load)
        monkeypatch.setattr(
            "conf.connect.publish_replica",
            lambda name: calls.setdefault("publish", name),
        )
        assert svc.load_and_publish(run_pipeline=False, skip_flatten=True) == {
            "bcda.patient": 1
        }
        assert (
            calls["load"]
            == {"skip_flatten": True, "run_pipeline": False, "reconcile": True}
            and calls["publish"] == "aco"
        )

    def test_replica_queries(self, tmp_path) -> None:
        db = tmp_path / "r.duckdb"
        con = duckdb.connect(str(db))
        con.execute("create schema bcda; create schema input_layer")
        con.execute(
            "create table bcda.patient as select 'A' bene_mbi_id, null::date bene_death_dt, 'TX' geo_usps_state_cd"
        )
        con.execute(
            "create table bcda.explanation_of_benefit as select 'DDPS' system_type, 'PDE' eob_type, 'FinalAction' final_action, 1.5 clm_pmt_amt"
        )
        con.execute(
            "create table bcda.coverage as select 'Part A' class_plan, 'active' status, 'A' subscriber_id"
        )
        con.execute(
            "create table input_layer.medical_claim as select 'institutional' claim_type, 'c' claim_id, 1.0 paid_amount, 2.0 charge_amount"
        )
        con.close()
        ro = duckdb.connect(str(db), read_only=True)
        counts = svc.table_counts(ro)
        assert counts == {
            "bcda.coverage": 1,
            "bcda.explanation_of_benefit": 1,
            "bcda.patient": 1,
            "input_layer.medical_claim": 1,
        }
        for sql in svc.SUMMARY_QUERIES.values():
            assert isinstance(svc.run_query(ro, sql), pl.DataFrame)
        assert len(svc.run_query(ro, "select * from bcda.patient", limit=5)) == 1
        ro.close()

    def test_replica_connect_uses_conf(self, monkeypatch) -> None:
        monkeypatch.setattr(
            "conf.connect.duckdb", lambda name, read_only: (name, read_only)
        )
        assert svc.replica_connect() == ("aco", True)

    def test_dictionary_frames_from_seeds_and_reference_data(self, tmp_path) -> None:
        frames = svc.dictionary_frames(None)
        assert set(frames) == {
            "bcda_v3_data_dictionary",
            "bcda_v3_cclf_crosswalk",
            "bcda_v3_cclf_absent",
        }
        assert len(frames["bcda_v3_data_dictionary"]) > 300
        con = duckdb.connect(str(tmp_path / "ref.duckdb"))
        con.execute("create schema reference_data")
        for n, df in frames.items():
            con.register("df", df.to_arrow())
            con.execute(f"create table reference_data.{n} as select * from df")
        again = svc.dictionary_frames(con)
        assert len(again["bcda_v3_cclf_absent"]) == len(frames["bcda_v3_cclf_absent"])
        con.close()
        # a connection without the tables falls back to the seeds
        con = duckdb.connect()
        assert len(svc.dictionary_frames(con)) == 3
        con.close()

    def test_dictionary_frames_missing(self, monkeypatch, tmp_path) -> None:
        monkeypatch.setattr("conf.ROOT", tmp_path)
        assert svc.dictionary_frames(None) == {}

    def test_vendor_status_and_check(self, monkeypatch) -> None:
        status = svc.vendor_status()
        assert status["available"] and status["problems"] == []
        names = {s["name"] for s in status["sources"]}
        assert names == {"bcda-app", "bcda-site"}
        assert any(f["path"].endswith("swagger.json") for f in status["files"])
        mod = svc._vendor_module()
        monkeypatch.setattr(mod, "check", lambda m: ["bcda-app: swagger.json changed"])
        monkeypatch.setattr(svc, "_vendor_module", lambda: mod)
        assert svc.vendor_check() == ["bcda-app: swagger.json changed"]

    def test_vendor_unavailable(self, monkeypatch, tmp_path) -> None:
        monkeypatch.setattr("conf.ROOT", tmp_path)
        assert svc.vendor_status() == {"available": False}
        assert svc.vendor_check() == ["vendor script not available in this checkout"]

    def test_types_from_request(self) -> None:
        assert svc._types_from_request("GET x?_type=Patient%2CCoverage") == [
            "Patient",
            "Coverage",
        ]
        json.dumps(svc.SUMMARY_QUERIES)  # plain strings


# ── incremental pulls ────────────────────────────────────────


class TestSince:
    def _baseline(self, store: Store, key: str, rtype: str, tt: str) -> None:
        d = Path(store.root) / "exports" / key
        d.mkdir(parents=True)
        (d / f"{rtype}.ndjson").write_text("")
        store._append_file(
            job_key=key,
            resource_type=rtype,
            url=f"https://s/{key}",
            storage_path=str(d / f"{rtype}.ndjson"),
            status="complete",
        )
        store._append_job(
            key=key,
            endpoint="Group/all",
            types=rtype,
            status="complete",
            transaction_time=tt,
        )

    def test_resolve_and_checks(self, tmp_path) -> None:
        store = Store(path=str(tmp_path))
        assert svc.resolve_since(store, None, types=["Patient"]) == {"Patient": None}
        assert svc.resolve_since(
            store, "2026-10-01T00:00:00+00:00", types=["Patient"]
        ) == {"Patient": "2026-10-01T00:00:00+00:00"}
        assert svc.resolve_since(store, "auto") == dict.fromkeys(svc.RESOURCE_TYPES)
        self._baseline(store, "B1", "Patient", "2026-10-01T00:00:00.000+00:00")
        self._baseline(store, "B2", "Coverage", "2026-10-02T00:00:00.000+00:00")
        assert svc.resolve_since(store, "auto", types=["Patient", "Coverage"]) == {
            "Patient": "2026-10-01T00:00:00.000+00:00",
            "Coverage": "2026-10-02T00:00:00.000+00:00",
        }
        checks = svc.since_checks(
            store, "2026-10-01T00:00:00.000+00:00", types=["Patient", "Coverage"]
        )
        assert {c["code"] for c in checks} >= {"watermark", "overlap"}
        assert all(c["level"] != "block" for c in checks)

    def test_since_checks_use_attribution(self, tmp_path) -> None:
        store = Store(path=str(tmp_path))
        self._baseline(store, "B1", "Patient", "2026-10-01T00:00:00.000+00:00")

        def handler(req: httpx.Request) -> httpx.Response:
            if "/auth/token" in str(req.url):
                return _token()
            return httpx.Response(
                200,
                json={
                    "ingestion_dates": [
                        {
                            "type": "last_attribution_update",
                            "timestamp": "2026-10-05T00:00:00Z",
                        }
                    ]
                },
            )

        client = _client(handler, tmp_path=tmp_path)
        checks = svc.since_checks(
            store, "2026-10-01T00:00:00.000+00:00", types=["Patient"], client=client
        )
        assert "attribution_newer" in {c["code"] for c in checks}

        def broken(req: httpx.Request) -> httpx.Response:
            if "/auth/token" in str(req.url):
                return _token()
            return httpx.Response(500, json={"resourceType": "OperationOutcome"})

        client2 = _client(broken, tmp_path=tmp_path)
        checks = svc.since_checks(
            store, "2026-10-01T00:00:00.000+00:00", types=["Patient"], client=client2
        )
        assert "attribution_newer" not in {c["code"] for c in checks}
        client.close()
        client2.close()

    def test_combined_job_takes_earliest_watermark(self, tmp_path) -> None:
        store = Store(path=str(tmp_path))
        self._baseline(store, "B1", "Patient", "2026-10-01T00:00:00.000+00:00")
        self._baseline(store, "B2", "Coverage", "2026-10-02T00:00:00.000+00:00")
        seen: dict = {}

        def fake_export(client, **kw):  # noqa: ANN001, ANN003, ANN202
            seen.update(kw)
            return "KEY"

        store.export = fake_export  # type: ignore[method-assign]
        assert (
            svc.start_export(
                store, object(), types=["Patient", "Coverage"], since="auto"
            )  # type: ignore[arg-type]
            == "KEY"
        )
        assert seen["since"] == "2026-10-01T00:00:00.000+00:00"

    def test_sync_status_and_apply(self, tmp_path, monkeypatch) -> None:
        store = Store(path=str(tmp_path))
        monkeypatch.setattr(svc, "make_store", lambda path=None: store)
        st = svc.sync_status()
        assert st["watermarks"] == [] and st["pending"] == [] and st["current"] == {}
        rep = svc.apply_sync()
        assert rep == {"ok": True, "applied": [], "skipped": []}