Some checks failed
CI / lint (push) Successful in 58s
CI / notebooks-smoke (push) Successful in 1m44s
CI / test (push) Failing after 3m3s
Infra CI / notebooks (push) Successful in 1m13s
Infra CI / docs (push) Successful in 1m46s
Infra CI / llm (push) Successful in 49s
Infra CI / mc (push) Failing after 36s
Infra CI / zotero (push) Successful in 27s
Infra CI / api (push) Successful in 1m9s
Infra CI / bcda (push) Successful in 1m37s
CI / lint (pull_request) Successful in 41s
CI / notebooks-smoke (pull_request) Successful in 1m49s
CI / test (pull_request) Failing after 2m50s
Infra CI / docs (pull_request) Successful in 17s
Infra CI / notebooks (pull_request) Successful in 59s
Infra CI / api (pull_request) Successful in 14s
Infra CI / bcda (pull_request) Successful in 14s
Infra CI / llm (pull_request) Successful in 20s
Infra CI / zotero (pull_request) Successful in 14s
Infra CI / mc (pull_request) Failing after 34s
"No TIN available" is a fact about the data, not a bug in ours. Institutional claims never carry a TIN: the real Large-entity v3 pull has 0 billing TINs on every claim type and 0 CCNs on every institutional claim (the dictionary's NamingSystem/CCN identifier is not emitted and half the institutional claims have no billing provider at all), and CCLF1 has an OSCAR/CCN and NPIs but no TIN either. The only TIN on claims is the carrier/DME CLM_RNDRG_PRVDR_TAX_NUM, the TIN the line was billed under, so the connector now gives professional and DME lines that value as billing_tin too (institutional stays NULL; the facility is its NPI and CCN). CMS publishes no provider TIN or EIN on data.cms.gov — checked every enrollment, ownership, change-of-ownership, Provider of Services and cost-report family in the catalog. What can be vendored is CCN ↔ NPI ↔ organization. cms.datacms resolves a dataset family to its newest version in the data.cms.gov catalog, downloads the CSV (or pages the API) and names columns like the generated cms.table models. `stack load cms` (aco.load.cms) loads the institutional universe — hospital, SNF, HHA, hospice, FQHC and RHC enrollments, the Provider of Services file and the three cost reports — as cms.* with provenance columns and ingest_log rows, then derives cms.institutional_provider, cms.ccn_crosswalk (PECOS ∪ POS-only CCNs, 65,161 rows today), cms.provider_tin_observed (NPI → TIN as seen on our own claims) and cms.ccn_tin (CCN → NPI → TIN, with cms.aco_participant_tin as the authoritative hook when a participant list is loaded). The command prints facility-NPI → CCN → TIN coverage; the Data page gets the same query. On the sandbox nothing resolves — its 8 facility NPIs are synthetic — which is the expected answer there. refs #1139
750 lines
29 KiB
Python
750 lines
29 KiB
Python
"""bcda.ui.service — the manager's operations, with mock transports."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import duckdb
|
|
import httpx
|
|
import polars as pl
|
|
import pytest
|
|
|
|
from bcda.client import SANDBOX, Client
|
|
from bcda.store import Store
|
|
from bcda.ui import service as svc
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _token() -> httpx.Response:
|
|
return httpx.Response(200, json={"access_token": "t" * 40, "expires_in": 1200})
|
|
|
|
|
|
def _client(handler, version: str = "v3", tmp_path: Path | None = None) -> Client:
|
|
c = Client(
|
|
"cid", "csec", version=version, output_dir=tmp_path or Path("/tmp/bcda-ui-test")
|
|
)
|
|
c._http = httpx.Client(transport=httpx.MockTransport(handler))
|
|
return c
|
|
|
|
|
|
def _task(job_id: str, status: str, request: str, start: str, end: str) -> dict:
|
|
return {
|
|
"resourceType": "Task",
|
|
"identifier": [{"value": job_id}],
|
|
"status": status,
|
|
"input": [{"valueString": request}],
|
|
"executionPeriod": {"start": start, "end": end},
|
|
}
|
|
|
|
|
|
# ── connections ──────────────────────────────────────────────
|
|
|
|
|
|
class TestConnections:
|
|
def test_presets_are_public_sandbox_entities(self) -> None:
|
|
assert set(svc.PRESETS) >= {"Large Model Entity", "Small Model Entity"}
|
|
large = svc.PRESETS["Large Model Entity"]
|
|
assert (
|
|
large.base_url == SANDBOX
|
|
and large.environment == "sandbox"
|
|
and large.version == "v3"
|
|
)
|
|
assert svc.PRESETS["Legacy Large REACH ACO (v2)"].version == "v2"
|
|
assert large.masked_secret().startswith("96ac") and "…" in large.masked_secret()
|
|
assert svc.Connection("x", SANDBOX, "a", "short").masked_secret() == "…"
|
|
assert svc.Connection("x", svc.PRODUCTION, "a", "b").environment == "production"
|
|
|
|
def test_connection_from_env(self) -> None:
|
|
assert svc.connection_from_env({}) is None
|
|
assert svc.connection_from_env({"BCDA_CLIENT_ID": "a"}) is None
|
|
c = svc.connection_from_env({"BCDA_CLIENT_ID": "a", "BCDA_CLIENT_SECRET": "b"})
|
|
assert c is not None and c.base_url == SANDBOX and c.version == "v3"
|
|
c = svc.connection_from_env(
|
|
{
|
|
"BCDA_CLIENT_ID": "a",
|
|
"BCDA_CLIENT_SECRET": "b",
|
|
"BCDA_BASE_URL": svc.PRODUCTION,
|
|
"BCDA_API_VERSION": "v2",
|
|
}
|
|
)
|
|
assert c.environment == "production" and c.version == "v2"
|
|
|
|
def test_connection_from_real_env(self, monkeypatch) -> None:
|
|
monkeypatch.setenv("BCDA_CLIENT_ID", "id")
|
|
monkeypatch.setenv("BCDA_CLIENT_SECRET", "secret")
|
|
assert svc.connection_from_env().client_id == "id"
|
|
|
|
def test_make_client_and_store(self, tmp_path) -> None:
|
|
c = svc.make_client(svc.PRESETS["Small Model Entity"], output_dir=tmp_path)
|
|
assert c.version == "v3" and c.base_url == SANDBOX and c.output_dir == tmp_path
|
|
c.close()
|
|
s = svc.make_store(tmp_path)
|
|
assert str(tmp_path) in s._root
|
|
assert svc.store_path().name == "bcda"
|
|
|
|
|
|
# ── server side ──────────────────────────────────────────────
|
|
|
|
|
|
class TestServerSide:
|
|
def test_preflight_ok(self) -> None:
|
|
def handler(req: httpx.Request) -> httpx.Response:
|
|
url = str(req.url)
|
|
if url.endswith("/metadata"):
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"fhirVersion": "4.0.1",
|
|
"software": {"version": "r302", "releaseDate": "2026-10-25"},
|
|
"rest": [
|
|
{
|
|
"resource": [
|
|
{"type": "Patient"},
|
|
{"type": "ExplanationOfBenefit"},
|
|
]
|
|
}
|
|
],
|
|
},
|
|
)
|
|
if url.endswith("/_version"):
|
|
return httpx.Response(200, json={"version": "r302"})
|
|
return httpx.Response(200, json={"database": "ok"})
|
|
|
|
out = svc.preflight(_client(handler))
|
|
assert (
|
|
out["ok"] and out["fhir_version"] == "4.0.1" and out["software"] == "r302"
|
|
)
|
|
assert out["resources"] == ["Patient", "ExplanationOfBenefit"]
|
|
assert out["server_version"] == {"version": "r302"} and out["health"] == {
|
|
"database": "ok"
|
|
}
|
|
|
|
def test_preflight_degrades(self) -> None:
|
|
def handler(req: httpx.Request) -> httpx.Response:
|
|
url = str(req.url)
|
|
if url.endswith("/metadata"):
|
|
return httpx.Response(200, json={"fhirVersion": "4.0.1"})
|
|
return httpx.Response(
|
|
404,
|
|
json={
|
|
"resourceType": "OperationOutcome",
|
|
"issue": [{"diagnostics": "no"}],
|
|
},
|
|
)
|
|
|
|
out = svc.preflight(_client(handler))
|
|
assert (
|
|
out["ok"] and "error" in out["server_version"] and "error" in out["health"]
|
|
)
|
|
|
|
def down(req: httpx.Request) -> httpx.Response:
|
|
raise httpx.ConnectError("boom")
|
|
|
|
out = svc.preflight(_client(down))
|
|
assert out["ok"] is False and "boom" in out["error"]
|
|
|
|
def test_auth_check(self) -> None:
|
|
def handler(req: httpx.Request) -> httpx.Response:
|
|
if "/auth/token" in str(req.url):
|
|
return _token()
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"ingestion_dates": [
|
|
{"type": "last_attribution_update", "timestamp": "A"}
|
|
],
|
|
"expiration_dates": [
|
|
{"type": "attribution_access_expiration", "timestamp": "E"}
|
|
],
|
|
},
|
|
)
|
|
|
|
out = svc.auth_check(_client(handler))
|
|
assert out["token_chars"] == 40 and out["last_attribution_update"] == "A"
|
|
assert (
|
|
out["attribution_access_expiration"] == "E"
|
|
and out["last_runout_update"] is None
|
|
)
|
|
|
|
def test_server_jobs(self) -> None:
|
|
tasks = [
|
|
_task(
|
|
"1",
|
|
"completed",
|
|
f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Patient",
|
|
"2026-10-25T10:00:00Z",
|
|
"2026-10-25T10:30:00Z",
|
|
),
|
|
_task(
|
|
"2",
|
|
"completed",
|
|
f"GET {SANDBOX}/api/v3/Group/runout/$export",
|
|
"2026-10-20T10:00:00Z",
|
|
"2026-10-20T11:00:00Z",
|
|
),
|
|
_task(
|
|
"3",
|
|
"in-progress",
|
|
f"GET {SANDBOX}/api/v3/Patient/$export?_type=Coverage",
|
|
"2026-10-25T11:00:00Z",
|
|
"",
|
|
),
|
|
_task(
|
|
"4",
|
|
"completed",
|
|
f"GET {SANDBOX}/api/v2/Group/all/$export",
|
|
"2026-10-25T12:00:00Z",
|
|
"2026-10-25T12:30:00Z",
|
|
),
|
|
_task(
|
|
"5",
|
|
"cancelled",
|
|
f"GET {SANDBOX}/api/v3/Group/all/$export",
|
|
"2026-10-25T09:00:00Z",
|
|
"bad-instant",
|
|
),
|
|
]
|
|
|
|
def handler(req: httpx.Request) -> httpx.Response:
|
|
if "/auth/token" in str(req.url):
|
|
return _token()
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"resourceType": "Bundle",
|
|
"entry": [{"resource": t} for t in tasks],
|
|
},
|
|
)
|
|
|
|
now = datetime(2026, 10, 25, 12, 0, tzinfo=timezone.utc)
|
|
rows = svc.server_jobs(_client(handler), now=now)
|
|
assert [r["job_id"] for r in rows] == [
|
|
"3",
|
|
"1",
|
|
"5",
|
|
"2",
|
|
] # newest first, v2 dropped
|
|
by = {r["job_id"]: r for r in rows}
|
|
assert (
|
|
by["1"]["reusable"] is True
|
|
and by["1"]["types"] == "Patient"
|
|
and by["1"]["endpoint"] == "Group/all"
|
|
)
|
|
assert (
|
|
by["2"]["reusable"] is False
|
|
and by["2"]["types"] == "(all)"
|
|
and by["2"]["endpoint"] == "Group/runout"
|
|
)
|
|
assert by["3"]["endpoint"] == "Patient" and by["3"]["reusable"] is False
|
|
assert by["5"]["reusable"] is False
|
|
assert by["1"]["job_url"].endswith("/api/v3/jobs/1")
|
|
assert len(svc.server_jobs(_client(handler), limit=2, now=now)) == 2
|
|
assert svc._endpoint_from_request("GET https://x/api/v3/Other/$export") == "?"
|
|
|
|
def test_build_type_filter(self) -> None:
|
|
assert svc.build_type_filter() is None
|
|
assert (
|
|
svc.build_type_filter(system_types=["DDPS"])
|
|
== "ExplanationOfBenefit?_tag=https://bluebutton.cms.gov/fhir/CodeSystem/System-Type|DDPS"
|
|
)
|
|
tf = svc.build_type_filter(
|
|
system_types=["SharedSystem", "NationalClaimsHistory"],
|
|
outcome="partial",
|
|
service_date_from="2026-01-01",
|
|
)
|
|
assert (
|
|
tf.startswith("ExplanationOfBenefit?_tag=")
|
|
and "&outcome=partial&service-date=gt2026-01-01" in tf
|
|
)
|
|
with pytest.raises(ValueError, match="System-Type"):
|
|
svc.build_type_filter(system_types=["Nope"])
|
|
with pytest.raises(ValueError, match="outcome"):
|
|
svc.build_type_filter(outcome="maybe")
|
|
|
|
def _export_handler(self, calls: list[str]): # noqa: ANN202
|
|
def handler(req: httpx.Request) -> httpx.Response:
|
|
url = str(req.url)
|
|
calls.append(url)
|
|
if "/auth/token" in url:
|
|
return _token()
|
|
if "$export" in url:
|
|
n = sum(1 for c in calls if "$export" in c)
|
|
return httpx.Response(
|
|
202, headers={"Content-Location": f"{SANDBOX}/api/v3/jobs/{n}"}
|
|
)
|
|
if "/jobs/" in url and req.method == "GET":
|
|
jid = url.rsplit("/", 1)[1]
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"output": [
|
|
{"type": "Patient", "url": f"{SANDBOX}/data/{jid}/a.ndjson"}
|
|
]
|
|
},
|
|
)
|
|
if req.method == "DELETE":
|
|
return httpx.Response(202)
|
|
return httpx.Response(200, content=b'{"resourceType":"Patient"}\n')
|
|
|
|
return handler
|
|
|
|
def test_start_export_and_per_type(self, tmp_path) -> None:
|
|
calls: list[str] = []
|
|
client = _client(self._export_handler(calls), tmp_path=tmp_path)
|
|
store = Store(path=str(tmp_path))
|
|
# an empty store has no baseline to extend: the _since is refused …
|
|
with pytest.raises(svc.SinceBlocked, match="no_baseline"):
|
|
svc.start_export(
|
|
store,
|
|
client,
|
|
endpoint="Patient",
|
|
types=["Patient"],
|
|
since="2026-01-01T00:00:00+00:00",
|
|
poll_interval=0,
|
|
)
|
|
with pytest.raises(svc.SinceBlocked, match="no_baseline"):
|
|
svc.start_export(
|
|
store, client, endpoint="Patient", types=["Patient"], since="auto"
|
|
)
|
|
# … unless forced
|
|
key = svc.start_export(
|
|
store,
|
|
client,
|
|
endpoint="Patient",
|
|
types=["Patient"],
|
|
since="2026-01-01T00:00:00+00:00",
|
|
type_filter="ExplanationOfBenefit?outcome=complete",
|
|
poll_interval=0,
|
|
force=True,
|
|
)
|
|
assert store.get_job(key)["status"] == "complete"
|
|
assert any(
|
|
"/api/v3/Patient/$export" in c and "_since=" in c and "_typeFilter=" in c
|
|
for c in calls
|
|
)
|
|
with pytest.raises(ValueError, match="endpoint"):
|
|
svc.start_export(store, client, endpoint="Nope")
|
|
keys = svc.export_per_type(
|
|
store,
|
|
client,
|
|
type_filter="ExplanationOfBenefit?outcome=partial",
|
|
poll_interval=0,
|
|
)
|
|
assert set(keys) == set(svc.RESOURCE_TYPES)
|
|
exports = [c for c in calls if "$export" in c]
|
|
# the EOB job carries the _typeFilter, the Patient/Coverage jobs do not
|
|
assert sum(1 for c in exports if "_typeFilter" in c) == 2 # first call + EOB
|
|
assert all(store.get_job(k)["status"] == "complete" for k in keys.values())
|
|
client.close()
|
|
|
|
def test_adopt_and_cancel(self, tmp_path) -> None:
|
|
calls: list[str] = []
|
|
client = _client(self._export_handler(calls), tmp_path=tmp_path)
|
|
store = Store(path=str(tmp_path))
|
|
key = svc.adopt_job(
|
|
store, client, f"{SANDBOX}/api/v3/jobs/77", types=["Patient"]
|
|
)
|
|
assert store.get_job(key)["job_url"].endswith("/jobs/77")
|
|
svc.cancel_job(client, f"{SANDBOX}/api/v3/jobs/77")
|
|
assert any(c.endswith("/jobs/77") for c in calls)
|
|
assert svc.resume_job(store, client, key) == key
|
|
assert store.manifest(key).get("output") is not None
|
|
client.close()
|
|
|
|
|
|
# ── local side ───────────────────────────────────────────────
|
|
|
|
|
|
class TestLocalSide:
|
|
def test_store_jobs_files_and_dirs(self, tmp_path) -> None:
|
|
store = Store(path=str(tmp_path))
|
|
assert svc.store_jobs(store) == []
|
|
store._append_job(
|
|
key="K1",
|
|
status="complete",
|
|
endpoint="Group/all",
|
|
types="Patient",
|
|
job_url="u",
|
|
)
|
|
store._append_file(
|
|
job_key="K1",
|
|
resource_type="Patient",
|
|
url="u1",
|
|
storage_path="p",
|
|
file_size=10,
|
|
status="complete",
|
|
)
|
|
store._append_job(key="K2", status="failed", error="boom")
|
|
rows = svc.store_jobs(store)
|
|
assert [r["key"] for r in rows] == ["K2", "K1"]
|
|
k1 = next(r for r in rows if r["key"] == "K1")
|
|
assert k1["files"] == 1 and k1["bytes"] == 10
|
|
assert next(r for r in rows if r["key"] == "K2")["error"] == "boom"
|
|
assert svc.store_files(store, "K1")[0]["storage_path"] == "p"
|
|
assert svc.export_dirs(tmp_path) == []
|
|
d = tmp_path / "exports" / "K1"
|
|
d.mkdir(parents=True)
|
|
(d / "Patient.ndjson").write_text("{}\n")
|
|
(d / "Patient.1.ndjson").write_text("{}\n")
|
|
(tmp_path / "exports" / "stray.txt").write_text("x")
|
|
dirs = svc.export_dirs(tmp_path)
|
|
assert (
|
|
dirs[0]["key"] == "K1"
|
|
and dirs[0]["types"] == ["Patient"]
|
|
and dirs[0]["files"] == 2
|
|
and dirs[0]["bytes"] == 6
|
|
)
|
|
|
|
def test_load_and_publish(self, monkeypatch) -> None:
|
|
calls = {}
|
|
|
|
def fake_load(**kw): # noqa: ANN003, ANN202
|
|
calls["load"] = kw
|
|
return {"bcda.patient": 1}
|
|
|
|
monkeypatch.setattr("aco.load.bcda.load_bcda", fake_load)
|
|
monkeypatch.setattr(
|
|
"conf.connect.publish_replica",
|
|
lambda name: calls.setdefault("publish", name),
|
|
)
|
|
assert svc.load_and_publish(run_pipeline=False, skip_flatten=True) == {
|
|
"bcda.patient": 1
|
|
}
|
|
assert (
|
|
calls["load"]
|
|
== {"skip_flatten": True, "run_pipeline": False, "reconcile": True}
|
|
and calls["publish"] == "aco"
|
|
)
|
|
|
|
def test_replica_queries(self, tmp_path) -> None:
|
|
db = tmp_path / "r.duckdb"
|
|
con = duckdb.connect(str(db))
|
|
con.execute("create schema bcda; create schema input_layer")
|
|
con.execute(
|
|
"create table bcda.patient as select 'A' bene_mbi_id, null::date bene_death_dt, 'TX' geo_usps_state_cd"
|
|
)
|
|
con.execute(
|
|
"create table bcda.explanation_of_benefit as select 'DDPS' system_type, 'PDE' eob_type, 'FinalAction' final_action, 1.5 clm_pmt_amt, NULL::varchar fac_prvdr_npi_num"
|
|
)
|
|
con.execute(
|
|
"create table bcda.coverage as select 'Part A' class_plan, 'active' status, 'A' subscriber_id"
|
|
)
|
|
con.execute(
|
|
"create table input_layer.medical_claim as select 'institutional' claim_type, 'c' claim_id, 1.0 paid_amount, 2.0 charge_amount"
|
|
)
|
|
con.execute("create schema cms")
|
|
con.execute(
|
|
"create table cms.ccn_tin as select NULL::varchar npi, NULL::varchar ccn, NULL::varchar tin where false"
|
|
)
|
|
con.close()
|
|
ro = duckdb.connect(str(db), read_only=True)
|
|
counts = svc.table_counts(ro)
|
|
assert counts == {
|
|
"bcda.coverage": 1,
|
|
"bcda.explanation_of_benefit": 1,
|
|
"bcda.patient": 1,
|
|
"input_layer.medical_claim": 1,
|
|
}
|
|
for sql in svc.SUMMARY_QUERIES.values():
|
|
assert isinstance(svc.run_query(ro, sql), pl.DataFrame)
|
|
assert len(svc.run_query(ro, "select * from bcda.patient", limit=5)) == 1
|
|
ro.close()
|
|
|
|
def test_replica_connect_uses_conf(self, monkeypatch) -> None:
|
|
monkeypatch.setattr(
|
|
"conf.connect.duckdb", lambda name, read_only: (name, read_only)
|
|
)
|
|
assert svc.replica_connect() == ("aco", True)
|
|
|
|
def test_dictionary_frames_from_seeds_and_reference_data(self, tmp_path) -> None:
|
|
frames = svc.dictionary_frames(None)
|
|
assert set(frames) == {
|
|
"bcda_v3_data_dictionary",
|
|
"bcda_v3_cclf_crosswalk",
|
|
"bcda_v3_cclf_absent",
|
|
}
|
|
assert len(frames["bcda_v3_data_dictionary"]) > 300
|
|
con = duckdb.connect(str(tmp_path / "ref.duckdb"))
|
|
con.execute("create schema reference_data")
|
|
for n, df in frames.items():
|
|
con.register("df", df.to_arrow())
|
|
con.execute(f"create table reference_data.{n} as select * from df")
|
|
again = svc.dictionary_frames(con)
|
|
assert len(again["bcda_v3_cclf_absent"]) == len(frames["bcda_v3_cclf_absent"])
|
|
con.close()
|
|
# a connection without the tables falls back to the seeds
|
|
con = duckdb.connect()
|
|
assert len(svc.dictionary_frames(con)) == 3
|
|
con.close()
|
|
|
|
def test_dictionary_frames_missing(self, monkeypatch, tmp_path) -> None:
|
|
monkeypatch.setattr("conf.ROOT", tmp_path)
|
|
assert svc.dictionary_frames(None) == {}
|
|
|
|
def test_vendor_status_and_check(self, monkeypatch) -> None:
|
|
status = svc.vendor_status()
|
|
assert status["available"] and status["problems"] == []
|
|
names = {s["name"] for s in status["sources"]}
|
|
assert names == {"bcda-app", "bcda-site", "bcda-static-site"}
|
|
assert any(f["path"].endswith("swagger.json") for f in status["files"])
|
|
mod = svc._vendor_module()
|
|
monkeypatch.setattr(mod, "check", lambda m: ["bcda-app: swagger.json changed"])
|
|
monkeypatch.setattr(svc, "_vendor_module", lambda: mod)
|
|
assert svc.vendor_check() == ["bcda-app: swagger.json changed"]
|
|
|
|
def test_vendor_unavailable(self, monkeypatch, tmp_path) -> None:
|
|
monkeypatch.setattr("conf.ROOT", tmp_path)
|
|
assert svc.vendor_status() == {"available": False}
|
|
assert svc.vendor_check() == ["vendor script not available in this checkout"]
|
|
|
|
def test_types_from_request(self) -> None:
|
|
assert svc._types_from_request("GET x?_type=Patient%2CCoverage") == [
|
|
"Patient",
|
|
"Coverage",
|
|
]
|
|
json.dumps(svc.SUMMARY_QUERIES) # plain strings
|
|
|
|
|
|
# ── incremental pulls ────────────────────────────────────────
|
|
|
|
|
|
class TestSince:
|
|
def _baseline(self, store: Store, key: str, rtype: str, tt: str) -> None:
|
|
d = Path(store.root) / "exports" / key
|
|
d.mkdir(parents=True)
|
|
(d / f"{rtype}.ndjson").write_text("")
|
|
store._append_file(
|
|
job_key=key,
|
|
resource_type=rtype,
|
|
url=f"https://s/{key}",
|
|
storage_path=str(d / f"{rtype}.ndjson"),
|
|
status="complete",
|
|
)
|
|
store._append_job(
|
|
key=key,
|
|
endpoint="Group/all",
|
|
types=rtype,
|
|
status="complete",
|
|
transaction_time=tt,
|
|
)
|
|
|
|
def test_resolve_and_checks(self, tmp_path) -> None:
|
|
store = Store(path=str(tmp_path))
|
|
assert svc.resolve_since(store, None, types=["Patient"]) == {"Patient": None}
|
|
assert svc.resolve_since(
|
|
store, "2026-10-01T00:00:00+00:00", types=["Patient"]
|
|
) == {"Patient": "2026-10-01T00:00:00+00:00"}
|
|
assert svc.resolve_since(store, "auto") == dict.fromkeys(svc.RESOURCE_TYPES)
|
|
self._baseline(store, "B1", "Patient", "2026-10-01T00:00:00.000+00:00")
|
|
self._baseline(store, "B2", "Coverage", "2026-10-02T00:00:00.000+00:00")
|
|
assert svc.resolve_since(store, "auto", types=["Patient", "Coverage"]) == {
|
|
"Patient": "2026-10-01T00:00:00.000+00:00",
|
|
"Coverage": "2026-10-02T00:00:00.000+00:00",
|
|
}
|
|
checks = svc.since_checks(
|
|
store, "2026-10-01T00:00:00.000+00:00", types=["Patient", "Coverage"]
|
|
)
|
|
assert {c["code"] for c in checks} >= {"watermark", "overlap"}
|
|
assert all(c["level"] != "block" for c in checks)
|
|
|
|
def test_since_checks_use_attribution(self, tmp_path) -> None:
|
|
store = Store(path=str(tmp_path))
|
|
self._baseline(store, "B1", "Patient", "2026-10-01T00:00:00.000+00:00")
|
|
|
|
def handler(req: httpx.Request) -> httpx.Response:
|
|
if "/auth/token" in str(req.url):
|
|
return _token()
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"ingestion_dates": [
|
|
{
|
|
"type": "last_attribution_update",
|
|
"timestamp": "2026-10-05T00:00:00Z",
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
client = _client(handler, tmp_path=tmp_path)
|
|
checks = svc.since_checks(
|
|
store, "2026-10-01T00:00:00.000+00:00", types=["Patient"], client=client
|
|
)
|
|
assert "attribution_newer" in {c["code"] for c in checks}
|
|
|
|
def broken(req: httpx.Request) -> httpx.Response:
|
|
if "/auth/token" in str(req.url):
|
|
return _token()
|
|
return httpx.Response(500, json={"resourceType": "OperationOutcome"})
|
|
|
|
client2 = _client(broken, tmp_path=tmp_path)
|
|
checks = svc.since_checks(
|
|
store, "2026-10-01T00:00:00.000+00:00", types=["Patient"], client=client2
|
|
)
|
|
assert "attribution_newer" not in {c["code"] for c in checks}
|
|
client.close()
|
|
client2.close()
|
|
|
|
def test_combined_job_takes_earliest_watermark(self, tmp_path) -> None:
|
|
store = Store(path=str(tmp_path))
|
|
self._baseline(store, "B1", "Patient", "2026-10-01T00:00:00.000+00:00")
|
|
self._baseline(store, "B2", "Coverage", "2026-10-02T00:00:00.000+00:00")
|
|
seen: dict = {}
|
|
|
|
def fake_export(client, **kw): # noqa: ANN001, ANN003, ANN202
|
|
seen.update(kw)
|
|
return "KEY"
|
|
|
|
store.export = fake_export # type: ignore[method-assign]
|
|
assert (
|
|
svc.start_export(
|
|
store, object(), types=["Patient", "Coverage"], since="auto"
|
|
) # type: ignore[arg-type]
|
|
== "KEY"
|
|
)
|
|
assert seen["since"] == "2026-10-01T00:00:00.000+00:00"
|
|
|
|
def test_sync_status_and_apply(self, tmp_path, monkeypatch) -> None:
|
|
store = Store(path=str(tmp_path))
|
|
monkeypatch.setattr(svc, "make_store", lambda path=None: store)
|
|
st = svc.sync_status()
|
|
assert st["watermarks"] == [] and st["pending"] == [] and st["current"] == {}
|
|
rep = svc.apply_sync()
|
|
assert rep == {"ok": True, "applied": [], "skipped": []}
|
|
|
|
|
|
# ── _typeFilter grammar + server error texts ─────────────────
|
|
|
|
|
|
class TestTypeFilterGrammar:
|
|
def test_outcome_list_and_service_date_bounds(self) -> None:
|
|
assert svc.build_type_filter(outcome=["partial", "complete"]) == (
|
|
"ExplanationOfBenefit?outcome=partial,complete"
|
|
)
|
|
assert svc.build_type_filter(outcome="complete,partial,complete") == (
|
|
"ExplanationOfBenefit?outcome=complete,partial"
|
|
)
|
|
assert svc.build_type_filter(service_date_from="2026-01") == (
|
|
"ExplanationOfBenefit?service-date=gt2026-01"
|
|
)
|
|
assert svc.build_type_filter(
|
|
service_date_from="ge2026", service_date_to="lt2027"
|
|
) == ("ExplanationOfBenefit?service-date=ge2026&service-date=lt2027")
|
|
assert svc.build_type_filter(service_date_to="2026-06-30") == (
|
|
"ExplanationOfBenefit?service-date=le2026-06-30"
|
|
)
|
|
assert (
|
|
svc.build_type_filter(
|
|
system_types=["NationalClaimsHistory", "DDPS"],
|
|
outcome="complete",
|
|
service_date_from="2026-01-01",
|
|
).count("&")
|
|
== 2
|
|
)
|
|
|
|
def test_rejections_mirror_the_server(self) -> None:
|
|
with pytest.raises(ValueError, match="outcome"):
|
|
svc.build_type_filter(outcome="done")
|
|
with pytest.raises(ValueError, match="comma"):
|
|
svc.build_type_filter(service_date_from="2026-01-01,2026-02-01")
|
|
with pytest.raises(ValueError, match="YYYY"):
|
|
svc.build_type_filter(service_date_from="01/01/2026")
|
|
with pytest.raises(ValueError, match="prefix"):
|
|
svc.build_type_filter(service_date_to="gt2026-01-01")
|
|
with pytest.raises(ValueError, match="eq service-date"):
|
|
svc.build_type_filter(
|
|
service_date_from="eq2026-01-01", service_date_to="2026-12-31"
|
|
)
|
|
|
|
def test_explain_server_error(self) -> None:
|
|
from bcda.client import BcdaHttpError
|
|
|
|
def err(status: int, diag: str) -> BcdaHttpError:
|
|
resp = httpx.Response(
|
|
status,
|
|
json={
|
|
"resourceType": "OperationOutcome",
|
|
"issue": [{"diagnostics": diag}],
|
|
},
|
|
request=httpx.Request("GET", "https://s/x"),
|
|
)
|
|
return BcdaHttpError(resp, diag)
|
|
|
|
assert "later _since" in svc.explain_server_error(
|
|
err(
|
|
404,
|
|
"failed to start job; attribution file not found for given _since parameter.",
|
|
)
|
|
)
|
|
assert "January" in svc.explain_server_error(
|
|
err(
|
|
404, "No up-to-date attribution information is available for ACOID 'A1'"
|
|
)
|
|
)
|
|
assert "_typeFilter was rejected" in svc.explain_server_error(
|
|
err(
|
|
400,
|
|
"invalid service-date parameter value: conflicting prefix conditions",
|
|
)
|
|
)
|
|
assert "Retry-After" in svc.explain_server_error(err(429, "rate limit"))
|
|
assert svc.explain_server_error(err(500, "boom")) == "boom"
|
|
|
|
def test_start_export_records_attribution_and_explains_404(self, tmp_path) -> None:
|
|
calls: list[str] = []
|
|
|
|
def handler(req: httpx.Request) -> httpx.Response:
|
|
url = str(req.url)
|
|
calls.append(url)
|
|
if "/auth/token" in url:
|
|
return _token()
|
|
if "/attribution_status" in url:
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"ingestion_dates": [
|
|
{
|
|
"type": "last_attribution_update",
|
|
"timestamp": "2026-10-01T00:00:00Z",
|
|
}
|
|
],
|
|
"expiration_dates": [],
|
|
},
|
|
)
|
|
if "$export" in url:
|
|
return httpx.Response(
|
|
404,
|
|
json={
|
|
"resourceType": "OperationOutcome",
|
|
"issue": [
|
|
{
|
|
"diagnostics": "No up-to-date attribution information is available"
|
|
}
|
|
],
|
|
},
|
|
)
|
|
return httpx.Response(200, json={})
|
|
|
|
client = _client(handler, tmp_path=tmp_path)
|
|
store = Store(path=str(tmp_path))
|
|
from bcda.client import BcdaError
|
|
|
|
with pytest.raises(BcdaError, match="January") as exc:
|
|
svc.start_export(store, client, types=["Patient"], poll_interval=0)
|
|
assert not isinstance(exc.value, svc.SinceBlocked)
|
|
job = store.list_jobs()[0]
|
|
assert (
|
|
job["status"] == "failed"
|
|
and job["attribution_update"] == "2026-10-01T00:00:00Z"
|
|
)
|
|
client.close()
|
|
|
|
def test_job_error_diagnostics(self, tmp_path) -> None:
|
|
store = Store(path=str(tmp_path))
|
|
assert svc.job_error_diagnostics(store, "NOPE") == []
|
|
rows = svc.store_jobs(store)
|
|
assert rows == []
|