feat(bcda): BCDA v3 — client, store adoption and cancel retries, dictionary-driven v3 flatteners, streaming multi-part load
Client defaults to /api/v3; _typeFilter on start_export/export with type_filter_tag() for the System-Type tag; jobs() sends the bulk-data headers BCDA requires (Prefer: respond-async, Accept: application/fhir+json — the old call 400'd on v2 and v3) and unwraps the Task Bundle; completed_jobs() lists reusable exports; every non-2xx raises BcdaHttpError carrying the OperationOutcome diagnostics (still an httpx.HTTPStatusError); a poll that hits 404 'Job has been cancelled' raises JobCancelledError; /_version and /_health. Store.adopt(client, job_url) records and downloads a job this store did not start (completed jobs stay downloadable 24 h and the public sandbox entity is shared); export(cancel_retries=N) restarts after server-side cancellation, recording each 'cancelled' transition. bcda.express.v3 flattens C4BB 2.2.0 resources into the existing bcda.* tables, table-driven from the v3 Data Dictionary: v3 moved every BlueButton code off resources/variables/<name> onto fhir/StructureDefinition|CodeSystem/<CODE> and consolidated amounts into adjudication/total/item.adjudication, so the v2 suffix match returned NULL for almost everything. Dispatch is per resource (is_v3); v2 payloads are unchanged. Identity follows the migration guide: bene_id = Patient.id (BENE_SK), current MBI = us-mbi identifier without period.end, clm_id = C4BB 'uc', clm_cntl_num pairs SharedSystem with NCH. Race crosswalked from US Core OMB codes; Coverage is one row per plan+period (Part A/B/C/D + Dual) with v3-native columns; EOB gains System-Type / Final-Action / EOB-TYPE, IDR dates, header totals and PPS/IME/DSH amounts; items gain NDC quantity, revenue-centre indicators, ANSI codes, Part D/DME amounts. flatten_export streams <Type>.N.ndjson parts to Parquet with model-derived Arrow schemas (a Large-entity EOB export is 192 files / 2.2 GB); load_bcda() gathers the newest export dir per resource type (a v3 pull is three single-type jobs). cclf.stg_beneficiary_demographics prefers the Part A row and joins the Dual row; eligibility reads coalesce(dual_01, dual_stus_cd). bcda package at 100% line coverage; tests run on CMS's official v3 samples. refs #1139
This commit is contained in:
kert
@@ -10,8 +10,10 @@ Usage: stack load bcda [OPTIONS]
|
||||
|
||||
Flatten BCDA FHIR exports and load into DuckDB.
|
||||
|
||||
Reads Patient/EOB/Coverage ndjson files, flattens them to Parquet,
|
||||
and loads into the ``bcda`` schema.
|
||||
Reads Patient/EOB/Coverage ndjson files (v2 or v3 payloads, any
|
||||
number of parts), flattens them to Parquet, and loads into the
|
||||
``bcda`` schema. Without --ndjson-dir, uses the newest export per
|
||||
resource type under the BCDA store (a v3 pull is three jobs).
|
||||
|
||||
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
||||
│ --ndjson-dir PATH Directory containing FHIR ndjson files. │
|
||||
|
||||
@@ -28,8 +28,11 @@ def load_bcda(
|
||||
Parameters
|
||||
----------
|
||||
ndjson_dir : str or Path, optional
|
||||
Directory containing raw ndjson files. Defaults to the latest
|
||||
export directory under ``conf.path("storage.bcda")``.
|
||||
Directory containing raw ndjson files. Defaults to the newest
|
||||
``exports/<key>/`` directory per resource type under
|
||||
``conf.path("storage.bcda")`` — a v3 pull is usually three
|
||||
single-type jobs, so Patient, Coverage and EOB may each come
|
||||
from a different export directory.
|
||||
database : str, optional
|
||||
DuckDB database path. Defaults to ``conf.path("db.aco")``.
|
||||
skip_flatten : bool
|
||||
@@ -47,7 +50,7 @@ def load_bcda(
|
||||
db_path = database or str(path("db.aco"))
|
||||
store_path = path("storage.bcda")
|
||||
|
||||
if ndjson_dir is None: # pragma: no cover — tests always provide explicit dir
|
||||
if ndjson_dir is None:
|
||||
ndjson_dir = _find_latest_export(store_path)
|
||||
|
||||
if not skip_flatten:
|
||||
@@ -89,20 +92,31 @@ def load_bcda(
|
||||
return stats
|
||||
|
||||
|
||||
def _find_latest_export(store_path: Path) -> Path:
|
||||
"""Find the most recent export directory under the BCDA store."""
|
||||
def _find_latest_export(store_path: Path) -> list[Path]:
|
||||
"""Export directories holding the newest file of each resource type.
|
||||
|
||||
Oldest first, so :func:`bcda.express.flatten.resolve_ndjson`
|
||||
(later directories win) picks the newest ``Patient.ndjson``,
|
||||
``Coverage.ndjson`` and ``ExplanationOfBenefit.ndjson`` even when
|
||||
they came from three separate single-type jobs.
|
||||
"""
|
||||
exports_dir = store_path / "exports"
|
||||
if not exports_dir.exists():
|
||||
raise FileNotFoundError(
|
||||
f"No exports directory at {exports_dir}. "
|
||||
"Run a BCDA export first or specify --ndjson-dir."
|
||||
)
|
||||
# Export directories are named by job ID; pick the most recent
|
||||
# Export directories are named by job key; order by mtime.
|
||||
dirs = sorted(
|
||||
[d for d in exports_dir.iterdir() if d.is_dir()],
|
||||
key=lambda d: d.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
if not dirs:
|
||||
raise FileNotFoundError(f"No export directories in {exports_dir}")
|
||||
return dirs[0]
|
||||
newest: dict[str, Path] = {}
|
||||
for d in dirs:
|
||||
for rtype in ("Patient", "ExplanationOfBenefit", "Coverage"):
|
||||
if (d / f"{rtype}.ndjson").exists():
|
||||
newest[rtype] = d
|
||||
chosen = sorted(set(newest.values()), key=lambda d: d.stat().st_mtime)
|
||||
return chosen or [dirs[-1]]
|
||||
|
||||
@@ -20,8 +20,11 @@ Submodules
|
||||
----------
|
||||
|
||||
``bcda.client``
|
||||
HTTP client for the BCDA v2 bulk FHIR export API.
|
||||
Handles auth, job management, polling, and ndjson download.
|
||||
HTTP client for the BCDA v3 bulk FHIR export API (``version="v2"``
|
||||
still works for the legacy REACH sandbox entities). Handles auth,
|
||||
``_typeFilter``, job management, polling, cancellation detection
|
||||
and ndjson download; non-2xx responses carry the server's
|
||||
OperationOutcome diagnostics.
|
||||
|
||||
``bcda.store``
|
||||
Persistent storage and state management. Wraps the client
|
||||
@@ -35,9 +38,15 @@ Submodules
|
||||
append-only ``.jsonl`` file.
|
||||
|
||||
``bcda.express``
|
||||
Narwhals expression functions for transforming BCDA FHIR R4
|
||||
resources into input_layer tables (medical_claim,
|
||||
pharmacy_claim, eligibility).
|
||||
``flatten`` turns the raw ndjson into the ``bcda.*`` tables,
|
||||
detecting the payload dialect per resource: the v2
|
||||
(``resources/variables``) flattener in ``flatten`` itself, the v3
|
||||
(C4BB 2.2.0, ``fhir/StructureDefinition``) flattener in
|
||||
``bcda.express.v3`` — table-driven from the BCDA v3 Data
|
||||
Dictionary (``dev/vendor/bcda-site``, ``dev/seeds/bcda_v3_*.csv``).
|
||||
``cclf`` holds the Narwhals expression functions that transform
|
||||
those tables into input_layer (medical_claim, pharmacy_claim,
|
||||
eligibility).
|
||||
|
||||
``bcda.pipe``
|
||||
Pipeline definitions wiring express functions into executable
|
||||
@@ -52,16 +61,18 @@ Submodules
|
||||
Adjudicated Claims — FHIR R4 Resources
|
||||
----------------------------------------
|
||||
|
||||
Delivered via ``/api/v2/{Resource}/$export`` bulk endpoints:
|
||||
Delivered via ``/api/{v2,v3}/{Resource}/$export`` bulk endpoints
|
||||
(v3 since 2026-07-01; v1/v2 are discontinued on 2027-07-30):
|
||||
|
||||
::
|
||||
|
||||
| Table | Fields | FHIR R4 Resource |
|
||||
|-------------------------|--------|------------------------|
|
||||
| patient | 26 | Patient |
|
||||
| explanation_of_benefit | 79 | ExplanationOfBenefit |
|
||||
| eob_item | 23 | ExplanationOfBenefit |
|
||||
| coverage | 10 | Coverage |
|
||||
| Table | FHIR R4 Resource | v3 additions |
|
||||
|-------------------------|------------------------|----------------------------------|
|
||||
| patient | Patient | contact language, source keys |
|
||||
| explanation_of_benefit | ExplanationOfBenefit | System-Type/Final-Action tags, |
|
||||
| | | claim control number, IDR totals |
|
||||
| eob_item | ExplanationOfBenefit | consolidated line adjudication |
|
||||
| coverage | Coverage | one row per plan + period, Dual |
|
||||
|
||||
``ExplanationOfBenefit`` is split into header-level fields
|
||||
(``explanation_of_benefit``) and repeating line items
|
||||
@@ -70,9 +81,12 @@ Delivered via ``/api/v2/{Resource}/$export`` bulk endpoints:
|
||||
Partially Adjudicated Claims
|
||||
-----------------------------
|
||||
|
||||
Available via BCDA v2 only. Updated daily (2-4 day lag).
|
||||
Sourced from FISS (institutional) and MCS (professional)
|
||||
processing systems, mapped to FHIR R4 Claim and ClaimResponse:
|
||||
In v3 these arrive as ``ExplanationOfBenefit`` tagged
|
||||
``System-Type|SharedSystem`` (request them with ``_typeFilter``, see
|
||||
``bcda.client.type_filter_tag``); the tables below are the v2
|
||||
``Claim``/``ClaimResponse`` shape, kept for v2 exports. Updated
|
||||
daily (2-4 day lag). Sourced from FISS (institutional) and MCS
|
||||
(professional) processing systems:
|
||||
|
||||
::
|
||||
|
||||
|
||||
@@ -1,15 +1,23 @@
|
||||
"""BCDA bulk FHIR export client.
|
||||
|
||||
Wraps the CMS Beneficiary Claims Data API v2 four-step flow:
|
||||
Wraps the CMS Beneficiary Claims Data API v3 four-step flow:
|
||||
|
||||
1. Authenticate (POST ``/auth/token`` with client credentials)
|
||||
2. Start export job (GET ``/api/v2/Group/all/$export``)
|
||||
3. Poll until complete (GET ``/api/v2/jobs/{job_id}``)
|
||||
2. Start export job (GET ``/api/v3/Group/all/$export``)
|
||||
3. Poll until complete (GET ``/api/v3/jobs/{job_id}``)
|
||||
4. Download ndjson files (GET ``/data/{job_id}/{filename}``)
|
||||
|
||||
Only the ``/api/<version>/`` segment differs between v2 and v3 —
|
||||
``/auth`` and ``/data`` are unchanged (migration guide) — so
|
||||
``version="v2"`` still works for the legacy REACH sandbox entities.
|
||||
v3 adds ``_typeFilter`` (FHIR subquery, e.g. the ``System-Type``
|
||||
``_tag`` that selects SharedSystem / NationalClaimsHistory / DDPS
|
||||
claims); without it BCDA returns NationalClaimsHistory + DDPS only.
|
||||
|
||||
Handles token refresh (20-min expiry), exponential backoff on
|
||||
transient errors, 429 rate-limit back-off, and gzip-compressed
|
||||
ndjson downloads.
|
||||
ndjson downloads. Every non-2xx response surfaces the server's
|
||||
``OperationOutcome`` diagnostics (:class:`BcdaHttpError`).
|
||||
|
||||
Usage::
|
||||
|
||||
@@ -37,6 +45,18 @@ log = logging.getLogger(__name__)
|
||||
|
||||
SANDBOX = "https://sandbox.bcda.cms.gov"
|
||||
PRODUCTION = "https://api.bcda.cms.gov"
|
||||
DEFAULT_VERSION = "v3"
|
||||
|
||||
# FHIR Bulk Data headers BCDA requires on $export, job status and
|
||||
# job history (both v2 and v3 reject /jobs without ``Prefer``).
|
||||
FHIR_JSON = "application/fhir+json"
|
||||
ASYNC_HEADERS = {"Accept": FHIR_JSON, "Prefer": "respond-async"}
|
||||
|
||||
# ``_typeFilter`` building blocks (migration guide, "Adjudication
|
||||
# filtering"). ``type_filter_tag(*codes)`` renders the subquery; the
|
||||
# client URL-encodes it.
|
||||
SYSTEM_TYPE = "https://bluebutton.cms.gov/fhir/CodeSystem/System-Type"
|
||||
SYSTEM_TYPES = ("SharedSystem", "NationalClaimsHistory", "DDPS")
|
||||
|
||||
# Defaults matching the Go reference implementation.
|
||||
MAX_RETRIES = 3
|
||||
@@ -52,6 +72,80 @@ class JobExpiredError(BcdaError):
|
||||
"""Job has been archived (server returned 410 Gone)."""
|
||||
|
||||
|
||||
class JobCancelledError(BcdaError):
|
||||
"""Job status returned ``404 Job has been cancelled``.
|
||||
|
||||
Seen on the public sandbox, whose credentials are shared: any
|
||||
caller can ``DELETE`` any job of the same model entity.
|
||||
"""
|
||||
|
||||
|
||||
class BcdaHttpError(BcdaError, httpx.HTTPStatusError):
|
||||
"""A non-2xx response, carrying the ``OperationOutcome`` text.
|
||||
|
||||
Still an :class:`httpx.HTTPStatusError` so existing ``except``
|
||||
clauses keep working; ``diagnostics`` holds the joined
|
||||
``issue[].diagnostics`` strings (empty when the body was not an
|
||||
OperationOutcome).
|
||||
"""
|
||||
|
||||
def __init__(self, response: httpx.Response, diagnostics: str) -> None:
|
||||
self.diagnostics = diagnostics
|
||||
self.status_code = response.status_code
|
||||
msg = f"{response.status_code} {response.reason_phrase} for {response.url}"
|
||||
if diagnostics:
|
||||
msg = f"{msg}: {diagnostics}"
|
||||
httpx.HTTPStatusError.__init__(
|
||||
self, msg, request=response.request, response=response
|
||||
)
|
||||
|
||||
|
||||
def operation_outcome(resp: httpx.Response) -> str:
|
||||
"""Join the ``diagnostics`` of an OperationOutcome body, else ``""``."""
|
||||
try:
|
||||
body = resp.json()
|
||||
except ValueError:
|
||||
return ""
|
||||
if not isinstance(body, dict) or body.get("resourceType") != "OperationOutcome":
|
||||
return ""
|
||||
parts = []
|
||||
for issue in body.get("issue", []):
|
||||
text = issue.get("diagnostics") or (issue.get("details") or {}).get("text")
|
||||
if text:
|
||||
parts.append(str(text))
|
||||
return "; ".join(parts)
|
||||
|
||||
|
||||
def raise_for_status(resp: httpx.Response) -> None:
|
||||
"""``resp.raise_for_status()`` that keeps the server's diagnostics."""
|
||||
if resp.is_success:
|
||||
return
|
||||
raise BcdaHttpError(resp, operation_outcome(resp))
|
||||
|
||||
|
||||
def _requested_types(request: str) -> list[str]:
|
||||
"""``_type`` values from a job's originating request line."""
|
||||
from urllib.parse import parse_qs, unquote, urlsplit
|
||||
|
||||
url = request.split(" ", 1)[-1]
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
raw = query.get("_type", [""])[0]
|
||||
return [t for t in unquote(raw).split(",") if t]
|
||||
|
||||
|
||||
def type_filter_tag(*codes: str) -> str:
|
||||
"""Render a ``_typeFilter`` subquery selecting EOBs by System-Type.
|
||||
|
||||
``type_filter_tag("SharedSystem")`` → the v2 Claim/ClaimResponse
|
||||
equivalent; ``type_filter_tag(*SYSTEM_TYPES)`` → everything.
|
||||
"""
|
||||
unknown = [c for c in codes if c not in SYSTEM_TYPES]
|
||||
if unknown or not codes:
|
||||
raise ValueError(f"System-Type codes must be among {SYSTEM_TYPES}: {codes}")
|
||||
tags = ",".join(f"{SYSTEM_TYPE}|{c}" for c in codes)
|
||||
return f"ExplanationOfBenefit?_tag={tags}"
|
||||
|
||||
|
||||
class Client:
|
||||
"""BCDA bulk FHIR export client.
|
||||
|
||||
@@ -67,7 +161,8 @@ class Client:
|
||||
Directory for downloaded ndjson files.
|
||||
Defaults to ``data/bcda``.
|
||||
version : str
|
||||
API version path segment. Defaults to ``v2``.
|
||||
API version path segment. Defaults to ``v3``; ``"v2"`` for
|
||||
the legacy REACH sandbox entities.
|
||||
max_retries : int
|
||||
Max retry attempts for transient errors.
|
||||
retry_interval : float
|
||||
@@ -83,7 +178,7 @@ class Client:
|
||||
*,
|
||||
base_url: str = SANDBOX,
|
||||
output_dir: str | Path | None = None,
|
||||
version: str = "v2",
|
||||
version: str = DEFAULT_VERSION,
|
||||
max_retries: int = MAX_RETRIES,
|
||||
retry_interval: float = RETRY_INITIAL_INTERVAL,
|
||||
timeout: float = 120.0,
|
||||
@@ -133,7 +228,7 @@ class Client:
|
||||
headers={"Accept": "application/json"},
|
||||
content=b"",
|
||||
)
|
||||
resp.raise_for_status()
|
||||
raise_for_status(resp)
|
||||
data = resp.json()
|
||||
self._token = data["access_token"]
|
||||
expires_in = int(data.get("expires_in", TOKEN_LIFETIME))
|
||||
@@ -269,7 +364,7 @@ class Client:
|
||||
time.sleep(backoff)
|
||||
backoff *= 2
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
raise_for_status(resp)
|
||||
|
||||
# 410 — job expired / archived.
|
||||
if resp.status_code == 410:
|
||||
@@ -278,8 +373,9 @@ class Client:
|
||||
"Jobs expire 24 hours after completion."
|
||||
)
|
||||
|
||||
# Any other client error — fail immediately.
|
||||
resp.raise_for_status()
|
||||
# Any other client error — fail immediately, with the
|
||||
# server's OperationOutcome diagnostics in the message.
|
||||
raise_for_status(resp)
|
||||
return resp
|
||||
|
||||
# Should not reach here, but satisfy the type checker.
|
||||
@@ -295,6 +391,7 @@ class Client:
|
||||
endpoint: str = "Group/all",
|
||||
types: list[str] | None = None,
|
||||
since: str | None = None,
|
||||
type_filter: str | None = None,
|
||||
poll_interval: float = 10.0,
|
||||
) -> list[Path]:
|
||||
"""Run a full bulk export: start, poll, download.
|
||||
@@ -311,6 +408,9 @@ class Client:
|
||||
since : str | None
|
||||
FHIR instant — only return resources updated after
|
||||
this timestamp. Format: ``YYYY-MM-DDThh:mm:ss±zz:zz``.
|
||||
type_filter : str | None
|
||||
v3 ``_typeFilter`` subquery (not URL-encoded), e.g.
|
||||
:func:`type_filter_tag`.
|
||||
poll_interval : float
|
||||
Seconds between status polls. Default 10.
|
||||
|
||||
@@ -323,6 +423,7 @@ class Client:
|
||||
endpoint=endpoint,
|
||||
types=types,
|
||||
since=since,
|
||||
type_filter=type_filter,
|
||||
)
|
||||
result = self.poll(job_url, interval=poll_interval)
|
||||
return self.download(result)
|
||||
@@ -333,6 +434,7 @@ class Client:
|
||||
endpoint: str = "Group/all",
|
||||
types: list[str] | None = None,
|
||||
since: str | None = None,
|
||||
type_filter: str | None = None,
|
||||
) -> str:
|
||||
"""Start a bulk export job.
|
||||
|
||||
@@ -347,14 +449,13 @@ class Client:
|
||||
params["_type"] = ",".join(types)
|
||||
if since:
|
||||
params["_since"] = since
|
||||
if type_filter:
|
||||
params["_typeFilter"] = type_filter
|
||||
|
||||
resp = self._request(
|
||||
"GET",
|
||||
url,
|
||||
headers={
|
||||
"Accept": "application/fhir+json",
|
||||
"Prefer": "respond-async",
|
||||
},
|
||||
headers=dict(ASYNC_HEADERS),
|
||||
params=params,
|
||||
)
|
||||
|
||||
@@ -392,14 +493,22 @@ class Client:
|
||||
------
|
||||
JobExpiredError
|
||||
If the job has been archived (410 Gone).
|
||||
JobCancelledError
|
||||
If the server reports the job cancelled (404 with
|
||||
``Job has been cancelled`` diagnostics).
|
||||
BcdaError
|
||||
If timeout is exceeded.
|
||||
"""
|
||||
deadline = (time.monotonic() + timeout) if timeout > 0 else 0
|
||||
headers = {"Accept": "application/fhir+json"}
|
||||
headers = dict(ASYNC_HEADERS)
|
||||
|
||||
while True:
|
||||
resp = self._request("GET", job_url, headers=headers)
|
||||
try:
|
||||
resp = self._request("GET", job_url, headers=headers)
|
||||
except BcdaHttpError as exc:
|
||||
if exc.status_code == 404 and "cancel" in exc.diagnostics.lower():
|
||||
raise JobCancelledError(f"{job_url}: {exc.diagnostics}") from exc
|
||||
raise
|
||||
|
||||
if resp.status_code == 200:
|
||||
result = resp.json()
|
||||
@@ -423,7 +532,7 @@ class Client:
|
||||
continue
|
||||
|
||||
# Shouldn't get here — _request handles other status codes.
|
||||
resp.raise_for_status()
|
||||
raise_for_status(resp)
|
||||
|
||||
def download(
|
||||
self,
|
||||
@@ -497,7 +606,9 @@ class Client:
|
||||
"Accept-Encoding": "gzip",
|
||||
}
|
||||
with self._http.stream("GET", url, headers=headers) as resp:
|
||||
resp.raise_for_status()
|
||||
if not resp.is_success:
|
||||
resp.read()
|
||||
raise_for_status(resp)
|
||||
|
||||
content_encoding = resp.headers.get("Content-Encoding", "")
|
||||
raw_bytes = b"".join(resp.iter_bytes())
|
||||
@@ -539,14 +650,67 @@ class Client:
|
||||
Returns
|
||||
-------
|
||||
list[dict]
|
||||
Job history entries.
|
||||
Job history entries. v3 (and v2) answer with a FHIR
|
||||
``Bundle`` of ``Task`` resources — those resources are
|
||||
returned; a bare list (v1) is passed through. Each Task
|
||||
carries ``identifier[0].value`` (job id), ``status``,
|
||||
``input[0].valueString`` (the originating request) and
|
||||
``executionPeriod``.
|
||||
"""
|
||||
url = f"{self.base_url}/api/{self.version}/jobs"
|
||||
params: dict[str, str] = {}
|
||||
if status:
|
||||
params["_status"] = status
|
||||
resp = self._request("GET", url, params=params)
|
||||
return resp.json()
|
||||
# BCDA rejects /jobs without the bulk-data headers
|
||||
# ("Prefer header is required" / "application/fhir+json is
|
||||
# the only supported response format").
|
||||
try:
|
||||
resp = self._request("GET", url, headers=dict(ASYNC_HEADERS), params=params)
|
||||
except BcdaHttpError as exc:
|
||||
# bcda-app answers 404 when the filter matches nothing
|
||||
# (api_test.go TestJobsStatusNotFoundWithStatus).
|
||||
if exc.status_code == 404 and status:
|
||||
return []
|
||||
raise
|
||||
body = resp.json()
|
||||
if isinstance(body, dict) and body.get("resourceType") == "Bundle":
|
||||
return [e["resource"] for e in body.get("entry", []) if "resource" in e]
|
||||
return body
|
||||
|
||||
def completed_jobs(self) -> list[dict]:
|
||||
"""Completed jobs, newest first, as ``(job_id, request, end)`` dicts.
|
||||
|
||||
Convenience over :meth:`jobs` for "reuse the last export"
|
||||
logic: ``job_id`` is the numeric id, ``job_url`` the status
|
||||
URL for :meth:`poll`, ``request`` the originating ``$export``
|
||||
line, ``types`` the ``_type`` list it asked for, ``end`` the
|
||||
completion instant. Only jobs started on this client's API
|
||||
version are returned, since ``/data`` URLs from a different
|
||||
version serve different payloads. Output files stay
|
||||
downloadable for 24 hours after completion (then 410/404).
|
||||
"""
|
||||
out = []
|
||||
# Filter client-side: the sandbox's ``_status=Completed`` answered
|
||||
# 404 for jobs the unfiltered Bundle lists as ``completed``.
|
||||
for task in self.jobs():
|
||||
if str(task.get("status", "")).lower() != "completed":
|
||||
continue
|
||||
ident = (task.get("identifier") or [{}])[0]
|
||||
job_id = ident.get("value")
|
||||
request = ((task.get("input") or [{}])[0]).get("valueString", "")
|
||||
if not job_id or f"/api/{self.version}/" not in request:
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"job_id": job_id,
|
||||
"job_url": f"{self.base_url}/api/{self.version}/jobs/{job_id}",
|
||||
"request": request,
|
||||
"types": _requested_types(request),
|
||||
"end": (task.get("executionPeriod") or {}).get("end", ""),
|
||||
}
|
||||
)
|
||||
out.sort(key=lambda j: j["end"], reverse=True)
|
||||
return out
|
||||
|
||||
def attribution_status(self) -> dict:
|
||||
"""Get last ingestion timestamps for attribution data.
|
||||
@@ -572,6 +736,20 @@ class Client:
|
||||
resp = self._request("GET", url, auth_required=False, retry=False)
|
||||
return resp.json()
|
||||
|
||||
def server_version(self) -> dict:
|
||||
"""GET ``/_version`` — the deployed bcda-app release (no auth)."""
|
||||
resp = self._request(
|
||||
"GET", f"{self.base_url}/_version", auth_required=False, retry=False
|
||||
)
|
||||
return resp.json()
|
||||
|
||||
def health(self) -> dict:
|
||||
"""GET ``/_health`` — database liveness (no auth)."""
|
||||
resp = self._request(
|
||||
"GET", f"{self.base_url}/_health", auth_required=False, retry=False
|
||||
)
|
||||
return resp.json()
|
||||
|
||||
# ── Parsing helpers ──────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -552,21 +552,44 @@ def stg_beneficiary_demographics(
|
||||
Patient provides: name, dob, sex, race, death_dt, address
|
||||
Coverage provides: ms_cd, orec, enrollment periods
|
||||
|
||||
Join on bene_mbi_id = subscriber_id. Coverage may have
|
||||
multiple rows per beneficiary (Part A, B, D); we keep
|
||||
the first match after dedup.
|
||||
Join on bene_mbi_id = subscriber_id. Coverage has multiple rows
|
||||
per beneficiary (Part A, B, D in v2; Part A, B, C, D and a Dual
|
||||
Medicare/Medicaid row per period in v3). The Part A row is the
|
||||
demographics row — it carries ms_cd / orec / entitlement — with
|
||||
Part B as the fallback; the v3 Dual row is joined separately for
|
||||
``dual_stus_cd`` (v2 keeps ``dual_01`` on the Part A row).
|
||||
|
||||
Operations: join, unique
|
||||
Operations: filter, sort, unique, join
|
||||
Dependencies: bcda.patient, bcda.coverage
|
||||
"""
|
||||
cov = bcda__coverage.unique(subset=["subscriber_id"], keep="any")
|
||||
return bcda__patient.join(
|
||||
cov,
|
||||
cols = bcda__coverage.columns
|
||||
parts = bcda__coverage
|
||||
if "class_plan" in cols:
|
||||
parts = parts.filter(nw.col("class_plan").is_in(["Part A", "Part B"])).sort(
|
||||
"class_plan" # "Part A" < "Part B"
|
||||
)
|
||||
parts = parts.unique(subset=["subscriber_id"], keep="first", maintain_order=True)
|
||||
joined = bcda__patient.join(
|
||||
parts,
|
||||
left_on="bene_mbi_id",
|
||||
right_on="subscriber_id",
|
||||
how="left",
|
||||
suffix="_cov",
|
||||
).unique(subset=["bene_mbi_id"], keep="any")
|
||||
)
|
||||
if "dual_stus_cd" in cols and "class_plan" in cols:
|
||||
dual = (
|
||||
bcda__coverage.filter(nw.col("class_plan") == "Dual Medicare/Medicaid")
|
||||
.select("subscriber_id", nw.col("dual_stus_cd").alias("dual_stus_cd_dual"))
|
||||
.unique(subset=["subscriber_id"], keep="any")
|
||||
)
|
||||
joined = joined.join(
|
||||
dual,
|
||||
left_on="bene_mbi_id",
|
||||
right_on="subscriber_id",
|
||||
how="left",
|
||||
suffix="_dual",
|
||||
)
|
||||
return joined.unique(subset=["bene_mbi_id"], keep="any")
|
||||
|
||||
|
||||
@nw.narwhalify
|
||||
@@ -584,6 +607,12 @@ def eligibility(
|
||||
"""
|
||||
df = bcda___stg_beneficiary_demographics
|
||||
null_str = nw.lit(None).cast(nw.String)
|
||||
# v2 keeps dual status on the Part A row (dual_01); v3 carries it on
|
||||
# the Dual Medicare/Medicaid row that stg_beneficiary_demographics
|
||||
# joins in as dual_stus_cd_dual.
|
||||
dual = nw.col("dual_01")
|
||||
if "dual_stus_cd_dual" in df.columns:
|
||||
dual = nw.coalesce(nw.col("dual_01"), nw.col("dual_stus_cd_dual"))
|
||||
|
||||
gender = (
|
||||
nw.when(nw.col("bene_sex_cd") == "1")
|
||||
@@ -641,7 +670,7 @@ def eligibility(
|
||||
nw.lit("medicare").alias("payer_type"),
|
||||
nw.lit("medicare").alias("plan"),
|
||||
nw.col("orec").alias("original_reason_entitlement_code"),
|
||||
nw.col("dual_01").alias("dual_status_code"),
|
||||
dual.alias("dual_status_code"),
|
||||
nw.col("ms_cd").alias("medicare_status_code"),
|
||||
null_str.alias("group_id"),
|
||||
null_str.alias("group_name"),
|
||||
|
||||
@@ -35,6 +35,7 @@ from pathlib import Path
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from bcda.express import v3
|
||||
from bcda.table.coverage import Coverage
|
||||
from bcda.table.eob import EobItem, ExplanationOfBenefit
|
||||
from bcda.table.patient import Patient
|
||||
@@ -228,7 +229,13 @@ def _coding_code(
|
||||
|
||||
|
||||
def flatten_patient(resource: dict) -> Patient:
|
||||
"""Flatten a raw FHIR R4 Patient resource into a Patient model."""
|
||||
"""Flatten a raw FHIR R4 Patient resource into a Patient model.
|
||||
|
||||
v3 (C4BB 2.2.0) resources are routed to :mod:`bcda.express.v3`;
|
||||
the body below is the v2 (``resources/variables``) dialect.
|
||||
"""
|
||||
if v3.is_v3(resource):
|
||||
return v3.flatten_patient(resource)
|
||||
extensions = resource.get("extension", [])
|
||||
identifiers = resource.get("identifier", [])
|
||||
address = (resource.get("address") or [{}])[0]
|
||||
@@ -548,7 +555,12 @@ def _extract_item(
|
||||
def flatten_eob(
|
||||
resource: dict,
|
||||
) -> tuple[ExplanationOfBenefit, list[EobItem]]:
|
||||
"""Flatten a raw FHIR R4 EOB into header + item models."""
|
||||
"""Flatten a raw FHIR R4 EOB into header + item models.
|
||||
|
||||
v3 resources are routed to :mod:`bcda.express.v3`.
|
||||
"""
|
||||
if v3.is_v3(resource):
|
||||
return v3.flatten_eob(resource)
|
||||
extensions = resource.get("extension", [])
|
||||
identifiers = resource.get("identifier", [])
|
||||
billable = resource.get("billablePeriod", {})
|
||||
@@ -801,7 +813,12 @@ def flatten_eob(
|
||||
|
||||
|
||||
def flatten_coverage(resource: dict) -> Coverage:
|
||||
"""Flatten a raw FHIR R4 Coverage resource into a Coverage model."""
|
||||
"""Flatten a raw FHIR R4 Coverage resource into a Coverage model.
|
||||
|
||||
v3 resources are routed to :mod:`bcda.express.v3`.
|
||||
"""
|
||||
if v3.is_v3(resource):
|
||||
return v3.flatten_coverage(resource)
|
||||
extensions = resource.get("extension", [])
|
||||
|
||||
# Class discrimination
|
||||
@@ -940,6 +957,66 @@ def _read_ndjson(path: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in f if line.strip()]
|
||||
|
||||
|
||||
def _iter_ndjson(paths: list[Path]):
|
||||
"""Yield resources from several ndjson parts, one line at a time."""
|
||||
for path in paths:
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
if line.strip():
|
||||
yield json.loads(line)
|
||||
|
||||
|
||||
_ARROW_TYPES = {
|
||||
"str": pa.string(),
|
||||
"float": pa.float64(),
|
||||
"int": pa.int64(),
|
||||
"bool": pa.bool_(),
|
||||
"date": pa.date32(),
|
||||
"datetime": pa.timestamp("us"),
|
||||
}
|
||||
|
||||
|
||||
def arrow_schema(model: type) -> pa.Schema:
|
||||
"""Arrow schema for a ``SQLTable`` model (every column nullable).
|
||||
|
||||
Streaming writes need a fixed schema up front — inferring it from
|
||||
the first batch would type an all-NULL column as ``null`` and then
|
||||
reject a later batch that carries values.
|
||||
"""
|
||||
import types as _types
|
||||
|
||||
fields = []
|
||||
for name, info in model.model_fields.items():
|
||||
ann = info.annotation
|
||||
if isinstance(ann, _types.UnionType):
|
||||
ann = next(a for a in ann.__args__ if a is not type(None))
|
||||
key = getattr(ann, "__name__", str(ann))
|
||||
fields.append(pa.field(name, _ARROW_TYPES.get(key, pa.string())))
|
||||
return pa.schema(fields)
|
||||
|
||||
|
||||
class _ParquetSink:
|
||||
"""Append-by-batch Parquet writer over fsspec; counts rows."""
|
||||
|
||||
def __init__(self, fs: object, path: str, schema: pa.Schema) -> None:
|
||||
parent = path.rsplit("/", 1)[0]
|
||||
fs.mkdirs(parent, exist_ok=True)
|
||||
self._fh = fs.open(path, "wb")
|
||||
self._writer = pq.ParquetWriter(self._fh, schema)
|
||||
self._schema = schema
|
||||
self.rows = 0
|
||||
|
||||
def write(self, rows: list[dict]) -> None:
|
||||
if not rows:
|
||||
return
|
||||
self._writer.write_table(pa.Table.from_pylist(rows, schema=self._schema))
|
||||
self.rows += len(rows)
|
||||
|
||||
def close(self) -> None:
|
||||
self._writer.close()
|
||||
self._fh.close()
|
||||
|
||||
|
||||
def _write_parquet(
|
||||
fs: object,
|
||||
path: str,
|
||||
@@ -956,8 +1033,48 @@ def _write_parquet(
|
||||
return table.num_rows
|
||||
|
||||
|
||||
RESOURCE_TYPES = ("Patient", "ExplanationOfBenefit", "Coverage")
|
||||
|
||||
# Resources per Arrow batch while streaming an export to Parquet. A
|
||||
# Large-entity v3 EOB export is ~190 files / 2 GB of ndjson, so the
|
||||
# flattener never holds a whole resource type in memory.
|
||||
BATCH_ROWS = 2000
|
||||
|
||||
|
||||
def resolve_ndjson(ndjson_dir: str | Path | list[str | Path]) -> dict[str, list[Path]]:
|
||||
"""``{resource_type: [part, ...]}`` for the export files.
|
||||
|
||||
*ndjson_dir* may be one directory or several (a v3 pull is often
|
||||
three single-type jobs, each landing in its own ``exports/<key>/``
|
||||
directory — see ``aco.load.bcda``). Later directories win. A
|
||||
type's parts are ``<Type>.ndjson``, ``<Type>.1.ndjson``, … as the
|
||||
client and store name them (BCDA splits large exports into many
|
||||
output files); they are returned in part order.
|
||||
"""
|
||||
dirs = [ndjson_dir] if isinstance(ndjson_dir, (str, Path)) else list(ndjson_dir)
|
||||
found: dict[str, list[Path]] = {}
|
||||
for d in dirs:
|
||||
for rtype in RESOURCE_TYPES:
|
||||
parts = sorted(
|
||||
Path(d).glob(f"{rtype}*.ndjson"),
|
||||
key=lambda p: (
|
||||
int(p.name.split(".")[1])
|
||||
if p.name.count(".") == 2 and p.name.split(".")[1].isdigit()
|
||||
else -1
|
||||
),
|
||||
)
|
||||
parts = [
|
||||
p
|
||||
for p in parts
|
||||
if p.name == f"{rtype}.ndjson" or p.name.split(".")[1].isdigit()
|
||||
]
|
||||
if parts:
|
||||
found[rtype] = parts
|
||||
return found
|
||||
|
||||
|
||||
def flatten_export(
|
||||
ndjson_dir: str | Path,
|
||||
ndjson_dir: str | Path | list[str | Path],
|
||||
store_path: str | Path | None = None,
|
||||
*,
|
||||
storage_options: dict | None = None,
|
||||
@@ -965,9 +1082,10 @@ def flatten_export(
|
||||
"""Flatten raw FHIR ndjson files and write to Parquet.
|
||||
|
||||
Reads ``Patient.ndjson``, ``ExplanationOfBenefit.ndjson``, and
|
||||
``Coverage.ndjson`` from *ndjson_dir*, flattens them into the
|
||||
``bcda.table`` Pydantic models, and persists each as a Parquet
|
||||
file under *store_path*.
|
||||
``Coverage.ndjson`` from *ndjson_dir* (one directory or a list —
|
||||
see :func:`resolve_ndjson`), flattens them into the ``bcda.table``
|
||||
Pydantic models (v2 or v3 dialect per resource), and persists each
|
||||
as a Parquet file under *store_path*.
|
||||
|
||||
Output layout::
|
||||
|
||||
@@ -998,85 +1116,88 @@ def flatten_export(
|
||||
|
||||
store_path = path("storage.bcda")
|
||||
setup()
|
||||
ndjson_dir = Path(ndjson_dir)
|
||||
files = resolve_ndjson(ndjson_dir)
|
||||
root = str(store_path).rstrip("/")
|
||||
fs, _ = fsspec.core.url_to_fs(root, **(storage_options or {}))
|
||||
counts: dict[str, int] = {}
|
||||
|
||||
# Patient
|
||||
patient_path = ndjson_dir / "Patient.ndjson"
|
||||
if patient_path.exists():
|
||||
if files.get("Patient"):
|
||||
t0 = time.monotonic()
|
||||
raw = _read_ndjson(patient_path)
|
||||
patients = flatten_patients(raw)
|
||||
rows = [p.model_dump() for p in patients]
|
||||
n = _write_parquet(fs, f"{root}/flat/patient.parquet", rows)
|
||||
elapsed = time.monotonic() - t0
|
||||
counts["bcda.patient"] = n
|
||||
sink = _ParquetSink(fs, f"{root}/flat/patient.parquet", arrow_schema(Patient))
|
||||
batch: list[dict] = []
|
||||
for raw in _iter_ndjson(files["Patient"]):
|
||||
batch.append(flatten_patient(raw).model_dump())
|
||||
if len(batch) >= BATCH_ROWS:
|
||||
sink.write(batch)
|
||||
batch = []
|
||||
sink.write(batch)
|
||||
sink.close()
|
||||
counts["bcda.patient"] = sink.rows
|
||||
log.info(
|
||||
"Flattened %d patients in %.1fs",
|
||||
n,
|
||||
elapsed,
|
||||
extra={"table": "bcda.patient", "rows": n},
|
||||
sink.rows,
|
||||
time.monotonic() - t0,
|
||||
extra={"table": "bcda.patient", "rows": sink.rows},
|
||||
)
|
||||
|
||||
# ExplanationOfBenefit + EobItem
|
||||
eob_path = ndjson_dir / "ExplanationOfBenefit.ndjson"
|
||||
if eob_path.exists():
|
||||
if files.get("ExplanationOfBenefit"):
|
||||
t0 = time.monotonic()
|
||||
raw = _read_ndjson(eob_path)
|
||||
headers, items = flatten_eobs(raw)
|
||||
header_rows = [h.model_dump() for h in headers]
|
||||
item_rows = [i.model_dump() for i in items]
|
||||
n_headers = _write_parquet(
|
||||
head_sink = _ParquetSink(
|
||||
fs,
|
||||
f"{root}/flat/explanation_of_benefit.parquet",
|
||||
header_rows,
|
||||
arrow_schema(ExplanationOfBenefit),
|
||||
)
|
||||
n_items = _write_parquet(
|
||||
fs,
|
||||
f"{root}/flat/eob_item.parquet",
|
||||
item_rows,
|
||||
item_sink = _ParquetSink(
|
||||
fs, f"{root}/flat/eob_item.parquet", arrow_schema(EobItem)
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
counts["bcda.explanation_of_benefit"] = n_headers
|
||||
counts["bcda.eob_item"] = n_items
|
||||
heads: list[dict] = []
|
||||
items: list[dict] = []
|
||||
for raw in _iter_ndjson(files["ExplanationOfBenefit"]):
|
||||
header, lines = flatten_eob(raw)
|
||||
heads.append(header.model_dump())
|
||||
items.extend(i.model_dump() for i in lines)
|
||||
if len(heads) >= BATCH_ROWS:
|
||||
head_sink.write(heads)
|
||||
item_sink.write(items)
|
||||
heads, items = [], []
|
||||
head_sink.write(heads)
|
||||
item_sink.write(items)
|
||||
head_sink.close()
|
||||
item_sink.close()
|
||||
counts["bcda.explanation_of_benefit"] = head_sink.rows
|
||||
counts["bcda.eob_item"] = item_sink.rows
|
||||
log.info(
|
||||
"Flattened %d EOBs (%d items) in %.1fs",
|
||||
n_headers,
|
||||
n_items,
|
||||
elapsed,
|
||||
head_sink.rows,
|
||||
item_sink.rows,
|
||||
time.monotonic() - t0,
|
||||
extra={
|
||||
"table": "bcda.explanation_of_benefit",
|
||||
"rows": n_headers,
|
||||
"items": n_items,
|
||||
"rows": head_sink.rows,
|
||||
"items": item_sink.rows,
|
||||
},
|
||||
)
|
||||
|
||||
# Coverage
|
||||
coverage_path = ndjson_dir / "Coverage.ndjson"
|
||||
if coverage_path.exists():
|
||||
if files.get("Coverage"):
|
||||
t0 = time.monotonic()
|
||||
raw = _read_ndjson(coverage_path)
|
||||
coverages = flatten_coverages(raw)
|
||||
rows = [c.model_dump() for c in coverages]
|
||||
n = _write_parquet(
|
||||
fs,
|
||||
f"{root}/flat/coverage.parquet",
|
||||
rows,
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
counts["bcda.coverage"] = n
|
||||
sink = _ParquetSink(fs, f"{root}/flat/coverage.parquet", arrow_schema(Coverage))
|
||||
batch = []
|
||||
for raw in _iter_ndjson(files["Coverage"]):
|
||||
batch.append(flatten_coverage(raw).model_dump())
|
||||
if len(batch) >= BATCH_ROWS:
|
||||
sink.write(batch)
|
||||
batch = []
|
||||
sink.write(batch)
|
||||
sink.close()
|
||||
counts["bcda.coverage"] = sink.rows
|
||||
log.info(
|
||||
"Flattened %d coverage records in %.1fs",
|
||||
n,
|
||||
elapsed,
|
||||
extra={"table": "bcda.coverage", "rows": n},
|
||||
"Flattened %d coverages in %.1fs",
|
||||
sink.rows,
|
||||
time.monotonic() - t0,
|
||||
extra={"table": "bcda.coverage", "rows": sink.rows},
|
||||
)
|
||||
|
||||
log.info(
|
||||
"Flatten complete: %s",
|
||||
counts,
|
||||
extra={"counts": counts},
|
||||
)
|
||||
return counts
|
||||
|
||||
1212
src/bcda/express/v3.py
Normal file
1212
src/bcda/express/v3.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -21,7 +21,7 @@ Pipeline expressions (12 total)::
|
||||
stg_beneficiary_demographics Patient + Coverage
|
||||
eligibility demographics → eligibility
|
||||
|
||||
Source data — BCDA v2 bulk FHIR export:
|
||||
Source data — BCDA bulk FHIR export (v2 or v3, flattened to bcda.*):
|
||||
- ExplanationOfBenefit (header + items)
|
||||
- Patient
|
||||
- Coverage
|
||||
|
||||
@@ -19,7 +19,7 @@ the appropriate extra (``s3fs``, ``adlfs``) installed by the user.
|
||||
|
||||
Usage::
|
||||
|
||||
from bcda.client import Client
|
||||
from bcda.client import Client, JobCancelledError
|
||||
from bcda.store import Store
|
||||
|
||||
store = Store()
|
||||
@@ -51,7 +51,7 @@ import fsspec
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from bcda.client import Client
|
||||
from bcda.client import Client, JobCancelledError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -264,7 +264,9 @@ class Store:
|
||||
endpoint: str = "Group/all",
|
||||
types: list[str] | None = None,
|
||||
since: str | None = None,
|
||||
type_filter: str | None = None,
|
||||
poll_interval: float = 10.0,
|
||||
cancel_retries: int = 0,
|
||||
) -> str:
|
||||
"""Run a full export with persistent state tracking.
|
||||
|
||||
@@ -283,11 +285,22 @@ class Store:
|
||||
FHIR resource types to request.
|
||||
since : str | None
|
||||
``_since`` timestamp filter.
|
||||
type_filter : str | None
|
||||
v3 ``_typeFilter`` subquery (recorded alongside ``types``).
|
||||
poll_interval : float
|
||||
Seconds between status polls.
|
||||
cancel_retries : int
|
||||
How many times to restart the export after the server
|
||||
reports it cancelled. On the public sandbox every caller
|
||||
of a model entity can ``DELETE`` any job, and sweeps that
|
||||
cancel every in-progress job happen several times an hour;
|
||||
each cancellation is recorded as a ``cancelled`` transition
|
||||
under the same key before the next attempt.
|
||||
"""
|
||||
key = _generate_key()
|
||||
types_str = ",".join(types) if types else ""
|
||||
if type_filter:
|
||||
types_str = f"{types_str}|_typeFilter={type_filter}"
|
||||
log.info("Starting export %s", key, extra={"job_key": key})
|
||||
|
||||
# 1. Record pending.
|
||||
@@ -300,23 +313,47 @@ class Store:
|
||||
)
|
||||
|
||||
try:
|
||||
# 2. Start export.
|
||||
job_url = client.start_export(
|
||||
endpoint=endpoint,
|
||||
types=types,
|
||||
since=since,
|
||||
)
|
||||
self._append_job(
|
||||
key=key,
|
||||
job_url=job_url,
|
||||
endpoint=endpoint,
|
||||
types=types_str,
|
||||
since=since or "",
|
||||
status="polling",
|
||||
)
|
||||
for attempt in range(cancel_retries + 1):
|
||||
# 2. Start export.
|
||||
job_url = client.start_export(
|
||||
endpoint=endpoint,
|
||||
types=types,
|
||||
since=since,
|
||||
type_filter=type_filter,
|
||||
)
|
||||
self._append_job(
|
||||
key=key,
|
||||
job_url=job_url,
|
||||
endpoint=endpoint,
|
||||
types=types_str,
|
||||
since=since or "",
|
||||
status="polling",
|
||||
)
|
||||
|
||||
# 3. Poll until complete.
|
||||
result = client.poll(job_url, interval=poll_interval)
|
||||
# 3. Poll until complete.
|
||||
try:
|
||||
result = client.poll(job_url, interval=poll_interval)
|
||||
break
|
||||
except JobCancelledError as exc:
|
||||
self._append_job(
|
||||
key=key,
|
||||
job_url=job_url,
|
||||
endpoint=endpoint,
|
||||
types=types_str,
|
||||
since=since or "",
|
||||
status="cancelled",
|
||||
error=str(exc),
|
||||
)
|
||||
if attempt >= cancel_retries:
|
||||
raise
|
||||
log.warning(
|
||||
"Export %s cancelled server-side (%s); restarting (%d/%d)",
|
||||
key,
|
||||
job_url,
|
||||
attempt + 1,
|
||||
cancel_retries,
|
||||
extra={"job_key": key},
|
||||
)
|
||||
self._append_job(
|
||||
key=key,
|
||||
job_url=job_url,
|
||||
@@ -349,6 +386,60 @@ class Store:
|
||||
log.info("Export %s finished", key, extra={"job_key": key})
|
||||
return key
|
||||
|
||||
def adopt(
|
||||
self,
|
||||
client: Client,
|
||||
job_url: str,
|
||||
*,
|
||||
endpoint: str = "Group/all",
|
||||
types: list[str] | None = None,
|
||||
poll_interval: float = 10.0,
|
||||
) -> str:
|
||||
"""Record and download a job this store did not start.
|
||||
|
||||
BCDA keeps a completed job's files for 24 hours, and on the
|
||||
public sandbox every caller of a model entity sees the same
|
||||
job list — so a fresh export (6 min and 2 GB for the Large
|
||||
entity's EOBs) is wasted when ``client.completed_jobs()``
|
||||
already lists one. ``adopt`` polls *job_url* (returns at once
|
||||
when complete), appends the job under a new key with status
|
||||
``complete`` and downloads its outputs exactly as
|
||||
:meth:`export` would. Returns the job key.
|
||||
"""
|
||||
key = _generate_key()
|
||||
types_str = ",".join(types) if types else ""
|
||||
log.info("Adopting job %s as %s", job_url, key, extra={"job_key": key})
|
||||
self._append_job(
|
||||
key=key,
|
||||
job_url=job_url,
|
||||
endpoint=endpoint,
|
||||
types=types_str,
|
||||
status="polling",
|
||||
)
|
||||
try:
|
||||
result = client.poll(job_url, interval=poll_interval)
|
||||
self._append_job(
|
||||
key=key,
|
||||
job_url=job_url,
|
||||
endpoint=endpoint,
|
||||
types=types_str,
|
||||
status="complete",
|
||||
)
|
||||
self._download_outputs(client, key, result)
|
||||
except Exception as exc:
|
||||
log.error("Adopt %s failed: %s", key, exc, extra={"job_key": key})
|
||||
self._append_job(
|
||||
key=key,
|
||||
job_url=job_url,
|
||||
endpoint=endpoint,
|
||||
types=types_str,
|
||||
status="failed",
|
||||
error=str(exc),
|
||||
)
|
||||
raise
|
||||
log.info("Adopted %s finished", key, extra={"job_key": key})
|
||||
return key
|
||||
|
||||
def _download_outputs(
|
||||
self,
|
||||
client: Client,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""FHIR R4 Coverage resource — beneficiary enrollment and entitlement.
|
||||
|
||||
Flattened from the BCDA v2 Coverage ndjson export. Each beneficiary
|
||||
Flattened from the BCDA Coverage ndjson export (v2 or v3). Each beneficiary
|
||||
may have multiple Coverage records (Part A, Part B, Part D).
|
||||
|
||||
The Coverage resource carries enrollment status, dual eligibility,
|
||||
@@ -9,7 +9,14 @@ extensions for buy-in, HMO, cost sharing, Part C/D contracts, and
|
||||
Retiree Drug Subsidy. Each monthly series has 12 fields (01–12)
|
||||
representing January through December of the reference year.
|
||||
|
||||
Source: ``/api/v2/Coverage/$export`` bulk FHIR endpoint.
|
||||
v3 delivers one Coverage per plan per period (``class_plan`` ∈ Part A,
|
||||
Part B, Part C, Part D, Dual Medicare/Medicaid) with ``Coverage.period``
|
||||
instead of the v2 reference-year monthly series; the v3 extensions land
|
||||
in the ``*_cd`` columns below and the legacy part A/B/D begin/end dates
|
||||
are filled from the matching plan's period so downstream joins keep
|
||||
working.
|
||||
|
||||
Source: ``/api/{v2,v3}/Coverage/$export`` bulk FHIR endpoint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -563,3 +570,74 @@ class Coverage(SQLTable):
|
||||
|
||||
ma_12: str | None = None
|
||||
"""Medicare Advantage indicator — December."""
|
||||
|
||||
# ── v3-only (BCDA v3 Data Dictionary) ────────────────────
|
||||
period_start: date | None = None
|
||||
"""Coverage.period.start — enrollment / eligibility begin for this
|
||||
plan row (IDR BENE_RNG_BGN_DT, BENE_ENRLMT_BGN_DT, BENE_MDCD_ELGBLTY_BGN_DT)."""
|
||||
|
||||
period_end: date | None = None
|
||||
"""Coverage.period.end (IDR BENE_RNG_END_DT, BENE_ENRLMT_END_DT,
|
||||
BENE_MDCD_ELGBLTY_END_DT)."""
|
||||
|
||||
buyin_cd: str | None = None
|
||||
"""Buy-in code (extension BENE-BUYIN-CD; CCW MDCR_ENTLMT_BUYIN_*_IND)."""
|
||||
|
||||
entlmt_stus_cd: str | None = None
|
||||
"""Medicare entitlement status (BENE-MDCR-ENTLMT-STUS-CD; CCW
|
||||
BENE_PTA/PTB_TRMNTN_CD — also copied to ``a_trm_cd``/``b_trm_cd``)."""
|
||||
|
||||
enrlmt_rsn_cd: str | None = None
|
||||
"""Enrollment reason (BENE-ENRLMT-RSN-CD)."""
|
||||
|
||||
dsbld_stus_id: str | None = None
|
||||
"""Disabled status (BENE-DSBLD-STUS-ID)."""
|
||||
|
||||
dual_stus_cd: str | None = None
|
||||
"""Dual status (BENE-DUAL-STUS-CD) on the Dual Medicare/Medicaid
|
||||
row; CCW DUAL_STUS_CD_01..12."""
|
||||
|
||||
dual_type_cd: str | None = None
|
||||
"""Medicare/Medicaid dual type (BENE-DUAL-TYPE-CD)."""
|
||||
|
||||
medicaid_state_cd: str | None = None
|
||||
"""Medicaid state, USPS (MEDICAID-STATE-CD)."""
|
||||
|
||||
cntrct_num: str | None = None
|
||||
"""Part C/D contract number (BENE-CNTRCT-NUM; CCW PTC/PTD_CNTRCT_ID_*)."""
|
||||
|
||||
pbp_num: str | None = None
|
||||
"""Part C/D plan benefit package (BENE-PBP-NUM; CCW PTC/PTD_PBP_ID_*)."""
|
||||
|
||||
cvrg_type_cd: str | None = None
|
||||
"""Part C/D coverage type (BENE-CVRG-TYPE-CD; CCW PTC_PLAN_TYPE_CD_*)."""
|
||||
|
||||
pbp_sgmt_num: str | None = None
|
||||
"""Part D PBP segment (CNTRCT-PBP-SGMT-NUM; CCW PTD_SGMT_ID_*)."""
|
||||
|
||||
lis_efctv_cd: str | None = None
|
||||
"""Part D LIS effective code (BENE-LIS-EFCTV-CD; CCW CST_SHR_GRP_CD_*)."""
|
||||
|
||||
lis_copmt_lvl_cd: str | None = None
|
||||
"""Part D LIS copayment level (BENE-LIS-COPMT-LVL-CD)."""
|
||||
|
||||
lis_ptd_prm_pct: str | None = None
|
||||
"""Part D LIS premium subsidy percentage (BENE-LIS-PTD-PRM-PCT)."""
|
||||
|
||||
rds_sw: str | None = None
|
||||
"""Retiree drug subsidy switch (BENE-ENRLMT-EMPLR-SBSDY-SW; CCW RDS_IND_*)."""
|
||||
|
||||
rx_id: str | None = None
|
||||
"""Part D PDP member id (Coverage.class[rxid])."""
|
||||
|
||||
rx_group: str | None = None
|
||||
"""Part D PDP group (Coverage.class[rxgroup])."""
|
||||
|
||||
rx_pcn: str | None = None
|
||||
"""Part D PDP processor control number (Coverage.class[rxpcn])."""
|
||||
|
||||
rx_bin: str | None = None
|
||||
"""Part D PDP bank id (Coverage.class[rxbin])."""
|
||||
|
||||
cntrct_pbp_name: str | None = None
|
||||
"""Part C/D contract plan name (contained Organization.name)."""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""FHIR R4 ExplanationOfBenefit resource — adjudicated claims.
|
||||
|
||||
Flattened from the BCDA v2 ExplanationOfBenefit ndjson export.
|
||||
Flattened from the BCDA ExplanationOfBenefit ndjson export (v2 or v3).
|
||||
Covers Part A (institutional), Part B (professional/DME), and
|
||||
Part D (pharmacy) claims in a single resource type.
|
||||
|
||||
@@ -10,7 +10,14 @@ fields (line items with revenue codes, HCPCS, adjudication amounts).
|
||||
This model represents the header-level flattened fields; item-level
|
||||
arrays are modeled separately in ``EobItem``.
|
||||
|
||||
Source: ``/api/v2/ExplanationOfBenefit/$export`` bulk FHIR endpoint.
|
||||
v3 represents partially *and* fully adjudicated claims as EOBs (the
|
||||
``system_type`` tag says which: SharedSystem, NationalClaimsHistory,
|
||||
DDPS) and consolidates the v2 benefitBalance/extension amounts into
|
||||
``adjudication``/``total``/``item.adjudication`` under the
|
||||
``https://bluebutton.cms.gov/fhir/CodeSystem/Adjudication`` system.
|
||||
The v3-only columns below carry those amounts under their IDR names.
|
||||
|
||||
Source: ``/api/{v2,v3}/ExplanationOfBenefit/$export`` bulk FHIR endpoint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -360,6 +367,244 @@ class ExplanationOfBenefit(SQLTable):
|
||||
"""Pharmacy service type code
|
||||
(Eob.facility extension valueCoding)."""
|
||||
|
||||
# ── v3-only: identity and provenance (BCDA v3 Data Dictionary) ──
|
||||
clm_uniq_id: str | None = None
|
||||
"""IDR claim surrogate key (identifier[C4BB uc]; IDR CLM_UNIQ_ID).
|
||||
Same value as ``clm_id`` on v3 rows."""
|
||||
|
||||
clm_orig_cntl_num: str | None = None
|
||||
"""Original claim control number for adjusted claims
|
||||
(related[].reference where system CLM-CNTL-NUM)."""
|
||||
|
||||
system_type: str | None = None
|
||||
"""meta.tag System-Type: SharedSystem (partially adjudicated),
|
||||
NationalClaimsHistory, DDPS."""
|
||||
|
||||
final_action: str | None = None
|
||||
"""meta.tag Final-Action: FinalAction / NotFinalAction."""
|
||||
|
||||
eob_type: str | None = None
|
||||
"""BlueButton EOB-TYPE coding: INPATIENT, OUTPATIENT, SNF, HHA,
|
||||
HOSPICE, CARRIER, DME, PDE."""
|
||||
|
||||
clm_src_id: str | None = None
|
||||
"""meta.source (IDR CLM_SRC_ID / META_SRC_SK): NCH, DDPS, FISS, MCS, VMS."""
|
||||
|
||||
outcome: str | None = None
|
||||
"""ExplanationOfBenefit.outcome: complete / partial."""
|
||||
|
||||
clm_adjstmt_type_cd: str | None = None
|
||||
"""Claim adjustment type (supportingInfo CLM_ADJSTMT_TYPE_CD)."""
|
||||
|
||||
clm_idr_ld_dt: date | None = None
|
||||
"""IDR load date (supportingInfo CLM_IDR_LD_DT)."""
|
||||
|
||||
clm_cms_proc_dt: date | None = None
|
||||
"""CMS process date (supportingInfo CLM_CMS_PROC_DT)."""
|
||||
|
||||
clm_nch_wkly_proc_dt: date | None = None
|
||||
"""NCH weekly process date (supportingInfo CLM_NCH_WKLY_PROC_DT)."""
|
||||
|
||||
clm_submsn_dt: date | None = None
|
||||
"""Claim received date (supportingInfo clmrecvddate)."""
|
||||
|
||||
clm_pd_dt: date | None = None
|
||||
"""Payment date (payment.date; IDR CLM_PD_DT)."""
|
||||
|
||||
clm_sbmt_frmt_cd: str | None = None
|
||||
"""Submission format (supportingInfo CLM_SBMT_FRMT_CD)."""
|
||||
|
||||
clm_blg_prvdr_name: str | None = None
|
||||
"""Billing provider name (contained Organization.name)."""
|
||||
|
||||
clm_blg_prvdr_tax_num: str | None = None
|
||||
"""Billing provider tax number (contained Organization identifier
|
||||
urn:oid:2.16.840.1.113883.4.4)."""
|
||||
|
||||
clm_blg_prvdr_zip5_cd: str | None = None
|
||||
"""Billing provider ZIP (contained Organization.address.postalCode)."""
|
||||
|
||||
clm_dschrg_dt: date | None = None
|
||||
"""Discharge date (supportingInfo admissionperiod end)."""
|
||||
|
||||
clm_actv_care_thru_dt: date | None = None
|
||||
"""Covered care through date (supportingInfo CLM_ACTV_CARE_THRU_DT)."""
|
||||
|
||||
clm_mdcr_exhstd_dt: date | None = None
|
||||
"""Medicare benefits exhausted date (supportingInfo CLM_MDCR_EXHSTD_DT)."""
|
||||
|
||||
clm_ncvrd_from_dt: date | None = None
|
||||
"""Noncovered stay from (supportingInfo CLM_NCVRD_FROM_DT)."""
|
||||
|
||||
clm_ncvrd_thru_dt: date | None = None
|
||||
"""Noncovered stay through (supportingInfo CLM_NCVRD_THRU_DT)."""
|
||||
|
||||
clm_qlfy_stay_from_dt: date | None = None
|
||||
"""Qualified stay from (supportingInfo CLM_QLFY_STAY_FROM_DT)."""
|
||||
|
||||
clm_qlfy_stay_thru_dt: date | None = None
|
||||
"""Qualified stay through (supportingInfo CLM_QLFY_STAY_THRU_DT)."""
|
||||
|
||||
clm_pps_ind_cd: str | None = None
|
||||
"""PPS indicator (supportingInfo CLM_PPS_IND_CD)."""
|
||||
|
||||
clm_hha_lup_ind_cd: str | None = None
|
||||
"""HHA LUPA indicator (supportingInfo CLM_HHA_LUP_IND_CD)."""
|
||||
|
||||
clm_hha_rfrl_cd: str | None = None
|
||||
"""HHA referral code (supportingInfo CLM_HHA_RFRL_CD)."""
|
||||
|
||||
clm_clncl_tril_num: str | None = None
|
||||
"""Clinical trial number (supportingInfo CLM_CLNCL_TRIL_NUM)."""
|
||||
|
||||
clm_prcng_excptn_cd: str | None = None
|
||||
"""Pricing exception / benefit payment status reason
|
||||
(adjudication benefitpaymentstatus reason)."""
|
||||
|
||||
# ── v3-only: header totals (ExplanationOfBenefit.total) ──
|
||||
clm_sbmt_chrg_amt: float | None = None
|
||||
"""Total submitted charge (total CLM_SBMT_CHRG_AMT)."""
|
||||
|
||||
clm_alowd_chrg_amt: float | None = None
|
||||
"""Allowed charge (total CLM_ALOWD_CHRG_AMT)."""
|
||||
|
||||
clm_bene_pmt_amt: float | None = None
|
||||
"""Paid by beneficiary (total CLM_BENE_PMT_AMT)."""
|
||||
|
||||
clm_prvdr_pmt_amt: float | None = None
|
||||
"""Paid to provider (total CLM_PRVDR_PMT_AMT)."""
|
||||
|
||||
clm_mdcr_ddctbl_amt: float | None = None
|
||||
"""Deductible (total CLM_MDCR_DDCTBL_AMT)."""
|
||||
|
||||
clm_mdcr_coinsrnc_amt: float | None = None
|
||||
"""Coinsurance liability (total CLM_MDCR_COINSRNC_AMT)."""
|
||||
|
||||
clm_ncvrd_chrg_amt: float | None = None
|
||||
"""Noncovered charge (total CLM_NCVRD_CHRG_AMT)."""
|
||||
|
||||
clm_othr_tp_pd_amt: float | None = None
|
||||
"""Other third party paid (total CLM_OTHR_TP_PD_AMT)."""
|
||||
|
||||
clm_mdcr_instnl_bene_pd_amt: float | None = None
|
||||
"""Paid to patient, institutional (total CLM_MDCR_INSTNL_BENE_PD_AMT)."""
|
||||
|
||||
tot_rx_cst_amt: float | None = None
|
||||
"""Total drug cost, Part D (total TOT_RX_CST_AMT)."""
|
||||
|
||||
# ── v3-only: header adjudication (ExplanationOfBenefit.adjudication) ──
|
||||
clm_instnl_cvrd_day_cnt: float | None = None
|
||||
"""Utilization day count (CLM_INSTNL_CVRD_DAY_CNT)."""
|
||||
|
||||
clm_mdcr_ip_lrd_use_cnt: float | None = None
|
||||
"""Lifetime reserve days used (CLM_MDCR_IP_LRD_USE_CNT)."""
|
||||
|
||||
clm_instnl_mdcr_coins_day_cnt: float | None = None
|
||||
"""Coinsurance day count (CLM_INSTNL_MDCR_COINS_DAY_CNT)."""
|
||||
|
||||
clm_instnl_ncvrd_day_cnt: float | None = None
|
||||
"""Non-utilization day count (CLM_INSTNL_NCVRD_DAY_CNT)."""
|
||||
|
||||
clm_instnl_per_diem_amt: float | None = None
|
||||
"""Pass-through per diem (CLM_INSTNL_PER_DIEM_AMT)."""
|
||||
|
||||
clm_mdcr_ip_pps_drg_wt_num: float | None = None
|
||||
"""PPS capital DRG weight (CLM_MDCR_IP_PPS_DRG_WT_NUM)."""
|
||||
|
||||
clm_mdcr_ip_pps_cptl_fsp_amt: float | None = None
|
||||
"""PPS capital FSP amount (CLM_MDCR_IP_PPS_CPTL_FSP_AMT)."""
|
||||
|
||||
clm_mdcr_ip_pps_outlier_amt: float | None = None
|
||||
"""PPS capital outlier amount (CLM_MDCR_IP_PPS_OUTLIER_AMT)."""
|
||||
|
||||
clm_mdcr_ip_pps_excptn_amt: float | None = None
|
||||
"""PPS capital exception amount (CLM_MDCR_IP_PPS_EXCPTN_AMT)."""
|
||||
|
||||
clm_mdcr_ip_pps_cptl_hrmls_amt: float | None = None
|
||||
"""PPS old capital hold harmless (CLM_MDCR_IP_PPS_CPTL_HRMLS_AMT)."""
|
||||
|
||||
clm_mdcr_ip_pps_cptl_tot_amt: float | None = None
|
||||
"""PPS total capital (CLM_MDCR_IP_PPS_CPTL_TOT_AMT)."""
|
||||
|
||||
clm_instnl_drg_outlier_amt: float | None = None
|
||||
"""DRG outlier approved payment (CLM_INSTNL_DRG_OUTLIER_AMT)."""
|
||||
|
||||
clm_mdcr_instnl_prmry_pyr_amt: float | None = None
|
||||
"""Institutional primary payer paid (CLM_MDCR_INSTNL_PRMRY_PYR_AMT)."""
|
||||
|
||||
clm_mdcr_prfnl_prmry_pyr_amt: float | None = None
|
||||
"""Carrier primary payer paid (CLM_MDCR_PRFNL_PRMRY_PYR_AMT)."""
|
||||
|
||||
clm_instnl_prfnl_amt: float | None = None
|
||||
"""Professional component charge (CLM_INSTNL_PRFNL_AMT)."""
|
||||
|
||||
clm_blood_lblty_amt: float | None = None
|
||||
"""Blood deductible liability (CLM_BLOOD_LBLTY_AMT)."""
|
||||
|
||||
clm_mdcr_ip_bene_ddctbl_amt: float | None = None
|
||||
"""Inpatient deductible (CLM_MDCR_IP_BENE_DDCTBL_AMT)."""
|
||||
|
||||
clm_blood_chrg_amt: float | None = None
|
||||
"""Blood charge (CLM_BLOOD_CHRG_AMT)."""
|
||||
|
||||
clm_blood_ncvrd_chrg_amt: float | None = None
|
||||
"""Blood noncovered charge (CLM_BLOOD_NCVRD_CHRG_AMT)."""
|
||||
|
||||
clm_mdcr_hospc_prd_cnt: float | None = None
|
||||
"""Hospice period count (CLM_MDCR_HOSPC_PRD_CNT)."""
|
||||
|
||||
clm_mdcr_hha_tot_visit_cnt: float | None = None
|
||||
"""HHA total visit count (CLM_MDCR_HHA_TOT_VISIT_CNT)."""
|
||||
|
||||
clm_finl_stdzd_pymt_amt: float | None = None
|
||||
"""Final standardized payment (CLM_FINL_STDZD_PYMT_AMT)."""
|
||||
|
||||
clm_hac_rdctn_pymt_amt: float | None = None
|
||||
"""HAC reduction (CLM_HAC_RDCTN_PYMT_AMT)."""
|
||||
|
||||
clm_hipps_readmsn_rdctn_amt: float | None = None
|
||||
"""Readmission reduction (CLM_HIPPS_READMSN_RDCTN_AMT)."""
|
||||
|
||||
clm_hipps_vbp_amt: float | None = None
|
||||
"""Value-based purchasing (CLM_HIPPS_VBP_AMT)."""
|
||||
|
||||
clm_instnl_low_vol_pmt_amt: float | None = None
|
||||
"""Low volume payment (CLM_INSTNL_LOW_VOL_PMT_AMT)."""
|
||||
|
||||
# ── v3-only: Part D supporting info ──
|
||||
clm_brnd_gnrc_cd: str | None = None
|
||||
"""Brand/generic (supportingInfo brandgenericindicator)."""
|
||||
|
||||
clm_cmpnd_cd: str | None = None
|
||||
"""Compound code (supportingInfo compoundcode)."""
|
||||
|
||||
clm_line_rx_orgn_cd: str | None = None
|
||||
"""Rx origin (supportingInfo rxorigincode)."""
|
||||
|
||||
clm_ptnt_rsdnc_cd: str | None = None
|
||||
"""Patient residence (supportingInfo CLM_PTNT_RSDNC_CD)."""
|
||||
|
||||
clm_ltc_dspnsng_mthd_cd: str | None = None
|
||||
"""LTC dispensing method (supportingInfo CLM_LTC_DSPNSNG_MTHD_CD)."""
|
||||
|
||||
clm_drug_cvrg_stus_cd: str | None = None
|
||||
"""Drug coverage status (supportingInfo CLM_DRUG_CVRG_STUS_CD)."""
|
||||
|
||||
clm_ctstrphc_cvrg_ind_cd: str | None = None
|
||||
"""Catastrophic coverage indicator (supportingInfo CLM_CTSTRPHC_CVRG_IND_CD)."""
|
||||
|
||||
clm_dspnsng_stus_cd: str | None = None
|
||||
"""Dispensing status (supportingInfo CLM_DSPNSNG_STUS_CD)."""
|
||||
|
||||
clm_sbmtr_cntrct_num: str | None = None
|
||||
"""Submitter contract (supportingInfo CLM_SBMTR_CNTRCT_NUM)."""
|
||||
|
||||
clm_sbmtr_cntrct_pbp_num: str | None = None
|
||||
"""Submitter PBP (supportingInfo CLM_SBMTR_CNTRCT_PBP_NUM)."""
|
||||
|
||||
clm_line_authrzd_fill_num: float | None = None
|
||||
"""Authorized refills (supportingInfo refillsauthorized)."""
|
||||
|
||||
|
||||
class EobItem(SQLTable):
|
||||
"""FHIR R4 ExplanationOfBenefit.item — claim line items.
|
||||
@@ -571,3 +816,119 @@ class EobItem(SQLTable):
|
||||
tax_num: str | None = None
|
||||
"""Line-level provider tax number
|
||||
(Eob.item.extension[tax_num] valueCoding)."""
|
||||
|
||||
# ── v3-only (BCDA v3 Data Dictionary, item level) ────────
|
||||
clm_line_ndc_qty: float | None = None
|
||||
"""NDC quantity (item/detail quantity.value; IDR CLM_LINE_NDC_QTY)."""
|
||||
|
||||
clm_line_ndc_qty_qlfyr_cd: str | None = None
|
||||
"""NDC quantity unit (item/detail quantity.unit)."""
|
||||
|
||||
clm_line_dgns_cd: str | None = None
|
||||
"""Line diagnosis — header diagnosis at item.diagnosisSequence[0]."""
|
||||
|
||||
clm_rev_dscnt_ind_cd: str | None = None
|
||||
"""Revenue center discount indicator (extension CLM-REV-DSCNT-IND-CD)."""
|
||||
|
||||
clm_otaf_one_ind_cd: str | None = None
|
||||
"""Obligation to accept as full payment (extension CLM-OTAF-ONE-IND-CD)."""
|
||||
|
||||
clm_rev_packg_ind_cd: str | None = None
|
||||
"""Packaging indicator (extension CLM-REV-PACKG-IND-CD)."""
|
||||
|
||||
clm_rev_pmt_mthd_cd: str | None = None
|
||||
"""Payment method indicator (extension CLM-REV-PMT-MTHD-CD)."""
|
||||
|
||||
clm_line_ansthsa_unit_cnt: float | None = None
|
||||
"""Anesthesia units (extension CLM-LINE-ANSTHSA-UNIT-CNT)."""
|
||||
|
||||
clm_physn_astnt_cd: str | None = None
|
||||
"""Reduced payment physician assistant code (extension CLM-PHYSN-ASTNT-CD)."""
|
||||
|
||||
clm_line_carr_hpsa_scrcty_cd: str | None = None
|
||||
"""HPSA scarcity (extension CLM-LINE-CARR-HPSA-SCRCTY-CD)."""
|
||||
|
||||
clm_rndrg_prvdr_prtcptg_cd: str | None = None
|
||||
"""Rendering provider participating (extension CLM-RNDRG-PRVDR-PRTCPTG-CD)."""
|
||||
|
||||
clm_rndrg_prvdr_type_cd: str | None = None
|
||||
"""Rendering provider type (extension CLM-PRVDR-TYPE-CD)."""
|
||||
|
||||
clm_prvdr_spclty_cd: str | None = None
|
||||
"""Line provider specialty (extension CLM-PRVDR-SPCLTY-CD)."""
|
||||
|
||||
clm_1_rev_cntr_ansi_rsn_cd: str | None = None
|
||||
"""Adjustment reason 1 (adjudication reason ANSI-RSN-CD)."""
|
||||
|
||||
clm_1_rev_cntr_ansi_grp_cd: str | None = None
|
||||
"""Adjustment group 1 (adjudication reason ANSI-GRP-CD)."""
|
||||
|
||||
clm_2_rev_cntr_ansi_rsn_cd: str | None = None
|
||||
"""Adjustment reason 2."""
|
||||
|
||||
clm_2_rev_cntr_ansi_grp_cd: str | None = None
|
||||
"""Adjustment group 2."""
|
||||
|
||||
clm_3_rev_cntr_ansi_rsn_cd: str | None = None
|
||||
"""Adjustment reason 3."""
|
||||
|
||||
clm_3_rev_cntr_ansi_grp_cd: str | None = None
|
||||
"""Adjustment group 3."""
|
||||
|
||||
clm_4_rev_cntr_ansi_rsn_cd: str | None = None
|
||||
"""Adjustment reason 4."""
|
||||
|
||||
clm_4_rev_cntr_ansi_grp_cd: str | None = None
|
||||
"""Adjustment group 4."""
|
||||
|
||||
# ── v3-only: item adjudication amounts (CodeSystem/Adjudication) ──
|
||||
tot_rx_cst_amt: float | None = None
|
||||
"""Total drug cost, Part D (TOT_RX_CST_AMT)."""
|
||||
|
||||
clm_line_grs_above_thrshld_amt: float | None = None
|
||||
"""Gross drug cost above OOP threshold (CLM_LINE_GRS_ABOVE_THRSHLD_AMT)."""
|
||||
|
||||
clm_line_grs_blw_thrshld_amt: float | None = None
|
||||
"""Gross drug cost below OOP threshold (CLM_LINE_GRS_BLW_THRSHLD_AMT)."""
|
||||
|
||||
clm_line_lis_amt: float | None = None
|
||||
"""Low income cost sharing subsidy (CLM_LINE_LIS_AMT)."""
|
||||
|
||||
clm_line_plro_amt: float | None = None
|
||||
"""Patient liability reduction other paid (CLM_LINE_PLRO_AMT)."""
|
||||
|
||||
clm_line_rptd_gap_dscnt_amt: float | None = None
|
||||
"""Reported gap discount (CLM_LINE_RPTD_GAP_DSCNT_AMT)."""
|
||||
|
||||
clm_line_ncvrd_pd_amt: float | None = None
|
||||
"""Noncovered paid (CLM_LINE_NCVRD_PD_AMT)."""
|
||||
|
||||
clm_line_bene_pd_amt: float | None = None
|
||||
"""Paid to beneficiary (CLM_LINE_BENE_PD_AMT)."""
|
||||
|
||||
clm_line_mdcr_ddctbl_amt: float | None = None
|
||||
"""Part B / cash deductible (CLM_LINE_MDCR_DDCTBL_AMT)."""
|
||||
|
||||
clm_line_prfnl_dme_price_amt: float | None = None
|
||||
"""DME purchase price (CLM_LINE_PRFNL_DME_PRICE_AMT)."""
|
||||
|
||||
clm_line_dmerc_scrn_svgs_amt: float | None = None
|
||||
"""DMERC screen savings (CLM_LINE_DMERC_SCRN_SVGS_AMT)."""
|
||||
|
||||
clm_line_add_on_pymt_amt: float | None = None
|
||||
"""Add-on payment (CLM_LINE_ADD_ON_PYMT_AMT)."""
|
||||
|
||||
clm_rev_cntr_tdapa_amt: float | None = None
|
||||
"""Transitional drug add-on payment adjustment (CLM_REV_CNTR_TDAPA_AMT)."""
|
||||
|
||||
clm_line_carr_clncl_chrg_amt: float | None = None
|
||||
"""Carrier clinical charge (CLM_LINE_CARR_CLNCL_CHRG_AMT)."""
|
||||
|
||||
clm_line_carr_psych_ot_lmt_amt: float | None = None
|
||||
"""Therapy amount applied to limit (CLM_LINE_CARR_PSYCH_OT_LMT_AMT)."""
|
||||
|
||||
clm_line_prfnl_intrst_amt: float | None = None
|
||||
"""Professional interest (CLM_LINE_PRFNL_INTRST_AMT)."""
|
||||
|
||||
clm_mdcr_prmry_pyr_alowd_amt: float | None = None
|
||||
"""Line primary payer allowed (CLM_MDCR_PRMRY_PYR_ALOWD_AMT)."""
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
"""FHIR R4 Patient resource — beneficiary demographics and identifiers.
|
||||
|
||||
Flattened from the BCDA v2 Patient ndjson export. One row per
|
||||
Flattened from the BCDA Patient ndjson export (v2 or v3 — the
|
||||
flattener detects the dialect per resource). One row per
|
||||
beneficiary. Maps CCLF8 (demographics) and CCLF9 (XREF) fields
|
||||
to their FHIR R4 element paths.
|
||||
|
||||
Source: ``/api/v2/Patient/$export`` bulk FHIR endpoint.
|
||||
v3 differences (BCDA v3 Data Dictionary, ``ccwMapping`` column):
|
||||
``id`` is the IDR beneficiary surrogate key (BENE_SK), not the CCW
|
||||
BENE_ID; race moves to the US Core extension (OMB codes, crosswalked
|
||||
back into ``bene_race_cd``); the ``rfrnc_yr``/``dual_*`` extensions
|
||||
are gone (dual status lives on its own Coverage row); the contact
|
||||
language and historical source keys are new.
|
||||
|
||||
Source: ``/api/{v2,v3}/Patient/$export`` bulk FHIR endpoint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -166,3 +174,21 @@ class Patient(SQLTable):
|
||||
|
||||
dual_12: str | None = None
|
||||
"""Dual eligibility status — December."""
|
||||
|
||||
# ── v3-only (BCDA v3 Data Dictionary) ────────────────────
|
||||
cntct_lang_cd: str | None = None
|
||||
"""Contact language, BCP-47 (Patient.communication.language;
|
||||
IDR CNTCT_LANG_CD)."""
|
||||
|
||||
xref_efctv_sk: str | None = None
|
||||
"""Current Patient FHIR id after cross-reference — the
|
||||
``replaced-by`` link target when present, else ``id``
|
||||
(IDR BENE_XREF_EFCTV_SK)."""
|
||||
|
||||
bene_sk_list: str | None = None
|
||||
"""Semicolon-joined historical source keys
|
||||
(Patient.link.other.display ∪ Patient.id)."""
|
||||
|
||||
mbi_efctv_dt: date | None = None
|
||||
"""Current MBI effective date (Patient.identifier[us-mbi].period.start;
|
||||
IDR BENE_MBI_EFCTV_DT)."""
|
||||
|
||||
@@ -55,8 +55,10 @@ def bcda(
|
||||
) -> None:
|
||||
"""Flatten BCDA FHIR exports and load into DuckDB.
|
||||
|
||||
Reads Patient/EOB/Coverage ndjson files, flattens them to Parquet,
|
||||
and loads into the ``bcda`` schema.
|
||||
Reads Patient/EOB/Coverage ndjson files (v2 or v3 payloads, any
|
||||
number of parts), flattens them to Parquet, and loads into the
|
||||
``bcda`` schema. Without --ndjson-dir, uses the newest export per
|
||||
resource type under the BCDA store (a v3 pull is three jobs).
|
||||
"""
|
||||
from aco.load.bcda import load_bcda
|
||||
|
||||
|
||||
@@ -490,7 +490,7 @@ class TestLoadBcdaFindLatest:
|
||||
(exports / "job1").mkdir()
|
||||
|
||||
result = _find_latest_export(tmp_path)
|
||||
assert result.name == "job1"
|
||||
assert [d.name for d in result] == ["job1"]
|
||||
|
||||
def test_find_latest_no_exports_dir(self, tmp_path):
|
||||
from aco.load.bcda import _find_latest_export
|
||||
|
||||
@@ -29,4 +29,96 @@ class TestFindLatestExport:
|
||||
# Set old_job mtime to past so new_job is clearly newer
|
||||
os.utime(exports / "old_job", (1000000, 1000000))
|
||||
result = _find_latest_export(tmp_path)
|
||||
assert result.name == "new_job"
|
||||
assert [d.name for d in result] == ["new_job"]
|
||||
|
||||
def test_newest_per_resource_type(self, tmp_path: Path) -> None:
|
||||
"""Three single-type v3 jobs: each type comes from its own dir."""
|
||||
import os
|
||||
|
||||
exports = tmp_path / "exports"
|
||||
exports.mkdir()
|
||||
for i, (name, rtype) in enumerate(
|
||||
[
|
||||
("j_eob", "ExplanationOfBenefit"),
|
||||
("j_pat", "Patient"),
|
||||
("j_cov", "Coverage"),
|
||||
]
|
||||
):
|
||||
d = exports / name
|
||||
d.mkdir()
|
||||
(d / f"{rtype}.ndjson").write_text("{}\n")
|
||||
os.utime(d, (1000000 + i, 1000000 + i))
|
||||
# a newer Patient-only job supersedes j_pat
|
||||
d = exports / "j_pat2"
|
||||
d.mkdir()
|
||||
(d / "Patient.ndjson").write_text("{}\n")
|
||||
os.utime(d, (2000000, 2000000))
|
||||
result = _find_latest_export(tmp_path)
|
||||
assert [d.name for d in result] == ["j_eob", "j_cov", "j_pat2"]
|
||||
|
||||
|
||||
class TestLoadV3EndToEnd:
|
||||
"""Official v3 samples, split like three single-type jobs → DuckDB."""
|
||||
|
||||
def test_load_from_three_export_dirs(self, tmp_path: Path) -> None:
|
||||
import shutil
|
||||
|
||||
import duckdb
|
||||
|
||||
from aco.load.bcda import load_bcda
|
||||
from conf import path as conf_path
|
||||
|
||||
samples = Path(__file__).resolve().parents[2] / "dev" / "vendor" / "bcda-site"
|
||||
store = tmp_path / "store"
|
||||
for job, rtype in (
|
||||
("j1", "Patient"),
|
||||
("j2", "Coverage"),
|
||||
("j3", "ExplanationOfBenefit"),
|
||||
):
|
||||
d = store / "exports" / job
|
||||
d.mkdir(parents=True)
|
||||
shutil.copy(samples / f"v3_{rtype}.ndjson", d / f"{rtype}.ndjson")
|
||||
# a second EOB part in the same job dir
|
||||
shutil.copy(
|
||||
samples / "v3_ExplanationOfBenefit.ndjson",
|
||||
store / "exports" / "j3" / "ExplanationOfBenefit.1.ndjson",
|
||||
)
|
||||
db = tmp_path / "aco.duckdb"
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
real_path = conf_path
|
||||
|
||||
def fake_path(key: str):
|
||||
if key == "storage.bcda":
|
||||
return store
|
||||
return real_path(key)
|
||||
|
||||
with patch("conf.path", side_effect=fake_path):
|
||||
stats = load_bcda(database=str(db))
|
||||
assert stats == {
|
||||
"bcda.patient": 94,
|
||||
"bcda.explanation_of_benefit": 538,
|
||||
"bcda.eob_item": 2464,
|
||||
"bcda.coverage": 175,
|
||||
}
|
||||
con = duckdb.connect(str(db), read_only=True)
|
||||
plans = dict(
|
||||
con.execute(
|
||||
"select class_plan, count(*) from bcda.coverage group by 1"
|
||||
).fetchall()
|
||||
)
|
||||
assert set(plans) == {"Part A", "Part B", "Part C", "Dual Medicare/Medicaid"}
|
||||
sys_types = dict(
|
||||
con.execute(
|
||||
"select system_type, count(*) from bcda.explanation_of_benefit group by 1"
|
||||
).fetchall()
|
||||
)
|
||||
assert sys_types == {"NationalClaimsHistory": 282, "DDPS": 256}
|
||||
assert (
|
||||
con.execute(
|
||||
"select count(*) from bcda.patient where bene_mbi_id is null"
|
||||
).fetchone()[0]
|
||||
== 0
|
||||
)
|
||||
con.close()
|
||||
|
||||
@@ -49,7 +49,7 @@ class TestLoadBcda:
|
||||
from aco.load.bcda import _find_latest_export
|
||||
|
||||
result = _find_latest_export(tmp_path)
|
||||
assert result.is_dir()
|
||||
assert len(result) == 1 and result[0].is_dir()
|
||||
|
||||
|
||||
class TestStageAll:
|
||||
|
||||
@@ -846,7 +846,7 @@ class TestClientDefaults:
|
||||
def test_defaults(self) -> None:
|
||||
c = Client("cid", "csec")
|
||||
assert c.base_url == SANDBOX
|
||||
assert c.version == "v2"
|
||||
assert c.version == "v3"
|
||||
assert c.max_retries == 3
|
||||
from conf import path
|
||||
|
||||
|
||||
388
tests/bcda/test_client_v3.py
Normal file
388
tests/bcda/test_client_v3.py
Normal file
@@ -0,0 +1,388 @@
|
||||
"""bcda.client — v3 additions: typeFilter, OperationOutcome errors,
|
||||
job history Bundle, completed-job reuse, cancellation, version/health."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from bcda.client import (
|
||||
ASYNC_HEADERS,
|
||||
SANDBOX,
|
||||
SYSTEM_TYPES,
|
||||
BcdaHttpError,
|
||||
Client,
|
||||
JobCancelledError,
|
||||
_requested_types,
|
||||
operation_outcome,
|
||||
raise_for_status,
|
||||
type_filter_tag,
|
||||
)
|
||||
|
||||
|
||||
def _token_response(tok: str = "tok") -> httpx.Response:
|
||||
return httpx.Response(200, json={"access_token": tok, "expires_in": 1200})
|
||||
|
||||
|
||||
def _oo(text: str, *, code: int = 400, details: bool = False) -> httpx.Response:
|
||||
issue: dict = {"severity": "error", "code": "structure"}
|
||||
if details:
|
||||
issue["details"] = {"text": text}
|
||||
else:
|
||||
issue["diagnostics"] = text
|
||||
return httpx.Response(
|
||||
code, json={"resourceType": "OperationOutcome", "issue": [issue]}
|
||||
)
|
||||
|
||||
|
||||
def _client(handler, **kw) -> Client:
|
||||
kw.setdefault("output_dir", "/tmp/bcda-test")
|
||||
c = Client("cid", "csec", **kw)
|
||||
c._http = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
return c
|
||||
|
||||
|
||||
class TestHelpers:
|
||||
def test_type_filter_tag(self) -> None:
|
||||
assert type_filter_tag("DDPS") == (
|
||||
"ExplanationOfBenefit?_tag=https://bluebutton.cms.gov/fhir/CodeSystem/System-Type|DDPS"
|
||||
)
|
||||
assert type_filter_tag(*SYSTEM_TYPES).count("|") == 3
|
||||
with pytest.raises(ValueError):
|
||||
type_filter_tag("Bogus")
|
||||
with pytest.raises(ValueError):
|
||||
type_filter_tag()
|
||||
|
||||
def test_operation_outcome(self) -> None:
|
||||
assert (
|
||||
operation_outcome(_oo("Prefer header is required"))
|
||||
== "Prefer header is required"
|
||||
)
|
||||
assert operation_outcome(_oo("via details", details=True)) == "via details"
|
||||
assert operation_outcome(httpx.Response(500, content=b"<html>")) == ""
|
||||
assert (
|
||||
operation_outcome(httpx.Response(404, json={"resourceType": "Bundle"}))
|
||||
== ""
|
||||
)
|
||||
assert operation_outcome(httpx.Response(404, json=[1, 2])) == ""
|
||||
assert (
|
||||
operation_outcome(
|
||||
httpx.Response(
|
||||
400,
|
||||
json={
|
||||
"resourceType": "OperationOutcome",
|
||||
"issue": [{"severity": "error"}],
|
||||
},
|
||||
)
|
||||
)
|
||||
== ""
|
||||
)
|
||||
|
||||
def test_raise_for_status(self) -> None:
|
||||
raise_for_status(httpx.Response(200)) # no-op
|
||||
resp = _oo("bad", code=404)
|
||||
resp.request = httpx.Request("GET", "https://x/y")
|
||||
with pytest.raises(BcdaHttpError) as ei:
|
||||
raise_for_status(resp)
|
||||
assert ei.value.status_code == 404
|
||||
assert ei.value.diagnostics == "bad"
|
||||
assert "bad" in str(ei.value)
|
||||
assert isinstance(ei.value, httpx.HTTPStatusError)
|
||||
plain = httpx.Response(500, content=b"")
|
||||
plain.request = httpx.Request("GET", "https://x/y")
|
||||
with pytest.raises(BcdaHttpError) as ei:
|
||||
raise_for_status(plain)
|
||||
assert ei.value.diagnostics == "" and "500" in str(ei.value)
|
||||
|
||||
def test_requested_types(self) -> None:
|
||||
req = "GET https://sandbox.bcda.cms.gov/api/v3/Group/all/$export?_type=Patient%2CCoverage%2CExplanationOfBenefit"
|
||||
assert _requested_types(req) == ["Patient", "Coverage", "ExplanationOfBenefit"]
|
||||
assert _requested_types("GET https://x/api/v3/Group/all/$export") == []
|
||||
assert _requested_types("https://x/api/v3/Patient/$export?_type=Patient") == [
|
||||
"Patient"
|
||||
]
|
||||
|
||||
|
||||
class TestExportTypeFilter:
|
||||
def test_params_sent(self) -> None:
|
||||
captured: dict = {}
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
captured["url"] = str(req.url)
|
||||
captured["prefer"] = req.headers.get("prefer")
|
||||
return httpx.Response(
|
||||
202, headers={"Content-Location": f"{SANDBOX}/api/v3/jobs/1"}
|
||||
)
|
||||
|
||||
c = _client(handler)
|
||||
url = c.start_export(
|
||||
types=["ExplanationOfBenefit"],
|
||||
since="2026-01-01T00:00:00Z",
|
||||
type_filter=type_filter_tag("SharedSystem"),
|
||||
)
|
||||
assert url.endswith("/api/v3/jobs/1")
|
||||
assert "_typeFilter=ExplanationOfBenefit%3F_tag%3D" in captured["url"]
|
||||
assert "_since=2026-01-01" in captured["url"]
|
||||
assert captured["prefer"] == "respond-async"
|
||||
c.close()
|
||||
|
||||
def test_export_passes_type_filter(self, tmp_path) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
url = str(req.url)
|
||||
calls.append(url)
|
||||
if "/auth/token" in url:
|
||||
return _token_response()
|
||||
if "$export" in url:
|
||||
return httpx.Response(
|
||||
202, headers={"Content-Location": f"{SANDBOX}/api/v3/jobs/9"}
|
||||
)
|
||||
if url.endswith("/jobs/9"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"output": [
|
||||
{"type": "Patient", "url": f"{SANDBOX}/data/9/a.ndjson"}
|
||||
],
|
||||
"error": [],
|
||||
},
|
||||
)
|
||||
return httpx.Response(200, content=b'{"resourceType":"Patient"}\n')
|
||||
|
||||
c = _client(handler, output_dir=tmp_path)
|
||||
paths = c.export(
|
||||
types=["Patient"],
|
||||
type_filter="ExplanationOfBenefit?outcome=partial",
|
||||
poll_interval=0,
|
||||
)
|
||||
assert [p.name for p in paths] == ["Patient.ndjson"]
|
||||
assert any("_typeFilter=" in u for u in calls)
|
||||
c.close()
|
||||
|
||||
|
||||
class TestErrors:
|
||||
def test_client_error_carries_diagnostics(self) -> None:
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
return _oo("application/fhir+json is the only supported response format")
|
||||
|
||||
c = _client(handler)
|
||||
with pytest.raises(BcdaHttpError, match="only supported response format"):
|
||||
c._request("GET", f"{SANDBOX}/api/v3/jobs")
|
||||
c.close()
|
||||
|
||||
def test_auth_failure_diagnostics(self) -> None:
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
return _oo("invalid client", code=401)
|
||||
|
||||
c = _client(handler)
|
||||
with pytest.raises(BcdaHttpError, match="invalid client"):
|
||||
c.authenticate()
|
||||
c.close()
|
||||
|
||||
@patch("time.sleep")
|
||||
def test_5xx_exhausted_is_bcda_error(self, _sleep) -> None:
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
return httpx.Response(503, content=b"down")
|
||||
|
||||
c = _client(handler, max_retries=2)
|
||||
with pytest.raises(BcdaHttpError) as ei:
|
||||
c._request("GET", f"{SANDBOX}/x")
|
||||
assert ei.value.status_code == 503
|
||||
c.close()
|
||||
|
||||
def test_poll_cancelled(self) -> None:
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
return _oo("Job has been cancelled.", code=404)
|
||||
|
||||
c = _client(handler)
|
||||
with pytest.raises(JobCancelledError, match="cancelled"):
|
||||
c.poll(f"{SANDBOX}/api/v3/jobs/5", interval=0)
|
||||
c.close()
|
||||
|
||||
def test_poll_other_404_propagates(self) -> None:
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
return _oo("Job not found", code=404)
|
||||
|
||||
c = _client(handler)
|
||||
with pytest.raises(BcdaHttpError, match="not found"):
|
||||
c.poll(f"{SANDBOX}/api/v3/jobs/5", interval=0)
|
||||
c.close()
|
||||
|
||||
def test_poll_sends_bulk_headers(self) -> None:
|
||||
seen: dict = {}
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
seen.update({k.lower(): v for k, v in req.headers.items()})
|
||||
return httpx.Response(200, json={"output": []})
|
||||
|
||||
c = _client(handler)
|
||||
c.poll(f"{SANDBOX}/api/v3/jobs/5", interval=0)
|
||||
assert (
|
||||
seen["prefer"] == "respond-async"
|
||||
and seen["accept"] == ASYNC_HEADERS["Accept"]
|
||||
)
|
||||
c.close()
|
||||
|
||||
def test_download_error_has_diagnostics(self, tmp_path) -> None:
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
return _oo("file expired", code=410)
|
||||
|
||||
c = _client(handler, output_dir=tmp_path)
|
||||
with pytest.raises(BcdaHttpError, match="file expired"):
|
||||
c._download_file(f"{SANDBOX}/data/1/x.ndjson", tmp_path / "x.ndjson")
|
||||
c.close()
|
||||
|
||||
def test_download_gzip_fixture(self, tmp_path) -> None:
|
||||
"""The upstream gzip fixture decodes to the same bytes as the plain one."""
|
||||
from pathlib import Path
|
||||
|
||||
fixtures = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "dev"
|
||||
/ "vendor"
|
||||
/ "bcda-app"
|
||||
/ "shared_files"
|
||||
/ "gzip_feature_test"
|
||||
)
|
||||
gz = (fixtures / "test_gzip_encoded.ndjson").read_bytes()
|
||||
plain = (fixtures / "test_no_encoding.ndjson").read_bytes()
|
||||
assert gzip.decompress(gz).strip() == plain.strip()
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
return httpx.Response(200, content=gz, headers={"Content-Encoding": "gzip"})
|
||||
|
||||
c = _client(handler, output_dir=tmp_path)
|
||||
c._download_file(f"{SANDBOX}/data/1/x.ndjson", tmp_path / "x.ndjson")
|
||||
assert (tmp_path / "x.ndjson").read_bytes().strip() == plain.strip()
|
||||
c.close()
|
||||
|
||||
|
||||
def _task(job_id: str, status: str, request: str, end: str) -> dict:
|
||||
return {
|
||||
"resourceType": "Task",
|
||||
"identifier": [{"system": f"{SANDBOX}/api/v3/jobs", "value": job_id}],
|
||||
"status": status,
|
||||
"input": [{"type": {"text": "BULK FHIR Export"}, "valueString": request}],
|
||||
"executionPeriod": {"start": "2026-10-25T00:00:00Z", "end": end},
|
||||
}
|
||||
|
||||
|
||||
class TestJobs:
|
||||
def test_bundle_unwrapped_with_headers(self) -> None:
|
||||
seen: dict = {}
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
seen["prefer"] = req.headers.get("prefer")
|
||||
seen["accept"] = req.headers.get("accept")
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"resourceType": "Bundle",
|
||||
"type": "searchset",
|
||||
"entry": [
|
||||
{"resource": _task("1", "completed", "GET x", "2026")},
|
||||
{"fullUrl": "no-resource"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
c = _client(handler)
|
||||
jobs = c.jobs()
|
||||
assert len(jobs) == 1 and jobs[0]["status"] == "completed"
|
||||
assert seen == {"prefer": "respond-async", "accept": "application/fhir+json"}
|
||||
c.close()
|
||||
|
||||
def test_status_404_means_empty(self) -> None:
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
return _oo("no jobs", code=404)
|
||||
|
||||
c = _client(handler)
|
||||
assert c.jobs(status="Completed") == []
|
||||
with pytest.raises(BcdaHttpError):
|
||||
c.jobs() # unfiltered 404 is a real error
|
||||
c.close()
|
||||
|
||||
def test_completed_jobs(self) -> None:
|
||||
tasks = [
|
||||
_task(
|
||||
"10",
|
||||
"completed",
|
||||
f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Patient",
|
||||
"2026-10-25T10:00:00Z",
|
||||
),
|
||||
_task(
|
||||
"11",
|
||||
"completed",
|
||||
f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Coverage%2CPatient",
|
||||
"2026-10-25T12:00:00Z",
|
||||
),
|
||||
_task(
|
||||
"12",
|
||||
"cancelled",
|
||||
f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Patient",
|
||||
"2026-10-25T13:00:00Z",
|
||||
),
|
||||
_task(
|
||||
"13",
|
||||
"completed",
|
||||
f"GET {SANDBOX}/api/v2/Group/all/$export?_type=Patient",
|
||||
"2026-10-25T14:00:00Z",
|
||||
),
|
||||
{"resourceType": "Task", "status": "completed"},
|
||||
]
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return _token_response()
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"resourceType": "Bundle",
|
||||
"entry": [{"resource": t} for t in tasks],
|
||||
},
|
||||
)
|
||||
|
||||
c = _client(handler)
|
||||
got = c.completed_jobs()
|
||||
assert [j["job_id"] for j in got] == ["11", "10"] # newest first, v3 only
|
||||
assert got[0]["types"] == ["Coverage", "Patient"]
|
||||
assert got[0]["job_url"].endswith("/api/v3/jobs/11")
|
||||
c.close()
|
||||
|
||||
|
||||
class TestInfoEndpoints:
|
||||
def test_version_and_health_unauthenticated(self) -> None:
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
assert "Authorization" not in req.headers
|
||||
if str(req.url).endswith("/_version"):
|
||||
return httpx.Response(200, json={"version": "r302"})
|
||||
return httpx.Response(200, json={"database": "ok"})
|
||||
|
||||
c = _client(handler)
|
||||
assert c.server_version()["version"] == "r302"
|
||||
assert c.health()["database"] == "ok"
|
||||
c.close()
|
||||
@@ -2727,3 +2727,81 @@ class TestFlattenExport:
|
||||
assert "bcda.patient" in counts
|
||||
assert "bcda.explanation_of_benefit" not in counts
|
||||
assert "bcda.coverage" not in counts
|
||||
|
||||
|
||||
# ── v3 coverage shape through the CCLF demographics stage ───────
|
||||
|
||||
|
||||
class TestDemographicsV3Coverage:
|
||||
"""v3 Coverage: one row per plan; dual status on its own row."""
|
||||
|
||||
def _frames(self):
|
||||
import polars as pl
|
||||
|
||||
patient = pl.DataFrame(
|
||||
{
|
||||
"bene_mbi_id": ["M1", "M2", "M3"],
|
||||
"bene_sex_cd": ["1", "2", "1"],
|
||||
"bene_race_cd": ["1", "2", None],
|
||||
"bene_dob": [None, None, None],
|
||||
"bene_death_dt": [None, None, None],
|
||||
"bene_1st_name": ["a", "b", "c"],
|
||||
"bene_midl_name": [None, None, None],
|
||||
"bene_last_name": ["x", "y", "z"],
|
||||
"bene_line_1_adr": [None, None, None],
|
||||
"geo_zip_plc_name": [None, None, None],
|
||||
"geo_usps_state_cd": [None, None, None],
|
||||
"geo_zip5_cd": [None, None, None],
|
||||
}
|
||||
)
|
||||
coverage = pl.DataFrame(
|
||||
{
|
||||
"subscriber_id": ["M1", "M1", "M1", "M2", "M2", "M3"],
|
||||
"class_plan": [
|
||||
"Part C",
|
||||
"Part B",
|
||||
"Part A",
|
||||
"Part B",
|
||||
"Dual Medicare/Medicaid",
|
||||
"Part A",
|
||||
],
|
||||
"ms_cd": [None, "10", "20", "10", None, "30"],
|
||||
"orec": [None, "0", "1", "0", None, "2"],
|
||||
"dual_01": [None, None, None, None, None, "04"],
|
||||
"dual_stus_cd": [None, None, None, None, "08", None],
|
||||
"bene_part_a_enrlmt_bgn_dt": [None, None, None, None, None, None],
|
||||
"mdcr_covg_a_end_dt": [None, None, None, None, None, None],
|
||||
}
|
||||
)
|
||||
return patient, coverage
|
||||
|
||||
def test_part_a_preferred_and_dual_joined(self) -> None:
|
||||
from bcda.express.cclf import eligibility, stg_beneficiary_demographics
|
||||
|
||||
patient, coverage = self._frames()
|
||||
demo = stg_beneficiary_demographics(patient, coverage).sort("bene_mbi_id")
|
||||
assert demo["ms_cd"].to_list() == [
|
||||
"20",
|
||||
"10",
|
||||
"30",
|
||||
] # M1 → Part A row, not Part C/B
|
||||
assert demo["dual_stus_cd_dual"].to_list() == [None, "08", None]
|
||||
elig = eligibility(demo).sort("person_id")
|
||||
assert elig["dual_status_code"].to_list() == [None, "08", "04"]
|
||||
assert elig["medicare_status_code"].to_list() == ["20", "10", "30"]
|
||||
assert elig["original_reason_entitlement_code"].to_list() == ["1", "0", "2"]
|
||||
|
||||
def test_v2_frames_without_plan_columns_still_work(self) -> None:
|
||||
import polars as pl
|
||||
|
||||
from bcda.express.cclf import eligibility, stg_beneficiary_demographics
|
||||
|
||||
patient, coverage = self._frames()
|
||||
v2cov = coverage.drop("class_plan", "dual_stus_cd").unique(
|
||||
subset=["subscriber_id"], keep="first"
|
||||
)
|
||||
demo = stg_beneficiary_demographics(patient, v2cov)
|
||||
assert "dual_stus_cd_dual" not in demo.columns
|
||||
elig = eligibility(demo).sort("person_id")
|
||||
assert elig["dual_status_code"].to_list() == [None, None, "04"]
|
||||
assert isinstance(elig, pl.DataFrame)
|
||||
|
||||
148
tests/bcda/test_flatten_stream.py
Normal file
148
tests/bcda/test_flatten_stream.py
Normal file
@@ -0,0 +1,148 @@
|
||||
"""bcda.express.flatten — multi-part resolution and streaming Parquet output."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import fsspec
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from bcda.express import flatten
|
||||
from bcda.table.coverage import Coverage
|
||||
from bcda.table.eob import EobItem, ExplanationOfBenefit
|
||||
from bcda.table.patient import Patient
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SAMPLES = ROOT / "dev" / "vendor" / "bcda-site"
|
||||
|
||||
|
||||
class TestResolveNdjson:
|
||||
def test_parts_sorted_and_later_dirs_win(self, tmp_path: Path) -> None:
|
||||
a, b = tmp_path / "a", tmp_path / "b"
|
||||
a.mkdir()
|
||||
b.mkdir()
|
||||
for name in (
|
||||
"ExplanationOfBenefit.10.ndjson",
|
||||
"ExplanationOfBenefit.ndjson",
|
||||
"ExplanationOfBenefit.2.ndjson",
|
||||
"ExplanationOfBenefitEmpty.ndjson",
|
||||
"Patient.ndjson",
|
||||
):
|
||||
(a / name).write_text("{}\n")
|
||||
(b / "Patient.ndjson").write_text("{}\n")
|
||||
(b / "Coverage.ndjson").write_text("{}\n")
|
||||
found = flatten.resolve_ndjson([a, b])
|
||||
assert [p.name for p in found["ExplanationOfBenefit"]] == [
|
||||
"ExplanationOfBenefit.ndjson",
|
||||
"ExplanationOfBenefit.2.ndjson",
|
||||
"ExplanationOfBenefit.10.ndjson",
|
||||
]
|
||||
assert found["Patient"] == [b / "Patient.ndjson"]
|
||||
assert found["Coverage"] == [b / "Coverage.ndjson"]
|
||||
assert flatten.resolve_ndjson(str(a))["Patient"] == [a / "Patient.ndjson"]
|
||||
assert flatten.resolve_ndjson(tmp_path / "missing") == {}
|
||||
|
||||
|
||||
class TestArrowSchema:
|
||||
def test_types(self) -> None:
|
||||
s = flatten.arrow_schema(EobItem)
|
||||
assert s.field("clm_id").type == pa.string()
|
||||
assert s.field("clm_line_num").type == pa.float64()
|
||||
assert s.field("clm_line_from_dt").type == pa.date32()
|
||||
assert set(s.names) == set(EobItem.column_names())
|
||||
for model in (Patient, Coverage, ExplanationOfBenefit):
|
||||
assert flatten.arrow_schema(model).names == model.column_names()
|
||||
|
||||
def test_unknown_annotation_falls_back_to_string(self) -> None:
|
||||
from conf.table_base import SQLTable
|
||||
|
||||
class Odd(SQLTable):
|
||||
weird: dict | None = None
|
||||
plain: int = 0
|
||||
|
||||
s = flatten.arrow_schema(Odd)
|
||||
assert s.field("weird").type == pa.string()
|
||||
assert s.field("plain").type == pa.int64()
|
||||
|
||||
|
||||
class TestParquetSink:
|
||||
def test_writes_batches_and_skips_empty(self, tmp_path: Path) -> None:
|
||||
fs = fsspec.filesystem("file")
|
||||
path = str(tmp_path / "flat" / "p.parquet")
|
||||
sink = flatten._ParquetSink(fs, path, flatten.arrow_schema(Patient))
|
||||
sink.write([])
|
||||
sink.write([{"id": "1", "bene_dob": date(2000, 1, 1)}])
|
||||
sink.write([{"id": "2"}])
|
||||
sink.close()
|
||||
assert sink.rows == 2
|
||||
t = pq.read_table(path)
|
||||
assert t.num_rows == 2 and t.schema.field("bene_dob").type == pa.date32()
|
||||
assert t.column("id").to_pylist() == ["1", "2"]
|
||||
|
||||
|
||||
class TestFlattenExportStreaming:
|
||||
def test_official_samples_in_small_batches(
|
||||
self, tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
"""Batch size 7 forces many Arrow batches; counts must not change."""
|
||||
import shutil
|
||||
|
||||
monkeypatch.setattr(flatten, "BATCH_ROWS", 7)
|
||||
src = tmp_path / "ndjson"
|
||||
src.mkdir()
|
||||
shutil.copy(SAMPLES / "v3_Patient.ndjson", src / "Patient.ndjson")
|
||||
shutil.copy(SAMPLES / "v3_Coverage.ndjson", src / "Coverage.ndjson")
|
||||
# split the EOB sample into two parts to exercise multi-part reading
|
||||
lines = (SAMPLES / "v3_ExplanationOfBenefit.ndjson").read_text().splitlines()
|
||||
(src / "ExplanationOfBenefit.ndjson").write_text(
|
||||
"\n".join(lines[:100]) + "\n\n"
|
||||
)
|
||||
(src / "ExplanationOfBenefit.1.ndjson").write_text(
|
||||
"\n".join(lines[100:]) + "\n"
|
||||
)
|
||||
counts = flatten.flatten_export(src, tmp_path / "store")
|
||||
assert counts == {
|
||||
"bcda.patient": 94,
|
||||
"bcda.explanation_of_benefit": 269,
|
||||
"bcda.eob_item": 1232,
|
||||
"bcda.coverage": 175,
|
||||
}
|
||||
eob = pq.read_table(
|
||||
tmp_path / "store" / "flat" / "explanation_of_benefit.parquet"
|
||||
)
|
||||
assert eob.num_rows == 269
|
||||
assert set(eob.column("system_type").to_pylist()) == {
|
||||
"NationalClaimsHistory",
|
||||
"DDPS",
|
||||
}
|
||||
assert eob.schema.field("clm_from_dt").type == pa.date32()
|
||||
|
||||
def test_iter_ndjson_skips_blank_lines(self, tmp_path: Path) -> None:
|
||||
p = tmp_path / "x.ndjson"
|
||||
p.write_text('{"a":1}\n\n \n{"a":2}\n')
|
||||
assert [r["a"] for r in flatten._iter_ndjson([p])] == [1, 2]
|
||||
assert flatten._read_ndjson(p) == [{"a": 1}, {"a": 2}]
|
||||
|
||||
def test_missing_types_write_nothing(self, tmp_path: Path) -> None:
|
||||
src = tmp_path / "ndjson"
|
||||
src.mkdir()
|
||||
(src / "Patient.ndjson").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": "P",
|
||||
"meta": {
|
||||
"profile": [
|
||||
"http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Patient|2.2.0"
|
||||
]
|
||||
},
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
counts = flatten.flatten_export(src, tmp_path / "store")
|
||||
assert counts == {"bcda.patient": 1}
|
||||
assert sorted(os.listdir(tmp_path / "store" / "flat")) == ["patient.parquet"]
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pyarrow as pa
|
||||
@@ -656,3 +657,177 @@ class TestStorageOptions:
|
||||
def test_root_trailing_slash(self) -> None:
|
||||
store = Store("data/bcda/")
|
||||
assert store._root == "data/bcda"
|
||||
|
||||
|
||||
# ── v3: adopt completed jobs, type_filter passthrough ───────────
|
||||
|
||||
|
||||
class TestAdoptAndTypeFilter:
|
||||
def _client(self, handler):
|
||||
import httpx
|
||||
|
||||
from bcda.client import Client
|
||||
|
||||
c = Client("cid", "csec", output_dir="/tmp/bcda-store-test")
|
||||
c._http = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
return c
|
||||
|
||||
def test_adopt_downloads_completed_job(self, tmp_path) -> None:
|
||||
import httpx
|
||||
|
||||
from bcda.store import Store
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
url = str(req.url)
|
||||
if "/auth/token" in url:
|
||||
return httpx.Response(
|
||||
200, json={"access_token": "t", "expires_in": 1200}
|
||||
)
|
||||
if url.endswith("/jobs/77"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"output": [
|
||||
{"type": "Patient", "url": "https://s/data/77/p.ndjson"},
|
||||
{"type": "Patient", "url": "https://s/data/77/p2.ndjson"},
|
||||
]
|
||||
},
|
||||
)
|
||||
return httpx.Response(200, content=b'{"resourceType":"Patient"}\n')
|
||||
|
||||
store = Store(path=str(tmp_path))
|
||||
client = self._client(handler)
|
||||
key = store.adopt(
|
||||
client, "https://s/api/v3/jobs/77", types=["Patient"], poll_interval=0
|
||||
)
|
||||
job = store.get_job(key)
|
||||
assert job["status"] == "complete"
|
||||
assert job["job_url"].endswith("/jobs/77")
|
||||
assert job["types"] == "Patient"
|
||||
files = store.get_files(key)
|
||||
assert sorted(Path(f["storage_path"]).name for f in files) == [
|
||||
"Patient.1.ndjson",
|
||||
"Patient.ndjson",
|
||||
]
|
||||
assert all(f["status"] == "complete" for f in files)
|
||||
client.close()
|
||||
|
||||
def test_adopt_failure_recorded(self, tmp_path) -> None:
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from bcda.client import BcdaHttpError
|
||||
from bcda.store import Store
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
if "/auth/token" in str(req.url):
|
||||
return httpx.Response(
|
||||
200, json={"access_token": "t", "expires_in": 1200}
|
||||
)
|
||||
return httpx.Response(
|
||||
404,
|
||||
json={
|
||||
"resourceType": "OperationOutcome",
|
||||
"issue": [{"diagnostics": "Job has expired"}],
|
||||
},
|
||||
)
|
||||
|
||||
store = Store(path=str(tmp_path))
|
||||
client = self._client(handler)
|
||||
with pytest.raises(BcdaHttpError, match="expired"):
|
||||
store.adopt(client, "https://s/api/v3/jobs/78", poll_interval=0)
|
||||
failed = store.list_jobs(status="failed")
|
||||
assert len(failed) == 1 and "expired" in failed[0]["error"]
|
||||
client.close()
|
||||
|
||||
def test_export_records_type_filter(self, tmp_path) -> None:
|
||||
import httpx
|
||||
|
||||
from bcda.store import Store
|
||||
|
||||
seen: dict = {}
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
url = str(req.url)
|
||||
if "/auth/token" in url:
|
||||
return httpx.Response(
|
||||
200, json={"access_token": "t", "expires_in": 1200}
|
||||
)
|
||||
if "$export" in url:
|
||||
seen["url"] = url
|
||||
return httpx.Response(
|
||||
202, headers={"Content-Location": "https://s/api/v3/jobs/1"}
|
||||
)
|
||||
if url.endswith("/jobs/1"):
|
||||
return httpx.Response(200, json={"output": []})
|
||||
return httpx.Response(200, content=b"")
|
||||
|
||||
store = Store(path=str(tmp_path))
|
||||
client = self._client(handler)
|
||||
key = store.export(
|
||||
client,
|
||||
types=["ExplanationOfBenefit"],
|
||||
type_filter="ExplanationOfBenefit?outcome=partial",
|
||||
poll_interval=0,
|
||||
)
|
||||
assert "_typeFilter=" in seen["url"]
|
||||
assert (
|
||||
store.get_job(key)["types"]
|
||||
== "ExplanationOfBenefit|_typeFilter=ExplanationOfBenefit?outcome=partial"
|
||||
)
|
||||
client.close()
|
||||
|
||||
def test_export_retries_after_server_cancellation(self, tmp_path) -> None:
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from bcda.client import JobCancelledError
|
||||
from bcda.store import Store
|
||||
|
||||
polls = {"n": 0}
|
||||
|
||||
def handler(req: httpx.Request) -> httpx.Response:
|
||||
url = str(req.url)
|
||||
if "/auth/token" in url:
|
||||
return httpx.Response(
|
||||
200, json={"access_token": "t", "expires_in": 1200}
|
||||
)
|
||||
if "$export" in url:
|
||||
polls["n"] += 1
|
||||
return httpx.Response(
|
||||
202,
|
||||
headers={"Content-Location": f"https://s/api/v3/jobs/{polls['n']}"},
|
||||
)
|
||||
if url.endswith("/jobs/1") or url.endswith("/jobs/2"):
|
||||
return httpx.Response(
|
||||
404,
|
||||
json={
|
||||
"resourceType": "OperationOutcome",
|
||||
"issue": [{"diagnostics": "Job has been cancelled."}],
|
||||
},
|
||||
)
|
||||
if url.endswith("/jobs/3"):
|
||||
return httpx.Response(200, json={"output": []})
|
||||
return httpx.Response(200, content=b"")
|
||||
|
||||
store = Store(path=str(tmp_path))
|
||||
client = self._client(handler)
|
||||
key = store.export(client, types=["Patient"], poll_interval=0, cancel_retries=2)
|
||||
job = store.get_job(key)
|
||||
assert job["status"] == "complete" and job["job_url"].endswith("/jobs/3")
|
||||
history = store._read_parquet(
|
||||
store._jobs_path, store._read_parquet.__globals__["JOBS_SCHEMA"]
|
||||
)
|
||||
statuses = [r["status"] for r in history.to_pylist() if r["key"] == key]
|
||||
assert statuses.count("cancelled") == 2 and statuses[-1] == "complete"
|
||||
client.close()
|
||||
|
||||
# no retries left → the cancellation propagates and is recorded
|
||||
polls["n"] = 0
|
||||
client = self._client(handler)
|
||||
with pytest.raises(JobCancelledError):
|
||||
store.export(client, types=["Patient"], poll_interval=0)
|
||||
assert store.list_jobs(status="failed")[-1]["error"].startswith(
|
||||
"https://s/api/v3/jobs/1"
|
||||
)
|
||||
client.close()
|
||||
|
||||
1321
tests/bcda/test_v3.py
Normal file
1321
tests/bcda/test_v3.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1060,8 +1060,9 @@ class TestAcoLoadBcdaFindLatest:
|
||||
d1.mkdir()
|
||||
d2 = exports_dir / "export-002"
|
||||
d2.mkdir()
|
||||
(d2 / "Patient.ndjson").write_text("{}\n")
|
||||
result = _find_latest_export(tmp_path)
|
||||
assert result.is_dir() # line 51
|
||||
assert result == [d2] # only the dir that holds a resource file
|
||||
|
||||
|
||||
class TestAcoLoadSeedParquet:
|
||||
|
||||
Reference in New Issue
Block a user