From d08401dc53a11e0572d3a17c0df722fee3d69ad8 Mon Sep 17 00:00:00 2001 From: kert Date: Sun, 25 Oct 2026 13:08:13 -0400 Subject: [PATCH] =?UTF-8?q?feat(bcda):=20BCDA=20v3=20=E2=80=94=20client,?= =?UTF-8?q?=20store=20adoption=20and=20cancel=20retries,=20dictionary-driv?= =?UTF-8?q?en=20v3=20flatteners,=20streaming=20multi-part=20load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Client defaults to /api/v3; _typeFilter on start_export/export with type_filter_tag() for the System-Type tag; jobs() sends the bulk-data headers BCDA requires (Prefer: respond-async, Accept: application/fhir+json — the old call 400'd on v2 and v3) and unwraps the Task Bundle; completed_jobs() lists reusable exports; every non-2xx raises BcdaHttpError carrying the OperationOutcome diagnostics (still an httpx.HTTPStatusError); a poll that hits 404 'Job has been cancelled' raises JobCancelledError; /_version and /_health. Store.adopt(client, job_url) records and downloads a job this store did not start (completed jobs stay downloadable 24 h and the public sandbox entity is shared); export(cancel_retries=N) restarts after server-side cancellation, recording each 'cancelled' transition. bcda.express.v3 flattens C4BB 2.2.0 resources into the existing bcda.* tables, table-driven from the v3 Data Dictionary: v3 moved every BlueButton code off resources/variables/ onto fhir/StructureDefinition|CodeSystem/ and consolidated amounts into adjudication/total/item.adjudication, so the v2 suffix match returned NULL for almost everything. Dispatch is per resource (is_v3); v2 payloads are unchanged. Identity follows the migration guide: bene_id = Patient.id (BENE_SK), current MBI = us-mbi identifier without period.end, clm_id = C4BB 'uc', clm_cntl_num pairs SharedSystem with NCH. Race crosswalked from US Core OMB codes; Coverage is one row per plan+period (Part A/B/C/D + Dual) with v3-native columns; EOB gains System-Type / Final-Action / EOB-TYPE, IDR dates, header totals and PPS/IME/DSH amounts; items gain NDC quantity, revenue-centre indicators, ANSI codes, Part D/DME amounts. flatten_export streams .N.ndjson parts to Parquet with model-derived Arrow schemas (a Large-entity EOB export is 192 files / 2.2 GB); load_bcda() gathers the newest export dir per resource type (a v3 pull is three single-type jobs). cclf.stg_beneficiary_demographics prefers the Part A row and joins the Dual row; eligibility reads coalesce(dual_01, dual_stus_cd). bcda package at 100% line coverage; tests run on CMS's official v3 samples. refs #1139 --- docs/docs/cli/load-bcda.md | 6 +- src/aco/load/bcda.py | 30 +- src/bcda/__init__.py | 44 +- src/bcda/client.py | 220 ++++- src/bcda/express/cclf.py | 47 +- src/bcda/express/flatten.py | 241 ++++-- src/bcda/express/v3.py | 1212 ++++++++++++++++++++++++++ src/bcda/pipe/cclf.py | 2 +- src/bcda/store.py | 127 ++- src/bcda/table/coverage.py | 82 +- src/bcda/table/eob.py | 365 +++++++- src/bcda/table/patient.py | 30 +- src/cli/load.py | 6 +- tests/aco/test_coverage_gaps.py | 2 +- tests/aco/test_load_bcda.py | 94 +- tests/aco/test_load_modules.py | 2 +- tests/bcda/test_client.py | 2 +- tests/bcda/test_client_v3.py | 388 +++++++++ tests/bcda/test_express.py | 78 ++ tests/bcda/test_flatten_stream.py | 148 ++++ tests/bcda/test_store.py | 175 ++++ tests/bcda/test_v3.py | 1321 +++++++++++++++++++++++++++++ tests/test_final_coverage_gaps.py | 3 +- 23 files changed, 4478 insertions(+), 147 deletions(-) create mode 100644 src/bcda/express/v3.py create mode 100644 tests/bcda/test_client_v3.py create mode 100644 tests/bcda/test_flatten_stream.py create mode 100644 tests/bcda/test_v3.py diff --git a/docs/docs/cli/load-bcda.md b/docs/docs/cli/load-bcda.md index 8500f28..60a19e9 100644 --- a/docs/docs/cli/load-bcda.md +++ b/docs/docs/cli/load-bcda.md @@ -10,8 +10,10 @@ Usage: stack load bcda [OPTIONS] Flatten BCDA FHIR exports and load into DuckDB. - Reads Patient/EOB/Coverage ndjson files, flattens them to Parquet, - and loads into the ``bcda`` schema. + Reads Patient/EOB/Coverage ndjson files (v2 or v3 payloads, any + number of parts), flattens them to Parquet, and loads into the + ``bcda`` schema. Without --ndjson-dir, uses the newest export per + resource type under the BCDA store (a v3 pull is three jobs). ╭─ Options ────────────────────────────────────────────────────────────────────╮ │ --ndjson-dir PATH Directory containing FHIR ndjson files. │ diff --git a/src/aco/load/bcda.py b/src/aco/load/bcda.py index a434a01..f76a877 100644 --- a/src/aco/load/bcda.py +++ b/src/aco/load/bcda.py @@ -28,8 +28,11 @@ def load_bcda( Parameters ---------- ndjson_dir : str or Path, optional - Directory containing raw ndjson files. Defaults to the latest - export directory under ``conf.path("storage.bcda")``. + Directory containing raw ndjson files. Defaults to the newest + ``exports//`` directory per resource type under + ``conf.path("storage.bcda")`` — a v3 pull is usually three + single-type jobs, so Patient, Coverage and EOB may each come + from a different export directory. database : str, optional DuckDB database path. Defaults to ``conf.path("db.aco")``. skip_flatten : bool @@ -47,7 +50,7 @@ def load_bcda( db_path = database or str(path("db.aco")) store_path = path("storage.bcda") - if ndjson_dir is None: # pragma: no cover — tests always provide explicit dir + if ndjson_dir is None: ndjson_dir = _find_latest_export(store_path) if not skip_flatten: @@ -89,20 +92,31 @@ def load_bcda( return stats -def _find_latest_export(store_path: Path) -> Path: - """Find the most recent export directory under the BCDA store.""" +def _find_latest_export(store_path: Path) -> list[Path]: + """Export directories holding the newest file of each resource type. + + Oldest first, so :func:`bcda.express.flatten.resolve_ndjson` + (later directories win) picks the newest ``Patient.ndjson``, + ``Coverage.ndjson`` and ``ExplanationOfBenefit.ndjson`` even when + they came from three separate single-type jobs. + """ exports_dir = store_path / "exports" if not exports_dir.exists(): raise FileNotFoundError( f"No exports directory at {exports_dir}. " "Run a BCDA export first or specify --ndjson-dir." ) - # Export directories are named by job ID; pick the most recent + # Export directories are named by job key; order by mtime. dirs = sorted( [d for d in exports_dir.iterdir() if d.is_dir()], key=lambda d: d.stat().st_mtime, - reverse=True, ) if not dirs: raise FileNotFoundError(f"No export directories in {exports_dir}") - return dirs[0] + newest: dict[str, Path] = {} + for d in dirs: + for rtype in ("Patient", "ExplanationOfBenefit", "Coverage"): + if (d / f"{rtype}.ndjson").exists(): + newest[rtype] = d + chosen = sorted(set(newest.values()), key=lambda d: d.stat().st_mtime) + return chosen or [dirs[-1]] diff --git a/src/bcda/__init__.py b/src/bcda/__init__.py index 521469f..5ed3752 100644 --- a/src/bcda/__init__.py +++ b/src/bcda/__init__.py @@ -20,8 +20,11 @@ Submodules ---------- ``bcda.client`` - HTTP client for the BCDA v2 bulk FHIR export API. - Handles auth, job management, polling, and ndjson download. + HTTP client for the BCDA v3 bulk FHIR export API (``version="v2"`` + still works for the legacy REACH sandbox entities). Handles auth, + ``_typeFilter``, job management, polling, cancellation detection + and ndjson download; non-2xx responses carry the server's + OperationOutcome diagnostics. ``bcda.store`` Persistent storage and state management. Wraps the client @@ -35,9 +38,15 @@ Submodules append-only ``.jsonl`` file. ``bcda.express`` - Narwhals expression functions for transforming BCDA FHIR R4 - resources into input_layer tables (medical_claim, - pharmacy_claim, eligibility). + ``flatten`` turns the raw ndjson into the ``bcda.*`` tables, + detecting the payload dialect per resource: the v2 + (``resources/variables``) flattener in ``flatten`` itself, the v3 + (C4BB 2.2.0, ``fhir/StructureDefinition``) flattener in + ``bcda.express.v3`` — table-driven from the BCDA v3 Data + Dictionary (``dev/vendor/bcda-site``, ``dev/seeds/bcda_v3_*.csv``). + ``cclf`` holds the Narwhals expression functions that transform + those tables into input_layer (medical_claim, pharmacy_claim, + eligibility). ``bcda.pipe`` Pipeline definitions wiring express functions into executable @@ -52,16 +61,18 @@ Submodules Adjudicated Claims — FHIR R4 Resources ---------------------------------------- -Delivered via ``/api/v2/{Resource}/$export`` bulk endpoints: +Delivered via ``/api/{v2,v3}/{Resource}/$export`` bulk endpoints +(v3 since 2026-07-01; v1/v2 are discontinued on 2027-07-30): :: - | Table | Fields | FHIR R4 Resource | - |-------------------------|--------|------------------------| - | patient | 26 | Patient | - | explanation_of_benefit | 79 | ExplanationOfBenefit | - | eob_item | 23 | ExplanationOfBenefit | - | coverage | 10 | Coverage | + | Table | FHIR R4 Resource | v3 additions | + |-------------------------|------------------------|----------------------------------| + | patient | Patient | contact language, source keys | + | explanation_of_benefit | ExplanationOfBenefit | System-Type/Final-Action tags, | + | | | claim control number, IDR totals | + | eob_item | ExplanationOfBenefit | consolidated line adjudication | + | coverage | Coverage | one row per plan + period, Dual | ``ExplanationOfBenefit`` is split into header-level fields (``explanation_of_benefit``) and repeating line items @@ -70,9 +81,12 @@ Delivered via ``/api/v2/{Resource}/$export`` bulk endpoints: Partially Adjudicated Claims ----------------------------- -Available via BCDA v2 only. Updated daily (2-4 day lag). -Sourced from FISS (institutional) and MCS (professional) -processing systems, mapped to FHIR R4 Claim and ClaimResponse: +In v3 these arrive as ``ExplanationOfBenefit`` tagged +``System-Type|SharedSystem`` (request them with ``_typeFilter``, see +``bcda.client.type_filter_tag``); the tables below are the v2 +``Claim``/``ClaimResponse`` shape, kept for v2 exports. Updated +daily (2-4 day lag). Sourced from FISS (institutional) and MCS +(professional) processing systems: :: diff --git a/src/bcda/client.py b/src/bcda/client.py index 4292380..b0f0220 100644 --- a/src/bcda/client.py +++ b/src/bcda/client.py @@ -1,15 +1,23 @@ """BCDA bulk FHIR export client. -Wraps the CMS Beneficiary Claims Data API v2 four-step flow: +Wraps the CMS Beneficiary Claims Data API v3 four-step flow: 1. Authenticate (POST ``/auth/token`` with client credentials) -2. Start export job (GET ``/api/v2/Group/all/$export``) -3. Poll until complete (GET ``/api/v2/jobs/{job_id}``) +2. Start export job (GET ``/api/v3/Group/all/$export``) +3. Poll until complete (GET ``/api/v3/jobs/{job_id}``) 4. Download ndjson files (GET ``/data/{job_id}/{filename}``) +Only the ``/api//`` segment differs between v2 and v3 — +``/auth`` and ``/data`` are unchanged (migration guide) — so +``version="v2"`` still works for the legacy REACH sandbox entities. +v3 adds ``_typeFilter`` (FHIR subquery, e.g. the ``System-Type`` +``_tag`` that selects SharedSystem / NationalClaimsHistory / DDPS +claims); without it BCDA returns NationalClaimsHistory + DDPS only. + Handles token refresh (20-min expiry), exponential backoff on transient errors, 429 rate-limit back-off, and gzip-compressed -ndjson downloads. +ndjson downloads. Every non-2xx response surfaces the server's +``OperationOutcome`` diagnostics (:class:`BcdaHttpError`). Usage:: @@ -37,6 +45,18 @@ log = logging.getLogger(__name__) SANDBOX = "https://sandbox.bcda.cms.gov" PRODUCTION = "https://api.bcda.cms.gov" +DEFAULT_VERSION = "v3" + +# FHIR Bulk Data headers BCDA requires on $export, job status and +# job history (both v2 and v3 reject /jobs without ``Prefer``). +FHIR_JSON = "application/fhir+json" +ASYNC_HEADERS = {"Accept": FHIR_JSON, "Prefer": "respond-async"} + +# ``_typeFilter`` building blocks (migration guide, "Adjudication +# filtering"). ``type_filter_tag(*codes)`` renders the subquery; the +# client URL-encodes it. +SYSTEM_TYPE = "https://bluebutton.cms.gov/fhir/CodeSystem/System-Type" +SYSTEM_TYPES = ("SharedSystem", "NationalClaimsHistory", "DDPS") # Defaults matching the Go reference implementation. MAX_RETRIES = 3 @@ -52,6 +72,80 @@ class JobExpiredError(BcdaError): """Job has been archived (server returned 410 Gone).""" +class JobCancelledError(BcdaError): + """Job status returned ``404 Job has been cancelled``. + + Seen on the public sandbox, whose credentials are shared: any + caller can ``DELETE`` any job of the same model entity. + """ + + +class BcdaHttpError(BcdaError, httpx.HTTPStatusError): + """A non-2xx response, carrying the ``OperationOutcome`` text. + + Still an :class:`httpx.HTTPStatusError` so existing ``except`` + clauses keep working; ``diagnostics`` holds the joined + ``issue[].diagnostics`` strings (empty when the body was not an + OperationOutcome). + """ + + def __init__(self, response: httpx.Response, diagnostics: str) -> None: + self.diagnostics = diagnostics + self.status_code = response.status_code + msg = f"{response.status_code} {response.reason_phrase} for {response.url}" + if diagnostics: + msg = f"{msg}: {diagnostics}" + httpx.HTTPStatusError.__init__( + self, msg, request=response.request, response=response + ) + + +def operation_outcome(resp: httpx.Response) -> str: + """Join the ``diagnostics`` of an OperationOutcome body, else ``""``.""" + try: + body = resp.json() + except ValueError: + return "" + if not isinstance(body, dict) or body.get("resourceType") != "OperationOutcome": + return "" + parts = [] + for issue in body.get("issue", []): + text = issue.get("diagnostics") or (issue.get("details") or {}).get("text") + if text: + parts.append(str(text)) + return "; ".join(parts) + + +def raise_for_status(resp: httpx.Response) -> None: + """``resp.raise_for_status()`` that keeps the server's diagnostics.""" + if resp.is_success: + return + raise BcdaHttpError(resp, operation_outcome(resp)) + + +def _requested_types(request: str) -> list[str]: + """``_type`` values from a job's originating request line.""" + from urllib.parse import parse_qs, unquote, urlsplit + + url = request.split(" ", 1)[-1] + query = parse_qs(urlsplit(url).query) + raw = query.get("_type", [""])[0] + return [t for t in unquote(raw).split(",") if t] + + +def type_filter_tag(*codes: str) -> str: + """Render a ``_typeFilter`` subquery selecting EOBs by System-Type. + + ``type_filter_tag("SharedSystem")`` → the v2 Claim/ClaimResponse + equivalent; ``type_filter_tag(*SYSTEM_TYPES)`` → everything. + """ + unknown = [c for c in codes if c not in SYSTEM_TYPES] + if unknown or not codes: + raise ValueError(f"System-Type codes must be among {SYSTEM_TYPES}: {codes}") + tags = ",".join(f"{SYSTEM_TYPE}|{c}" for c in codes) + return f"ExplanationOfBenefit?_tag={tags}" + + class Client: """BCDA bulk FHIR export client. @@ -67,7 +161,8 @@ class Client: Directory for downloaded ndjson files. Defaults to ``data/bcda``. version : str - API version path segment. Defaults to ``v2``. + API version path segment. Defaults to ``v3``; ``"v2"`` for + the legacy REACH sandbox entities. max_retries : int Max retry attempts for transient errors. retry_interval : float @@ -83,7 +178,7 @@ class Client: *, base_url: str = SANDBOX, output_dir: str | Path | None = None, - version: str = "v2", + version: str = DEFAULT_VERSION, max_retries: int = MAX_RETRIES, retry_interval: float = RETRY_INITIAL_INTERVAL, timeout: float = 120.0, @@ -133,7 +228,7 @@ class Client: headers={"Accept": "application/json"}, content=b"", ) - resp.raise_for_status() + raise_for_status(resp) data = resp.json() self._token = data["access_token"] expires_in = int(data.get("expires_in", TOKEN_LIFETIME)) @@ -269,7 +364,7 @@ class Client: time.sleep(backoff) backoff *= 2 continue - resp.raise_for_status() + raise_for_status(resp) # 410 — job expired / archived. if resp.status_code == 410: @@ -278,8 +373,9 @@ class Client: "Jobs expire 24 hours after completion." ) - # Any other client error — fail immediately. - resp.raise_for_status() + # Any other client error — fail immediately, with the + # server's OperationOutcome diagnostics in the message. + raise_for_status(resp) return resp # Should not reach here, but satisfy the type checker. @@ -295,6 +391,7 @@ class Client: endpoint: str = "Group/all", types: list[str] | None = None, since: str | None = None, + type_filter: str | None = None, poll_interval: float = 10.0, ) -> list[Path]: """Run a full bulk export: start, poll, download. @@ -311,6 +408,9 @@ class Client: since : str | None FHIR instant — only return resources updated after this timestamp. Format: ``YYYY-MM-DDThh:mm:ss±zz:zz``. + type_filter : str | None + v3 ``_typeFilter`` subquery (not URL-encoded), e.g. + :func:`type_filter_tag`. poll_interval : float Seconds between status polls. Default 10. @@ -323,6 +423,7 @@ class Client: endpoint=endpoint, types=types, since=since, + type_filter=type_filter, ) result = self.poll(job_url, interval=poll_interval) return self.download(result) @@ -333,6 +434,7 @@ class Client: endpoint: str = "Group/all", types: list[str] | None = None, since: str | None = None, + type_filter: str | None = None, ) -> str: """Start a bulk export job. @@ -347,14 +449,13 @@ class Client: params["_type"] = ",".join(types) if since: params["_since"] = since + if type_filter: + params["_typeFilter"] = type_filter resp = self._request( "GET", url, - headers={ - "Accept": "application/fhir+json", - "Prefer": "respond-async", - }, + headers=dict(ASYNC_HEADERS), params=params, ) @@ -392,14 +493,22 @@ class Client: ------ JobExpiredError If the job has been archived (410 Gone). + JobCancelledError + If the server reports the job cancelled (404 with + ``Job has been cancelled`` diagnostics). BcdaError If timeout is exceeded. """ deadline = (time.monotonic() + timeout) if timeout > 0 else 0 - headers = {"Accept": "application/fhir+json"} + headers = dict(ASYNC_HEADERS) while True: - resp = self._request("GET", job_url, headers=headers) + try: + resp = self._request("GET", job_url, headers=headers) + except BcdaHttpError as exc: + if exc.status_code == 404 and "cancel" in exc.diagnostics.lower(): + raise JobCancelledError(f"{job_url}: {exc.diagnostics}") from exc + raise if resp.status_code == 200: result = resp.json() @@ -423,7 +532,7 @@ class Client: continue # Shouldn't get here — _request handles other status codes. - resp.raise_for_status() + raise_for_status(resp) def download( self, @@ -497,7 +606,9 @@ class Client: "Accept-Encoding": "gzip", } with self._http.stream("GET", url, headers=headers) as resp: - resp.raise_for_status() + if not resp.is_success: + resp.read() + raise_for_status(resp) content_encoding = resp.headers.get("Content-Encoding", "") raw_bytes = b"".join(resp.iter_bytes()) @@ -539,14 +650,67 @@ class Client: Returns ------- list[dict] - Job history entries. + Job history entries. v3 (and v2) answer with a FHIR + ``Bundle`` of ``Task`` resources — those resources are + returned; a bare list (v1) is passed through. Each Task + carries ``identifier[0].value`` (job id), ``status``, + ``input[0].valueString`` (the originating request) and + ``executionPeriod``. """ url = f"{self.base_url}/api/{self.version}/jobs" params: dict[str, str] = {} if status: params["_status"] = status - resp = self._request("GET", url, params=params) - return resp.json() + # BCDA rejects /jobs without the bulk-data headers + # ("Prefer header is required" / "application/fhir+json is + # the only supported response format"). + try: + resp = self._request("GET", url, headers=dict(ASYNC_HEADERS), params=params) + except BcdaHttpError as exc: + # bcda-app answers 404 when the filter matches nothing + # (api_test.go TestJobsStatusNotFoundWithStatus). + if exc.status_code == 404 and status: + return [] + raise + body = resp.json() + if isinstance(body, dict) and body.get("resourceType") == "Bundle": + return [e["resource"] for e in body.get("entry", []) if "resource" in e] + return body + + def completed_jobs(self) -> list[dict]: + """Completed jobs, newest first, as ``(job_id, request, end)`` dicts. + + Convenience over :meth:`jobs` for "reuse the last export" + logic: ``job_id`` is the numeric id, ``job_url`` the status + URL for :meth:`poll`, ``request`` the originating ``$export`` + line, ``types`` the ``_type`` list it asked for, ``end`` the + completion instant. Only jobs started on this client's API + version are returned, since ``/data`` URLs from a different + version serve different payloads. Output files stay + downloadable for 24 hours after completion (then 410/404). + """ + out = [] + # Filter client-side: the sandbox's ``_status=Completed`` answered + # 404 for jobs the unfiltered Bundle lists as ``completed``. + for task in self.jobs(): + if str(task.get("status", "")).lower() != "completed": + continue + ident = (task.get("identifier") or [{}])[0] + job_id = ident.get("value") + request = ((task.get("input") or [{}])[0]).get("valueString", "") + if not job_id or f"/api/{self.version}/" not in request: + continue + out.append( + { + "job_id": job_id, + "job_url": f"{self.base_url}/api/{self.version}/jobs/{job_id}", + "request": request, + "types": _requested_types(request), + "end": (task.get("executionPeriod") or {}).get("end", ""), + } + ) + out.sort(key=lambda j: j["end"], reverse=True) + return out def attribution_status(self) -> dict: """Get last ingestion timestamps for attribution data. @@ -572,6 +736,20 @@ class Client: resp = self._request("GET", url, auth_required=False, retry=False) return resp.json() + def server_version(self) -> dict: + """GET ``/_version`` — the deployed bcda-app release (no auth).""" + resp = self._request( + "GET", f"{self.base_url}/_version", auth_required=False, retry=False + ) + return resp.json() + + def health(self) -> dict: + """GET ``/_health`` — database liveness (no auth).""" + resp = self._request( + "GET", f"{self.base_url}/_health", auth_required=False, retry=False + ) + return resp.json() + # ── Parsing helpers ────────────────────────────────────── @staticmethod diff --git a/src/bcda/express/cclf.py b/src/bcda/express/cclf.py index 2b844b1..82b85a8 100644 --- a/src/bcda/express/cclf.py +++ b/src/bcda/express/cclf.py @@ -552,21 +552,44 @@ def stg_beneficiary_demographics( Patient provides: name, dob, sex, race, death_dt, address Coverage provides: ms_cd, orec, enrollment periods - Join on bene_mbi_id = subscriber_id. Coverage may have - multiple rows per beneficiary (Part A, B, D); we keep - the first match after dedup. + Join on bene_mbi_id = subscriber_id. Coverage has multiple rows + per beneficiary (Part A, B, D in v2; Part A, B, C, D and a Dual + Medicare/Medicaid row per period in v3). The Part A row is the + demographics row — it carries ms_cd / orec / entitlement — with + Part B as the fallback; the v3 Dual row is joined separately for + ``dual_stus_cd`` (v2 keeps ``dual_01`` on the Part A row). - Operations: join, unique + Operations: filter, sort, unique, join Dependencies: bcda.patient, bcda.coverage """ - cov = bcda__coverage.unique(subset=["subscriber_id"], keep="any") - return bcda__patient.join( - cov, + cols = bcda__coverage.columns + parts = bcda__coverage + if "class_plan" in cols: + parts = parts.filter(nw.col("class_plan").is_in(["Part A", "Part B"])).sort( + "class_plan" # "Part A" < "Part B" + ) + parts = parts.unique(subset=["subscriber_id"], keep="first", maintain_order=True) + joined = bcda__patient.join( + parts, left_on="bene_mbi_id", right_on="subscriber_id", how="left", suffix="_cov", - ).unique(subset=["bene_mbi_id"], keep="any") + ) + if "dual_stus_cd" in cols and "class_plan" in cols: + dual = ( + bcda__coverage.filter(nw.col("class_plan") == "Dual Medicare/Medicaid") + .select("subscriber_id", nw.col("dual_stus_cd").alias("dual_stus_cd_dual")) + .unique(subset=["subscriber_id"], keep="any") + ) + joined = joined.join( + dual, + left_on="bene_mbi_id", + right_on="subscriber_id", + how="left", + suffix="_dual", + ) + return joined.unique(subset=["bene_mbi_id"], keep="any") @nw.narwhalify @@ -584,6 +607,12 @@ def eligibility( """ df = bcda___stg_beneficiary_demographics null_str = nw.lit(None).cast(nw.String) + # v2 keeps dual status on the Part A row (dual_01); v3 carries it on + # the Dual Medicare/Medicaid row that stg_beneficiary_demographics + # joins in as dual_stus_cd_dual. + dual = nw.col("dual_01") + if "dual_stus_cd_dual" in df.columns: + dual = nw.coalesce(nw.col("dual_01"), nw.col("dual_stus_cd_dual")) gender = ( nw.when(nw.col("bene_sex_cd") == "1") @@ -641,7 +670,7 @@ def eligibility( nw.lit("medicare").alias("payer_type"), nw.lit("medicare").alias("plan"), nw.col("orec").alias("original_reason_entitlement_code"), - nw.col("dual_01").alias("dual_status_code"), + dual.alias("dual_status_code"), nw.col("ms_cd").alias("medicare_status_code"), null_str.alias("group_id"), null_str.alias("group_name"), diff --git a/src/bcda/express/flatten.py b/src/bcda/express/flatten.py index 05152b1..9fe9a1a 100644 --- a/src/bcda/express/flatten.py +++ b/src/bcda/express/flatten.py @@ -35,6 +35,7 @@ from pathlib import Path import pyarrow as pa import pyarrow.parquet as pq +from bcda.express import v3 from bcda.table.coverage import Coverage from bcda.table.eob import EobItem, ExplanationOfBenefit from bcda.table.patient import Patient @@ -228,7 +229,13 @@ def _coding_code( def flatten_patient(resource: dict) -> Patient: - """Flatten a raw FHIR R4 Patient resource into a Patient model.""" + """Flatten a raw FHIR R4 Patient resource into a Patient model. + + v3 (C4BB 2.2.0) resources are routed to :mod:`bcda.express.v3`; + the body below is the v2 (``resources/variables``) dialect. + """ + if v3.is_v3(resource): + return v3.flatten_patient(resource) extensions = resource.get("extension", []) identifiers = resource.get("identifier", []) address = (resource.get("address") or [{}])[0] @@ -548,7 +555,12 @@ def _extract_item( def flatten_eob( resource: dict, ) -> tuple[ExplanationOfBenefit, list[EobItem]]: - """Flatten a raw FHIR R4 EOB into header + item models.""" + """Flatten a raw FHIR R4 EOB into header + item models. + + v3 resources are routed to :mod:`bcda.express.v3`. + """ + if v3.is_v3(resource): + return v3.flatten_eob(resource) extensions = resource.get("extension", []) identifiers = resource.get("identifier", []) billable = resource.get("billablePeriod", {}) @@ -801,7 +813,12 @@ def flatten_eob( def flatten_coverage(resource: dict) -> Coverage: - """Flatten a raw FHIR R4 Coverage resource into a Coverage model.""" + """Flatten a raw FHIR R4 Coverage resource into a Coverage model. + + v3 resources are routed to :mod:`bcda.express.v3`. + """ + if v3.is_v3(resource): + return v3.flatten_coverage(resource) extensions = resource.get("extension", []) # Class discrimination @@ -940,6 +957,66 @@ def _read_ndjson(path: Path) -> list[dict]: return [json.loads(line) for line in f if line.strip()] +def _iter_ndjson(paths: list[Path]): + """Yield resources from several ndjson parts, one line at a time.""" + for path in paths: + with open(path) as f: + for line in f: + if line.strip(): + yield json.loads(line) + + +_ARROW_TYPES = { + "str": pa.string(), + "float": pa.float64(), + "int": pa.int64(), + "bool": pa.bool_(), + "date": pa.date32(), + "datetime": pa.timestamp("us"), +} + + +def arrow_schema(model: type) -> pa.Schema: + """Arrow schema for a ``SQLTable`` model (every column nullable). + + Streaming writes need a fixed schema up front — inferring it from + the first batch would type an all-NULL column as ``null`` and then + reject a later batch that carries values. + """ + import types as _types + + fields = [] + for name, info in model.model_fields.items(): + ann = info.annotation + if isinstance(ann, _types.UnionType): + ann = next(a for a in ann.__args__ if a is not type(None)) + key = getattr(ann, "__name__", str(ann)) + fields.append(pa.field(name, _ARROW_TYPES.get(key, pa.string()))) + return pa.schema(fields) + + +class _ParquetSink: + """Append-by-batch Parquet writer over fsspec; counts rows.""" + + def __init__(self, fs: object, path: str, schema: pa.Schema) -> None: + parent = path.rsplit("/", 1)[0] + fs.mkdirs(parent, exist_ok=True) + self._fh = fs.open(path, "wb") + self._writer = pq.ParquetWriter(self._fh, schema) + self._schema = schema + self.rows = 0 + + def write(self, rows: list[dict]) -> None: + if not rows: + return + self._writer.write_table(pa.Table.from_pylist(rows, schema=self._schema)) + self.rows += len(rows) + + def close(self) -> None: + self._writer.close() + self._fh.close() + + def _write_parquet( fs: object, path: str, @@ -956,8 +1033,48 @@ def _write_parquet( return table.num_rows +RESOURCE_TYPES = ("Patient", "ExplanationOfBenefit", "Coverage") + +# Resources per Arrow batch while streaming an export to Parquet. A +# Large-entity v3 EOB export is ~190 files / 2 GB of ndjson, so the +# flattener never holds a whole resource type in memory. +BATCH_ROWS = 2000 + + +def resolve_ndjson(ndjson_dir: str | Path | list[str | Path]) -> dict[str, list[Path]]: + """``{resource_type: [part, ...]}`` for the export files. + + *ndjson_dir* may be one directory or several (a v3 pull is often + three single-type jobs, each landing in its own ``exports//`` + directory — see ``aco.load.bcda``). Later directories win. A + type's parts are ``.ndjson``, ``.1.ndjson``, … as the + client and store name them (BCDA splits large exports into many + output files); they are returned in part order. + """ + dirs = [ndjson_dir] if isinstance(ndjson_dir, (str, Path)) else list(ndjson_dir) + found: dict[str, list[Path]] = {} + for d in dirs: + for rtype in RESOURCE_TYPES: + parts = sorted( + Path(d).glob(f"{rtype}*.ndjson"), + key=lambda p: ( + int(p.name.split(".")[1]) + if p.name.count(".") == 2 and p.name.split(".")[1].isdigit() + else -1 + ), + ) + parts = [ + p + for p in parts + if p.name == f"{rtype}.ndjson" or p.name.split(".")[1].isdigit() + ] + if parts: + found[rtype] = parts + return found + + def flatten_export( - ndjson_dir: str | Path, + ndjson_dir: str | Path | list[str | Path], store_path: str | Path | None = None, *, storage_options: dict | None = None, @@ -965,9 +1082,10 @@ def flatten_export( """Flatten raw FHIR ndjson files and write to Parquet. Reads ``Patient.ndjson``, ``ExplanationOfBenefit.ndjson``, and - ``Coverage.ndjson`` from *ndjson_dir*, flattens them into the - ``bcda.table`` Pydantic models, and persists each as a Parquet - file under *store_path*. + ``Coverage.ndjson`` from *ndjson_dir* (one directory or a list — + see :func:`resolve_ndjson`), flattens them into the ``bcda.table`` + Pydantic models (v2 or v3 dialect per resource), and persists each + as a Parquet file under *store_path*. Output layout:: @@ -998,85 +1116,88 @@ def flatten_export( store_path = path("storage.bcda") setup() - ndjson_dir = Path(ndjson_dir) + files = resolve_ndjson(ndjson_dir) root = str(store_path).rstrip("/") fs, _ = fsspec.core.url_to_fs(root, **(storage_options or {})) counts: dict[str, int] = {} # Patient - patient_path = ndjson_dir / "Patient.ndjson" - if patient_path.exists(): + if files.get("Patient"): t0 = time.monotonic() - raw = _read_ndjson(patient_path) - patients = flatten_patients(raw) - rows = [p.model_dump() for p in patients] - n = _write_parquet(fs, f"{root}/flat/patient.parquet", rows) - elapsed = time.monotonic() - t0 - counts["bcda.patient"] = n + sink = _ParquetSink(fs, f"{root}/flat/patient.parquet", arrow_schema(Patient)) + batch: list[dict] = [] + for raw in _iter_ndjson(files["Patient"]): + batch.append(flatten_patient(raw).model_dump()) + if len(batch) >= BATCH_ROWS: + sink.write(batch) + batch = [] + sink.write(batch) + sink.close() + counts["bcda.patient"] = sink.rows log.info( "Flattened %d patients in %.1fs", - n, - elapsed, - extra={"table": "bcda.patient", "rows": n}, + sink.rows, + time.monotonic() - t0, + extra={"table": "bcda.patient", "rows": sink.rows}, ) # ExplanationOfBenefit + EobItem - eob_path = ndjson_dir / "ExplanationOfBenefit.ndjson" - if eob_path.exists(): + if files.get("ExplanationOfBenefit"): t0 = time.monotonic() - raw = _read_ndjson(eob_path) - headers, items = flatten_eobs(raw) - header_rows = [h.model_dump() for h in headers] - item_rows = [i.model_dump() for i in items] - n_headers = _write_parquet( + head_sink = _ParquetSink( fs, f"{root}/flat/explanation_of_benefit.parquet", - header_rows, + arrow_schema(ExplanationOfBenefit), ) - n_items = _write_parquet( - fs, - f"{root}/flat/eob_item.parquet", - item_rows, + item_sink = _ParquetSink( + fs, f"{root}/flat/eob_item.parquet", arrow_schema(EobItem) ) - elapsed = time.monotonic() - t0 - counts["bcda.explanation_of_benefit"] = n_headers - counts["bcda.eob_item"] = n_items + heads: list[dict] = [] + items: list[dict] = [] + for raw in _iter_ndjson(files["ExplanationOfBenefit"]): + header, lines = flatten_eob(raw) + heads.append(header.model_dump()) + items.extend(i.model_dump() for i in lines) + if len(heads) >= BATCH_ROWS: + head_sink.write(heads) + item_sink.write(items) + heads, items = [], [] + head_sink.write(heads) + item_sink.write(items) + head_sink.close() + item_sink.close() + counts["bcda.explanation_of_benefit"] = head_sink.rows + counts["bcda.eob_item"] = item_sink.rows log.info( "Flattened %d EOBs (%d items) in %.1fs", - n_headers, - n_items, - elapsed, + head_sink.rows, + item_sink.rows, + time.monotonic() - t0, extra={ "table": "bcda.explanation_of_benefit", - "rows": n_headers, - "items": n_items, + "rows": head_sink.rows, + "items": item_sink.rows, }, ) # Coverage - coverage_path = ndjson_dir / "Coverage.ndjson" - if coverage_path.exists(): + if files.get("Coverage"): t0 = time.monotonic() - raw = _read_ndjson(coverage_path) - coverages = flatten_coverages(raw) - rows = [c.model_dump() for c in coverages] - n = _write_parquet( - fs, - f"{root}/flat/coverage.parquet", - rows, - ) - elapsed = time.monotonic() - t0 - counts["bcda.coverage"] = n + sink = _ParquetSink(fs, f"{root}/flat/coverage.parquet", arrow_schema(Coverage)) + batch = [] + for raw in _iter_ndjson(files["Coverage"]): + batch.append(flatten_coverage(raw).model_dump()) + if len(batch) >= BATCH_ROWS: + sink.write(batch) + batch = [] + sink.write(batch) + sink.close() + counts["bcda.coverage"] = sink.rows log.info( - "Flattened %d coverage records in %.1fs", - n, - elapsed, - extra={"table": "bcda.coverage", "rows": n}, + "Flattened %d coverages in %.1fs", + sink.rows, + time.monotonic() - t0, + extra={"table": "bcda.coverage", "rows": sink.rows}, ) - log.info( - "Flatten complete: %s", - counts, - extra={"counts": counts}, - ) return counts diff --git a/src/bcda/express/v3.py b/src/bcda/express/v3.py new file mode 100644 index 0000000..d4945f1 --- /dev/null +++ b/src/bcda/express/v3.py @@ -0,0 +1,1212 @@ +"""BCDA v3 flatteners — FHIR R4 (C4BB 2.2.0) resources → ``bcda.*`` tables. + +The v3 payload moves every BlueButton code off the retired +``https://bluebutton.cms.gov/resources/variables/`` URLs onto +``https://bluebutton.cms.gov/fhir/StructureDefinition/`` (extensions) +and ``https://bluebutton.cms.gov/fhir/CodeSystem/`` (code systems), +and consolidates amounts into ``adjudication`` / ``total`` / +``item.adjudication`` under the ``Adjudication`` CodeSystem. The v2 +flattener matched on the old URL suffix, so on a v3 resource it returns +NULL for almost everything without complaint. + +Every map below is read off the **BCDA v3 Data Dictionary** +(``dev/vendor/bcda-site/BCDA_v3_Data_Dictionary.xlsx``, also flattened +to ``dev/seeds/bcda_v3_data_dictionary.csv``): the dictionary's +``fhirPath`` says where a field lives, ``sourceColumn`` is the IDR name +used for the v3-only columns, and ``ccwMapping`` is the crosswalk onto +the CCW-named columns the tables already had. Comments cite the +``sourceColumn`` so a reader can check any line against the sheet. + +Identity follows the migration guide: ``Patient.id`` is the IDR +beneficiary key (``bene_id``), the current MBI is the ``us-mbi`` +identifier without a ``period.end``, the claim id is the C4BB ``uc`` +identifier and the claim control number pairs a SharedSystem EOB with +its NationalClaimsHistory successor. + +Dialect detection lives in :func:`is_v3`; ``bcda.express.flatten`` +dispatches here per resource, so v2 exports keep flattening unchanged. +""" + +from __future__ import annotations + +import re +from datetime import date +from typing import Any + +from bcda.table.coverage import Coverage +from bcda.table.eob import EobItem, ExplanationOfBenefit +from bcda.table.patient import Patient + +# ── URL vocabulary ─────────────────────────────────────────── + +BB_FHIR = "https://bluebutton.cms.gov/fhir/" +SD = BB_FHIR + "StructureDefinition/" +CS = BB_FHIR + "CodeSystem/" +CS_ADJ = CS + "Adjudication" +CS_SI = CS + "Supporting-Information" +CS_SYSTEM_TYPE = CS + "System-Type" +CS_FINAL_ACTION = CS + "Final-Action" +CS_EOB_TYPE = CS + "EOB-TYPE" +CS_CLM_TYPE = CS + "CLM-TYPE-CD" +CS_ANSI_RSN = CS + "ANSI-RSN-CD" +CS_ANSI_GRP = CS + "ANSI-GRP-CD" +ID_CLM_CNTL = "https://bluebutton.cms.gov/identifiers/CLM-CNTL-NUM" +GENERIC_ID = "bluebutton.cms.gov/fhir/identifiers/GenericIdNum" + +C4BB_ID_TYPE = "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType" +C4BB_SI = "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBSupportingInfoType" +C4BB_ADJ_DISC = ( + "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBAdjudicationDiscriminator" +) +US_MBI = "http://hl7.org/fhir/sid/us-mbi" +US_NPI = "http://hl7.org/fhir/sid/us-npi" +CCN = "http://terminology.hl7.org/NamingSystem/CCN" +TAX_ID = "urn:oid:2.16.840.1.113883.4.4" +US_CORE_RACE = "http://hl7.org/fhir/us/core/StructureDefinition/us-core-race" +US_CORE_ETHNICITY = "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity" +US_CORE_SEX = "http://hl7.org/fhir/us/core/StructureDefinition/us-core-sex" +NDC = "http://hl7.org/fhir/sid/ndc" +HIPPS = "HIPPSCodes" # https://www.cms.gov/.../ProspMedicareFeeSvcPmtGen/HIPPSCodes +HCPCS_SYSTEMS = ("http://www.ama-assn.org/go/cpt", "HCPCSReleaseCodeSets") +POS = "Place_of_Service_Code_Set" + +# CCW BENE_RACE_CD from the US Core OMB race/ethnicity codes +# (CCW codebook: 0 Unknown, 1 White, 2 Black, 3 Other, 4 Asian/Pacific +# Islander, 5 Hispanic, 6 North American Native). Hispanic ethnicity +# wins when present, as in the CCW RTI race code derivation. +OMB_TO_CCW_RACE = { + "2106-3": "1", # White + "2054-5": "2", # Black or African American + "2028-9": "4", # Asian + "2076-8": "4", # Native Hawaiian or Other Pacific Islander + "1002-5": "6", # American Indian or Alaska Native + "2131-1": "3", # Other Race + "UNK": "0", + "ASKU": "0", +} +HISPANIC_OMB = "2135-2" + +# SNOMED sex (us-core-sex) → CCW BENE_SEX_IDENT_CD +SNOMED_SEX = {"248153007": "1", "248152002": "2"} +GENDER_TO_CCW = {"male": "1", "female": "2"} + +# ── Primitives (one per dictionary fhirPath idiom) ─────────── + + +def _get(d: Any, *keys: str) -> Any: + for k in keys: + if not isinstance(d, dict): + return None + d = d.get(k) + return d + + +def _date(value: Any) -> date | None: + if not value or not isinstance(value, str): + return None + try: + return date.fromisoformat(value[:10]) + except ValueError: + return None + + +def _num(value: Any) -> float | None: + if value is None or isinstance(value, bool): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _ref_id(ref: Any) -> str | None: + """``"Patient/-123"`` → ``"-123"``; ``"#org"`` → ``"org"``.""" + if not isinstance(ref, str) or not ref: + return None + return ref.rsplit("/", 1)[-1].lstrip("#") + + +def _coding(cc: Any, system: str | None = None) -> str | None: + """Code from a CodeableConcept, by exact system or system suffix.""" + for c in (cc or {}).get("coding", []) if isinstance(cc, dict) else []: + sys_ = c.get("system", "") + if system is None or sys_ == system or sys_.endswith(system): + return c.get("code") + return None + + +def _has_coding(cc: Any, system: str, code: str) -> bool: + for c in (cc or {}).get("coding", []) if isinstance(cc, dict) else []: + if c.get("system") == system and c.get("code") == code: + return True + return False + + +def _ext_value(ext: dict) -> Any: + """The ``value[x]`` of an extension, as the dictionary reads it.""" + if "valueCoding" in ext: + return _get(ext, "valueCoding", "code") + for key in ( + "valueCode", + "valueString", + "valueDate", + "valueDateTime", + "valueBoolean", + ): + if key in ext: + return ext[key] + for key in ("valueDecimal", "valueInteger", "valueUnsignedInt", "valuePositiveInt"): + if key in ext: + return ext[key] + if "valueQuantity" in ext: + return _get(ext, "valueQuantity", "value") + if "valueMoney" in ext: + return _get(ext, "valueMoney", "value") + if "valueIdentifier" in ext: + return _get(ext, "valueIdentifier", "value") + return None + + +def _ext(extensions: list[dict] | None, code: str) -> Any: + """``extension.where(url = SD + code).value``.""" + for ext in extensions or []: + if ext.get("url") == SD + code: + return _ext_value(ext) + return None + + +def _ext_str(extensions: list[dict] | None, code: str) -> str | None: + v = _ext(extensions, code) + return None if v is None else str(v) + + +def _identifier(identifiers: list[dict] | None, system: str) -> str | None: + for ident in identifiers or []: + sys_ = ident.get("system", "") + if sys_ == system or sys_.endswith(system): + return ident.get("value") + return None + + +def _identifier_typed( + identifiers: list[dict] | None, system: str, code: str +) -> str | None: + """``identifier.where(type.coding.system = system and type.coding.code = code).value``.""" + for ident in identifiers or []: + if _has_coding(ident.get("type"), system, code): + return ident.get("value") + return None + + +def _tag(resource: dict, system: str) -> str | None: + for t in _get(resource, "meta", "tag") or []: + if t.get("system") == system: + return t.get("code") + return None + + +def _si(infos: list[dict] | None, code: str, system: str = CS_SI) -> dict | None: + """``supportingInfo.where(category.coding.where(system, code).exists())``.""" + for info in infos or []: + if _has_coding(info.get("category"), system, code): + return info + return None + + +def _si_all(infos: list[dict] | None, code: str, system: str = CS_SI) -> list[dict]: + return [i for i in infos or [] if _has_coding(i.get("category"), system, code)] + + +def _si_code( + infos: list[dict] | None, + code: str, + *, + system: str = CS_SI, + code_system: str | None = None, +) -> str | None: + info = _si(infos, code, system) + return _coding(info.get("code"), code_system) if info else None + + +def _si_code_by_system(infos: list[dict] | None, code_system: str) -> str | None: + """``supportingInfo.where(code.coding.system = X).code.coding.code`` — the + dictionary's idiom for codes without their own category.""" + for info in infos or []: + c = _coding(info.get("code"), code_system) + if c is not None: + return c + return None + + +def _si_date(infos: list[dict] | None, code: str, system: str = CS_SI) -> date | None: + info = _si(infos, code, system) + if not info: + return None + return _date(info.get("timingDate") or _get(info, "timingPeriod", "start")) + + +def _si_period( + infos: list[dict] | None, code: str, system: str = CS_SI +) -> tuple[date | None, date | None]: + info = _si(infos, code, system) + if not info: + return None, None + period = info.get("timingPeriod") or {} + start = period.get("start") or info.get("timingDate") + return _date(start), _date(period.get("end")) + + +def _si_value(infos: list[dict] | None, code: str, system: str = CS_SI) -> Any: + info = _si(infos, code, system) + if not info: + return None + if "valueString" in info: + return info["valueString"] + if "valueQuantity" in info: + return _get(info, "valueQuantity", "value") + if "valueBoolean" in info: + return info["valueBoolean"] + return None + + +def _adj_by_code( + adjudications: list[dict] | None, code: str, system: str = CS_ADJ +) -> dict | None: + for adj in adjudications or []: + if _has_coding(adj.get("category"), system, code): + return adj + return None + + +def _adj_amount(adjudications: list[dict] | None, code: str) -> float | None: + adj = _adj_by_code(adjudications, code) + return _num(_get(adj, "amount", "value")) if adj else None + + +def _adj_value(adjudications: list[dict] | None, code: str) -> float | None: + adj = _adj_by_code(adjudications, code) + return _num(adj.get("value")) if adj else None + + +def _adj_quantity(adjudications: list[dict] | None, code: str) -> float | None: + """``adjudication.value`` — bcda-app emits counts as ``value`` + (decimal) on the sample files; accept ``quantity.value`` too.""" + adj = _adj_by_code(adjudications, code) + if not adj: + return None + if "value" in adj: + return _num(adj["value"]) + return _num(_get(adj, "quantity", "value")) + + +def _contained(resource: dict, ref: Any) -> dict | None: + rid = _ref_id(ref) + if not rid: + return None + for c in resource.get("contained", []) or []: + if c.get("id") == rid: + return c + return None + + +def _display_name(name: Any) -> str | None: + """Organization.name is a string; a contained Practitioner carries + HumanName[] — render the first as ``Family, Given``.""" + if name is None: + return None + if isinstance(name, str): + return name + if isinstance(name, list): + if not name: + return None + first = name[0] if isinstance(name[0], dict) else {} + family = first.get("family") + given = " ".join(first.get("given") or []) + text = first.get("text") + if text: + return str(text) + return ", ".join(p for p in (family, given) if p) or None + return str(name) + + +def _care_team(members: list[dict] | None) -> dict[str, list[dict]]: + """Members grouped by C4BB/FHIR care team role code.""" + by_role: dict[str, list[dict]] = {} + for m in members or []: + roles = { + c.get("code") for c in _get(m, "role", "coding") or [] if c.get("code") + } + for r in roles: + by_role.setdefault(r, []).append(m) + return by_role + + +def _npi(member: dict | None) -> str | None: + """``provider.identifier.value`` when it is an NPI.""" + ident = _get(member, "provider", "identifier") + if not isinstance(ident, dict): + return None + if ident.get("system") == US_NPI or _has_coding( + ident.get("type"), "http://terminology.hl7.org/CodeSystem/v2-0203", "NPI" + ): + return ident.get("value") + return None + + +def _provider_value(member: dict | None) -> str | None: + return _get(member, "provider", "identifier", "value") + + +# ── Dialect detection ──────────────────────────────────────── + +_V3_PROFILE = re.compile(r"carin-bb/StructureDefinition/C4BB-[A-Za-z-]+\|2\.") + + +def is_v3(resource: dict) -> bool: + """True when ``resource`` is a BCDA v3 (C4BB 2.x) payload. + + v3 resources carry versioned C4BB profiles in ``meta.profile``; + failing that, any ``bluebutton.cms.gov/fhir/`` URL in an + extension, identifier, tag or coding is v3-only (v2 used + ``bluebutton.cms.gov/resources/``). + """ + for p in _get(resource, "meta", "profile") or []: + if isinstance(p, str) and _V3_PROFILE.search(p): + return True + for t in _get(resource, "meta", "tag") or []: + if str(t.get("system", "")).startswith(BB_FHIR): + return True + for ext in resource.get("extension", []) or []: + if str(ext.get("url", "")).startswith(BB_FHIR): + return True + for ident in resource.get("identifier", []) or []: + if str(ident.get("system", "")).startswith( + "https://bluebutton.cms.gov/identifiers/" + ): + return True + return False + + +# ── Patient ────────────────────────────────────────────────── + + +def _mbis(identifiers: list[dict]) -> tuple[dict | None, dict | None]: + """(current, most recent historic) ``us-mbi`` identifiers. + + Current = no ``period.end``; when every MBI has an end, the latest + end wins (the dictionary's BENE_MBI_OBSLT_DT), and there is no + separate historic row. + """ + mbis = [ + i for i in identifiers or [] if i.get("system") == US_MBI and i.get("value") + ] + if not mbis: + return None, None + open_ = [i for i in mbis if not _get(i, "period", "end")] + closed = sorted( + (i for i in mbis if _get(i, "period", "end")), + key=lambda i: _get(i, "period", "end") or "", + reverse=True, + ) + if open_: + return open_[0], (closed[0] if closed else None) + return closed[0], (closed[1] if len(closed) > 1 else None) + + +def _race(extensions: list[dict]) -> tuple[str | None, str | None]: + """(us_core_race OMB code, CCW bene_race_cd) — Patient BENE_RACE_CD.""" + omb_race = None + hispanic = False + for ext in extensions or []: + url = ext.get("url") + if url not in (US_CORE_RACE, US_CORE_ETHNICITY): + continue + for sub in ext.get("extension", []) or []: + if sub.get("url") != "ombCategory": + continue + code = _get(sub, "valueCoding", "code") + if url == US_CORE_RACE and omb_race is None: + omb_race = code + if url == US_CORE_ETHNICITY and code == HISPANIC_OMB: + hispanic = True + if hispanic: + return omb_race, "5" + if omb_race is None: + return None, None + return omb_race, OMB_TO_CCW_RACE.get(omb_race, "3") + + +def flatten_patient(resource: dict) -> Patient: + """Flatten a v3 Patient (C4BB-Patient 2.2.0 / us-core-patient 7).""" + identifiers = resource.get("identifier", []) or [] + extensions = resource.get("extension", []) or [] + address = (resource.get("address") or [{}])[0] + name = (resource.get("name") or [{}])[0] + given = name.get("given") or [] + lines = address.get("line") or [] + postal = address.get("postalCode") or "" + + current, historic = _mbis(identifiers) + sex_snomed = None + for ext in extensions: + if ext.get("url") == US_CORE_SEX: + sex_snomed = ext.get("valueCode") + us_core_race, bene_race_cd = _race(extensions) + + # Cross-reference: BENE_XREF_EFCTV_SK and the source-key list. + xref = None + sk_list = [] + for link in resource.get("link", []) or []: + other = link.get("other") or {} + if link.get("type") == "replaced-by" and other.get("reference"): + xref = _ref_id(other["reference"]) + if other.get("display"): + sk_list.append(str(other["display"])) + if resource.get("id") is not None: + sk_list.append(str(resource["id"])) + + lang = None + for comm in resource.get("communication", []) or []: + lang = _coding(comm.get("language"), "urn:ietf:bcp:47") or lang + + gender = resource.get("gender") + bene_sex_cd = ( + SNOMED_SEX.get(sex_snomed or "") + or GENDER_TO_CCW.get(gender or "") + or ("0" if gender else None) + ) + + return Patient( + id=resource.get("id"), + last_updated=_get(resource, "meta", "lastUpdated"), + bene_id=resource.get("id"), # BENE_SK — EOB patient.reference uses it + bene_mbi_id=current.get("value") if current else None, # BENE_MBI_ID + identifier_currency="current" + if current and not _get(current, "period", "end") + else ("historic" if current else None), + hicn_mbi_xref_ind="M", + crnt_num=current.get("value") if current else None, + prvs_num=historic.get("value") if historic else None, + efctv_bgn_dt=_date(_get(historic, "period", "start")) + if historic + else None, # BENE_MBI_EFCTV_DT (historic) + efctv_end_dt=_date(_get(historic, "period", "end")) + if historic + else None, # BENE_MBI_OBSLT_DT + mbi_efctv_dt=_date(_get(current, "period", "start")) if current else None, + bene_1st_name=given[0] if len(given) > 0 else None, # BENE_1ST_NAME + bene_midl_name=given[1] if len(given) > 1 else None, # BENE_MIDL_NAME + bene_last_name=name.get("family"), # BENE_LAST_NAME + bene_dob=_date(resource.get("birthDate")), # BENE_BRTH_DT + bene_sex_cd=bene_sex_cd, # BENE_SEX_CD ← us-core-sex, CCW-coded + us_core_sex=sex_snomed, + bene_race_cd=bene_race_cd, # BENE_RACE_CD ← us-core-race (OMB→CCW) + us_core_race=us_core_race, + bene_death_dt=_date(resource.get("deceasedDateTime")), # BENE_DEATH_DT + rfrnc_yr=None, + bene_fips_state_cd=None, # v3 carries USPS state only + bene_zip_cd=postal or None, + bene_line_1_adr=lines[0] if len(lines) > 0 else None, + bene_line_2_adr=lines[1] if len(lines) > 1 else None, + bene_line_3_adr=lines[2] if len(lines) > 2 else None, + bene_line_4_adr=lines[3] if len(lines) > 3 else None, + bene_line_5_adr=lines[4] if len(lines) > 4 else None, + bene_line_6_adr=lines[5] if len(lines) > 5 else None, + geo_zip_plc_name=address.get("city"), # GEO_ZIP_PLC_NAME + geo_usps_state_cd=address.get("state"), # GEO_USPS_STATE_CD + geo_zip5_cd=postal[:5] if postal else None, # GEO_ZIP5_CD + geo_zip4_cd=postal[5:] if len(postal) > 5 else None, + cntct_lang_cd=lang, # CNTCT_LANG_CD + xref_efctv_sk=xref or resource.get("id"), # BENE_XREF_EFCTV_SK + bene_sk_list=";".join(dict.fromkeys(sk_list)) or None, + ) + + +# ── Coverage ───────────────────────────────────────────────── + +# extension code → Coverage column (dictionary sourceColumn in comment) +_COV_EXT = { + "BENE-MDCR-STUS-CD": "ms_cd", # BENE_MDCR_STUS_CD (CCW BENE_MDCR_STATUS_CD) + "BENE-MDCR-ENTLMT-RSN-CD": "orec", # BENE_MDCR_ENTLMT_RSN_CD (CCW BENE_ENTLMT_RSN_ORIG) + "BENE-ESRD-STUS-ID": "esrd_ind", # BENE_ESRD_STUS_ID (CCW BENE_ESRD_IND) + "BENE-MDCR-ENTLMT-STUS-CD": "entlmt_stus_cd", # → a_trm_cd / b_trm_cd by plan + "BENE-BUYIN-CD": "buyin_cd", # BENE_BUYIN_CD + "BENE-ENRLMT-RSN-CD": "enrlmt_rsn_cd", # BENE_MDCR_ENRLMT_RSN_CD + "BENE-DSBLD-STUS-ID": "dsbld_stus_id", # BENE_DSBLD_STUS_ID + "BENE-DUAL-STUS-CD": "dual_stus_cd", # BENE_DUAL_STUS_CD + "BENE-DUAL-TYPE-CD": "dual_type_cd", # BENE_DUAL_TYPE_CD + "MEDICAID-STATE-CD": "medicaid_state_cd", # GEO_USPS_STATE_CD (Medicaid) + "BENE-CNTRCT-NUM": "cntrct_num", # BENE_CNTRCT_NUM + "BENE-PBP-NUM": "pbp_num", # BENE_PBP_NUM + "BENE-CVRG-TYPE-CD": "cvrg_type_cd", # BENE_CVRG_TYPE_CD + "CNTRCT-PBP-SGMT-NUM": "pbp_sgmt_num", # CNTRCT_PBP_SGMT_NUM + "BENE-LIS-EFCTV-CD": "lis_efctv_cd", # BENE_LIS_EFCTV_CD + "BENE-LIS-COPMT-LVL-CD": "lis_copmt_lvl_cd", # BENE_LIS_COPMT_LVL_CD + "BENE-LIS-PTD-PRM-PCT": "lis_ptd_prm_pct", # BENE_LIS_PTD_PRM_PCT + "BENE-ENRLMT-EMPLR-SBSDY-SW": "rds_sw", # BENE_ENRLMT_EMPLR_SBSDY_SW +} +_COV_CLASS = { + "rxid": "rx_id", + "rxgroup": "rx_group", + "rxpcn": "rx_pcn", + "rxbin": "rx_bin", +} + + +def flatten_coverage(resource: dict) -> Coverage: + """Flatten a v3 Coverage (one row per plan per period).""" + extensions = resource.get("extension", []) or [] + kwargs: dict[str, Any] = {} + + class_plan = None + class_group = None + for cls in resource.get("class", []) or []: + kind = _coding(cls.get("type")) + if kind == "plan": + class_plan = cls.get("value") + elif kind == "group": + class_group = cls.get("value") + elif kind in _COV_CLASS: + kwargs[_COV_CLASS[kind]] = cls.get("value") + + for code, col in _COV_EXT.items(): + v = _ext(extensions, code) + if v is not None: + kwargs[col] = str(v) + + period = resource.get("period") or {} + start, end = _date(period.get("start")), _date(period.get("end")) + plan = (class_plan or "").lower() + if "part a" in plan: + kwargs["bene_part_a_enrlmt_bgn_dt"] = start # CCW COVSTART + kwargs["mdcr_covg_a_end_dt"] = end + kwargs["a_trm_cd"] = kwargs.get("entlmt_stus_cd") # CCW BENE_PTA_TRMNTN_CD + elif "part b" in plan: + kwargs["bene_part_b_enrlmt_bgn_dt"] = start + kwargs["mdcr_covg_b_end_dt"] = end + kwargs["b_trm_cd"] = kwargs.get("entlmt_stus_cd") # CCW BENE_PTB_TRMNTN_CD + elif "part d" in plan: + kwargs["mdcr_covg_d_bgn_dt"] = start + kwargs["mdcr_covg_d_end_dt"] = end + + if "part c" in plan or "part d" in plan: + for c in resource.get("contained", []) or []: + if c.get("resourceType") == "Organization" and c.get("id") != "cms-org": + kwargs["cntrct_pbp_name"] = c.get("name") # CNTRCT_PBP_NAME + + return Coverage( + id=resource.get("id"), + last_updated=_get(resource, "meta", "lastUpdated"), + status=resource.get("status"), + subscriber_id=resource.get("subscriberId") + or _identifier(resource.get("identifier"), US_MBI), # BENE_MBI_ID + beneficiary_reference=_ref_id( + _get(resource, "beneficiary", "reference") + ), # BENE_SK + coverage_type=_coding(resource.get("type")), + class_group=class_group, + class_plan=class_plan, # Coverage Part + period_start=start, + period_end=end, + **kwargs, + ) + + +# ── ExplanationOfBenefit ───────────────────────────────────── + +# header ``total`` Adjudication code → columns (sourceColumn / ccwMapping) +_TOTALS = { + "CLM_SBMT_CHRG_AMT": ( + "clm_sbmt_chrg_amt", + "clm_mdcr_instnl_tot_chrg_amt", + ), # CLM_TOT_CHRG_AMT, NCH_CARR_CLM_SBMTD_CHRG_AMT + "CLM_ALOWD_CHRG_AMT": ("clm_alowd_chrg_amt",), # NCH_CARR_CLM_ALOWD_AMT + "CLM_BENE_PMT_AMT": ("clm_bene_pmt_amt",), # PTNT_PAY_AMT + "CLM_PRVDR_PMT_AMT": ("clm_prvdr_pmt_amt",), + "CLM_MDCR_DDCTBL_AMT": ( + "clm_mdcr_ddctbl_amt", + ), # NCH_BENE_PTB_DDCTBL_AMT, NCH_IP_TOT_DDCTN_AMT, CARR_CLM_CASH_DDCTBL_APLD_AMT + "CLM_MDCR_COINSRNC_AMT": ("clm_mdcr_coinsrnc_amt",), # NCH_BENE_PTA/PTB_COINSRNC + "CLM_NCVRD_CHRG_AMT": ("clm_ncvrd_chrg_amt",), # NCH_IP_NCVRD_CHRG_AMT + "CLM_OTHR_TP_PD_AMT": ("clm_othr_tp_pd_amt",), + "CLM_MDCR_INSTNL_BENE_PD_AMT": ( + "clm_mdcr_instnl_bene_pd_amt", + ), # CLM_OP_BENE_PMT_AMT + "CLM_BENE_PD_AMT": ("clm_mdcr_instnl_bene_pd_amt",), + "TOT_RX_CST_AMT": ("tot_rx_cst_amt",), +} + +# header ``adjudication`` Adjudication code → columns; ``amount`` unless listed in _ADJ_VALUES +_ADJ_HEADER = { + "CLM_MDCR_IP_PPS_CPTL_IME_AMT": ( + "clm_mdcr_ip_pps_cptl_ime_amt", + ), # CLM_PPS_CPTL_IME_AMT + "CLM_OPRTNL_IME_AMT": ( + "clm_oprtnl_ime_amt", + "ime_op_clm_val_amt", + ), # IME_OP_CLM_VAL_AMT + "CLM_MDCR_IP_PPS_DSPRPRTNT_AMT": ( + "clm_mdcr_ip_pps_dsprprtnt_amt", + ), # CLM_PPS_CPTL_DSPRPRTNT_SHR_AMT + "CLM_HIPPS_UNCOMPD_CARE_AMT": ( + "clm_hipps_uncompd_care_amt", + "clm_uncompd_care_pmt_amt", + ), # CLM_UNCOMPD_CARE_PMT_AMT + "CLM_OPRTNL_DSPRTNT_AMT": ( + "clm_oprtnl_dsprtnt_amt", + "dsh_op_clm_val_amt", + ), # DSH_OP_CLM_VAL_AMT + "CLM_INSTNL_CVRD_DAY_CNT": ("clm_instnl_cvrd_day_cnt",), + "CLM_MDCR_IP_LRD_USE_CNT": ("clm_mdcr_ip_lrd_use_cnt",), # BENE_LRD_USED_CNT + "CLM_INSTNL_MDCR_COINS_DAY_CNT": ( + "clm_instnl_mdcr_coins_day_cnt", + ), # BENE_TOT_COINSRNC_DAYS_CNT + "CLM_INSTNL_NCVRD_DAY_CNT": ( + "clm_instnl_ncvrd_day_cnt", + ), # CLM_NON_UTLZTN_DAYS_CNT + "CLM_INSTNL_PER_DIEM_AMT": ( + "clm_instnl_per_diem_amt", + ), # CLM_PASS_THRU_PER_DIEM_AMT + "CLM_MDCR_IP_PPS_DRG_WT_NUM": ( + "clm_mdcr_ip_pps_drg_wt_num", + ), # CLM_PPS_CPTL_DRG_WT_NUM + "CLM_MDCR_IP_PPS_CPTL_FSP_AMT": ("clm_mdcr_ip_pps_cptl_fsp_amt",), + "CLM_MDCR_IP_PPS_OUTLIER_AMT": ("clm_mdcr_ip_pps_outlier_amt",), + "CLM_MDCR_IP_PPS_EXCPTN_AMT": ("clm_mdcr_ip_pps_excptn_amt",), + "CLM_MDCR_IP_PPS_CPTL_HRMLS_AMT": ("clm_mdcr_ip_pps_cptl_hrmls_amt",), + "CLM_MDCR_IP_PPS_CPTL_TOT_AMT": ("clm_mdcr_ip_pps_cptl_tot_amt",), + "CLM_INSTNL_DRG_OUTLIER_AMT": ( + "clm_instnl_drg_outlier_amt", + ), # NCH_DRG_OUTLIER_APRVD_PMT_AMT + "CLM_MDCR_INSTNL_PRMRY_PYR_AMT": ( + "clm_mdcr_instnl_prmry_pyr_amt", + ), # NCH_PRMRY_PYR_CLM_PD_AMT + "CLM_MDCR_PRFNL_PRMRY_PYR_AMT": ( + "clm_mdcr_prfnl_prmry_pyr_amt", + ), # CARR_CLM_PRMRY_PYR_PD_AMT + "CLM_INSTNL_PRFNL_AMT": ("clm_instnl_prfnl_amt",), # NCH_PROFNL_CMPNT_CHRG_AMT + "CLM_BLOOD_LBLTY_AMT": ("clm_blood_lblty_amt",), # NCH_BENE_BLOOD_DDCTBL_LBLTY_AM + "CLM_MDCR_IP_BENE_DDCTBL_AMT": ( + "clm_mdcr_ip_bene_ddctbl_amt", + ), # NCH_BENE_IP_DDCTBL_AMT + "CLM_BLOOD_CHRG_AMT": ("clm_blood_chrg_amt",), + "CLM_BLOOD_NCVRD_CHRG_AMT": ("clm_blood_ncvrd_chrg_amt",), + "CLM_MDCR_HOSPC_PRD_CNT": ("clm_mdcr_hospc_prd_cnt",), # BENE_HOSPC_PRD_CNT + "CLM_MDCR_HHA_TOT_VISIT_CNT": ( + "clm_mdcr_hha_tot_visit_cnt", + ), # CLM_HHA_TOT_VISIT_CNT + "CLM_FINL_STDZD_PYMT_AMT": ("clm_finl_stdzd_pymt_amt",), + "CLM_HAC_RDCTN_PYMT_AMT": ("clm_hac_rdctn_pymt_amt",), + "CLM_HIPPS_READMSN_RDCTN_AMT": ("clm_hipps_readmsn_rdctn_amt",), + "CLM_HIPPS_VBP_AMT": ("clm_hipps_vbp_amt",), + "CLM_INSTNL_LOW_VOL_PMT_AMT": ("clm_instnl_low_vol_pmt_amt",), +} +_ADJ_VALUES = { # counts / weights come as ``value``, not ``amount`` + "CLM_INSTNL_CVRD_DAY_CNT", + "CLM_MDCR_IP_LRD_USE_CNT", + "CLM_INSTNL_MDCR_COINS_DAY_CNT", + "CLM_INSTNL_NCVRD_DAY_CNT", + "CLM_MDCR_IP_PPS_DRG_WT_NUM", + "CLM_MDCR_HOSPC_PRD_CNT", + "CLM_MDCR_HHA_TOT_VISIT_CNT", +} + +# supportingInfo (BB Supporting-Information category) code → column; ``code`` path +_SI_CODES = { + "CLM_NRLN_RIC_CD": "nch_near_line_rec_ident_cd", # NCH_NEAR_LINE_REC_IDENT_CD + "CLM_QUERY_CD": "clm_query_cd", # CLAIM_QUERY_CODE, CARR_CLM_ENTRY_CD + "CLM_DISP_CD": "clm_disp_cd", # CLM_DISP_CD + "CLM_ADJSTMT_TYPE_CD": "clm_adjstmt_type_cd", + "CLM_SBMT_FRMT_CD": "clm_sbmt_frmt_cd", # NSTD_FRMT_CD + "CLM_NCH_PRMRY_PYR_CD": "clm_nch_prmry_pyr_cd", # NCH_PRMRY_PYR_CD + "CLM_MDCR_INSTNL_MCO_PD_SW": "clm_mco_pd_sw", # CLM_MCO_PD_SW + "CLM_MDCR_NPMT_RSN_CD": "clm_mdcr_npmt_rsn_cd", # CLM_MDCR_NON_PMT_RSN_CD + "CLM_CARR_PMT_DNL_CD": "clm_carr_pmt_dnl_cd", # CARR_CLM_PMT_DNL_CD + "CLM_FI_ACTN_CD": "fi_clm_actn_cd", # FI_CLM_ACTN_CD + "CLM_PPS_IND_CD": "clm_pps_ind_cd", + "CLM_HHA_LUP_IND_CD": "clm_hha_lup_ind_cd", # CLM_HHA_LUPA_IND_CD + "CLM_HHA_RFRL_CD": "clm_hha_rfrl_cd", + "CLM_CLNCL_TRIL_NUM": "clm_clncl_tril_num", + "CLM_MDCR_PRFNL_PRVDR_ASGNMT_SW": "asgmntcd", # CARR_CLM_PRVDR_ASGNMT_IND_SW + "CLM_OP_SRVC_TYPE_CD": "clm_op_srvc_type_cd", + "CLM_PHRMCY_SRVC_TYPE_CD": "clm_phrmcy_srvc_type_cd", # PHRMCY_SRVC_TYPE_CD + "CLM_PTNT_RSDNC_CD": "clm_ptnt_rsdnc_cd", # PTNT_RSDNC_CD + "CLM_LTC_DSPNSNG_MTHD_CD": "clm_ltc_dspnsng_mthd_cd", # SUBMSN_CLR_CD + "CLM_DRUG_CVRG_STUS_CD": "clm_drug_cvrg_stus_cd", # DRUG_CVRG_STUS_CD + "CLM_CTSTRPHC_CVRG_IND_CD": "clm_ctstrphc_cvrg_ind_cd", # CTSTRPHC_CVRG_CD + "CLM_DSPNSNG_STUS_CD": "clm_dspnsng_stus_cd", # DSPNSNG_STUS_CD + "PRVDR_FAC_FIPS_ST_CD": "prvdr_fac_fips_st_cd", # GEO_BLG_SSA_STATE_CD + "RNDRG_PRVDR_FIPS_ST_CD": "rndrg_prvdr_fips_st_cd", +} +_SI_DATES = { + "CLM_IDR_LD_DT": "clm_idr_ld_dt", + "CLM_CMS_PROC_DT": "clm_cms_proc_dt", + "CLM_NCH_WKLY_PROC_DT": "clm_nch_wkly_proc_dt", # NCH_WKLY_PROC_DT + "CLM_ACTV_CARE_THRU_DT": "clm_actv_care_thru_dt", # NCH_ACTV_OR_CVRD_LVL_CARE_THRU + "CLM_MDCR_EXHSTD_DT": "clm_mdcr_exhstd_dt", # NCH_BENE_MDCR_BNFTS_EXHTD_DT_I + "CLM_NCVRD_FROM_DT": "clm_ncvrd_from_dt", # NCH_VRFD_NCVRD_STAY_FROM_DT + "CLM_NCVRD_THRU_DT": "clm_ncvrd_thru_dt", + "CLM_QLFY_STAY_FROM_DT": "clm_qlfy_stay_from_dt", # NCH_QLFYD_STAY_FROM_DT + "CLM_QLFY_STAY_THRU_DT": "clm_qlfy_stay_thru_dt", +} +_SI_STRINGS = { + "CLM_SBMTR_CNTRCT_NUM": "clm_sbmtr_cntrct_num", # PLAN_CNTRCT_REC_ID + "CLM_SBMTR_CNTRCT_PBP_NUM": "clm_sbmtr_cntrct_pbp_num", # PLAN_PBP_REC_NUM + "CLM_LINE_RX_NUM": "clm_line_rx_srvc_rfrnc_num", # RX_SRVC_RFRNC_NUM +} +# C4BB supportingInfo categories (code system of the answer in the tuple) +_SI_C4BB_CODES = { + "discharge-status": ( + "bene_ptnt_stus_cd", + CS + "BENE-PTNT-STUS-CD", + ), # NCH_PTNT_STATUS_IND_CD / PTNT_DSCHRG_STUS_CD + "drg": ("dgns_drg_cd", None), # CLM_DRG_CD + "admtype": ("clm_admsn_type_cd", CS + "CLM-ADMSN-TYPE-CD"), # CLM_IP_ADMSN_TYPE_CD + "pointoforigin": ( + "clm_admsn_src_cd", + CS + "CLM-ADMSN-SRC-CD", + ), # CLM_SRC_IP_ADMSN_CD + "dawcode": ("clm_daw_prod_slctn_cd", None), # DAW_PROD_SLCTN_CD + "rxorigincode": ("clm_line_rx_orgn_cd", None), # RX_ORGN_CD + "brandgenericindicator": ("clm_brnd_gnrc_cd", None), # BRND_GNRC_CD + "compoundcode": ("clm_cmpnd_cd", None), # CMPND_CD +} +_SI_C4BB_QUANTITIES = { + "dayssupply": "clm_line_days_suply_qty", # DAYS_SUPLY_NUM + "refillnum": "clm_line_rx_fill_num", # FILL_NUM + "refillsauthorized": "clm_line_authrzd_fill_num", +} +_TYPEOFBILL = { + "CLM-BILL-FAC-TYPE-CD": ("clm_bill_fac_type_cd",), # CLM_FAC_TYPE_CD + "CLM-BILL-CLSFCTN-CD": ( + "clm_bill_clsfctn_cd", + "clm_srvc_clsfctn_type_cd", + ), # CLM_SRVC_CLSFCTN_TYPE_CD + "CLM-BILL-FREQ-CD": ("clm_bill_freq_cd",), # CLM_FREQ_CD +} + + +def _diagnoses(diagnosis_list: list[dict]) -> dict[str, str | None]: + out: dict[str, str | None] = { + "prncpl_dgns_cd": None, + "admtg_dgns_cd": None, + "dgns_prcdr_icd_ind": None, + "clm_poa_ind": None, + } + for i in range(1, 13): + out[f"clm_dgns_{i}_cd"] = None + for dx in diagnosis_list or []: + cc = dx.get("diagnosisCodeableConcept") or {} + code = _coding(cc) + system = (_get(cc, "coding") or [{}])[0].get("system", "") + if "icd-10" in system: + out["dgns_prcdr_icd_ind"] = "0" + elif "icd-9" in system: + out["dgns_prcdr_icd_ind"] = "9" + seq = dx.get("sequence") + if isinstance(seq, int) and 1 <= seq <= 12: + out[f"clm_dgns_{seq}_cd"] = code + for t in dx.get("type", []) or []: + tc = _coding(t) + if tc == "principal": + out["prncpl_dgns_cd"] = code + elif tc == "admitting": + out["admtg_dgns_cd"] = code + poa = _coding(dx.get("onAdmission")) + if poa and out["clm_poa_ind"] is None: + out["clm_poa_ind"] = poa # CLM_POA_IND + return out + + +def _diagnosis_by_sequence(diagnosis_list: list[dict]) -> dict[int, str | None]: + return { + dx.get("sequence"): _coding(dx.get("diagnosisCodeableConcept")) + for dx in diagnosis_list or [] + if isinstance(dx.get("sequence"), int) + } + + +# item.adjudication Adjudication code → EobItem columns (ccwMapping) +_ITEM_ADJ = { + "CLM_LINE_CVRD_PD_AMT": ( + "clm_line_cvrd_pd_amt", + "rev_cntr_pmt_amt", + ), # CVRD_D_PLAN_PD_AMT, REV_CNTR_PMT_AMT_AMT + "CLM_LINE_ALOWD_CHRG_AMT": ("clm_line_alowd_chrg_amt",), # LINE_ALOWD_CHRG_AMT + "CLM_LINE_SBMT_CHRG_AMT": ( + "clm_line_sbmtd_chrg_amt", + "rev_cntr_tot_chrg_amt", + ), # LINE_SBMTD_CHRG_AMT, REV_CNTR_TOT_CHRG_AMT + "CLM_LINE_BENE_PMT_AMT": ("clm_line_bene_pmt_amt",), # PTNT_PAY_AMT + "CLM_LINE_PRVDR_PMT_AMT": ( + "clm_line_prvdr_pmt_amt", + "rev_cntr_prvdr_pmt_amt", + ), # LINE_PRVDR_PMT_AMT, REV_CNTR_PRVDR_PMT_AMT + "CLM_BENE_PRMRY_PYR_PD_AMT": ( + "clm_line_bene_prmry_pyr_pd_amt", + ), # LINE_BENE_PRMRY_PYR_PD_AMT + "CLM_LINE_MDCR_COINSRNC_AMT": ("clm_line_coinsrnc_amt",), # LINE_COINSRNC_AMT + "CLM_LINE_MDCR_DDCTBL_AMT": ( + "clm_line_mdcr_ddctbl_amt", + "clm_line_bene_ptb_ddctbl_amt", + "rev_cntr_cash_ddctbl_amt", + ), # LINE_BENE_PTB_DDCTBL_AMT, REV_CNTR_CASH_DDCTBL_AMT + "CLM_LINE_NCVRD_CHRG_AMT": ( + "clm_line_ncvrd_chrg_amt", + "rev_cntr_ncvrd_chrg_amt", + ), # REV_CNTR_NCVRD_CHRG_AMT + "CLM_LINE_INSTNL_RATE_AMT": ("rev_cntr_rate_amt",), # REV_CNTR_RATE_AMT + "CLM_LINE_BENE_PD_AMT": ( + "clm_line_bene_pd_amt", + "rev_cntr_bene_pmt_amt", + ), # LINE_BENE_PMT_AMT, REV_CNTR_BENE_PMT_AMT + "CLM_LINE_BLOOD_DDCTBL_AMT": ( + "rev_cntr_blood_ddctbl_amt", + ), # REV_CNTR_BLOOD_DDCTBL_AMT + "CLM_LINE_INSTNL_ADJSTD_AMT": ( + "rev_cntr_coinsrnc_wge_adjstd_c", + ), # REV_CNTR_COINSRNC_WGE_ADJSTD_C + "CLM_LINE_INSTNL_RDCD_AMT": ( + "rev_cntr_rdcd_coinsrnc_amt", + ), # REV_CNTR_RDCD_COINSRNC_AMT + "CLM_LINE_INSTNL_MSP1_PD_AMT": ( + "rev_cntr_1st_msp_pd_amt", + ), # REV_CNTR_1ST_MSP_PD_AMT + "CLM_LINE_INSTNL_MSP2_PD_AMT": ( + "rev_cntr_2nd_msp_pd_amt", + ), # REV_CNTR_2ND_MSP_PD_AMT + "TOT_RX_CST_AMT": ("tot_rx_cst_amt",), + "CLM_LINE_GRS_ABOVE_THRSHLD_AMT": ( + "clm_line_grs_above_thrshld_amt", + ), # GDC_ABV_OOPT_AMT + "CLM_LINE_GRS_BLW_THRSHLD_AMT": ( + "clm_line_grs_blw_thrshld_amt", + ), # GDC_BLW_OOPT_AMT + "CLM_LINE_LIS_AMT": ("clm_line_lis_amt",), # LICS_AMT + "CLM_LINE_PLRO_AMT": ("clm_line_plro_amt",), # PLRO_AMT + "CLM_LINE_RPTD_GAP_DSCNT_AMT": ( + "clm_line_rptd_gap_dscnt_amt", + ), # RPTD_GAP_DSCNT_NUM + "CLM_LINE_NCVRD_PD_AMT": ("clm_line_ncvrd_pd_amt",), # NCVRD_PLAN_PD_AMT + "CLM_LINE_PRFNL_DME_PRICE_AMT": ( + "clm_line_prfnl_dme_price_amt", + ), # LINE_DME_PRCHS_PRICE_AMT + "CLM_LINE_DMERC_SCRN_SVGS_AMT": ( + "clm_line_dmerc_scrn_svgs_amt", + ), # DMERC_LINE_SCRN_SVGS_AMT + "CLM_LINE_ADD_ON_PYMT_AMT": ("clm_line_add_on_pymt_amt",), + "CLM_REV_CNTR_TDAPA_AMT": ("clm_rev_cntr_tdapa_amt",), + "CLM_LINE_CARR_CLNCL_CHRG_AMT": ("clm_line_carr_clncl_chrg_amt",), + "CLM_LINE_CARR_PSYCH_OT_LMT_AMT": ("clm_line_carr_psych_ot_lmt_amt",), + "CLM_LINE_PRFNL_INTRST_AMT": ("clm_line_prfnl_intrst_amt",), + "CLM_MDCR_PRMRY_PYR_ALOWD_AMT": ("clm_mdcr_prmry_pyr_alowd_amt",), +} +# item.extension StructureDefinition code → EobItem column (ccwMapping) +_ITEM_EXT = { + "CLM-REV-CNTR-STUS-CD": "rev_cntr_stus_ind_cd", # REV_CNTR_STUS_IND_CD + "CLM-FED-TYPE-SRVC-CD": "clm_fed_type_srvc_cd", # LINE_CMS_TYPE_SRVC_CD + "CLM-PRCNG-LCLTY-CD": "carr_line_prcng_lclty_cd", # CARR_LINE_PRCNG_LCLTY_CD + "CLM-LINE-PRFNL-MTUS-CNT": "carr_line_mtus_cnt", # CARR_LINE_MTUS_CNT + "CLM-MTUS-IND-CD": "carr_line_mtus_cd", # CARR_LINE_MTUS_CD + "CLM-PMT-80-100-CD": "line_pmt_80_100_cd", # LINE_PMT_80_100_CD + "CLM-PRMRY-PYR-CD": "clm_line_prmry_pyr_cd", # LINE_BENE_PRMRY_PYR_CD + "CLM-PRCSG-IND-CD": "clm_prcsg_ind_cd", # LINE_PRCSG_IND_CD + "CLM-SRVC-DDCTBL-SW": "line_service_deductible", # LINE_SERVICE_DEDUCTIBLE + "CLM-PRVDR-TAX-NUM": "tax_num", + "CLM-REV-DSCNT-IND-CD": "clm_rev_dscnt_ind_cd", # REV_CNTR_DSCNT_IND_CD + "CLM-OTAF-ONE-IND-CD": "clm_otaf_one_ind_cd", # REV_CNTR_OTAF_PMT_CD + "CLM-REV-PACKG-IND-CD": "clm_rev_packg_ind_cd", # REV_CNTR_PACKG_IND_CD + "CLM-REV-PMT-MTHD-CD": "clm_rev_pmt_mthd_cd", # REV_CNTR_PMT_MTHD_IND_CD + "CLM-LINE-ANSTHSA-UNIT-CNT": "clm_line_ansthsa_unit_cnt", # CARR_LINE_ANSTHSA_UNIT_CNT + "CLM-PHYSN-ASTNT-CD": "clm_physn_astnt_cd", # CARR_LINE_RDCD_PMT_PHYS_ASTN_C + "CLM-LINE-CARR-HPSA-SCRCTY-CD": "clm_line_carr_hpsa_scrcty_cd", # HPSA_SCRCTY_IND_CD + "CLM-RNDRG-PRVDR-PRTCPTG-CD": "clm_rndrg_prvdr_prtcptg_cd", # PRTCPTNG_IND_CD + "CLM-PRVDR-TYPE-CD": "clm_rndrg_prvdr_type_cd", # CARR_LINE_PRVDR_TYPE_CD + "CLM-PRVDR-SPCLTY-CD": "clm_prvdr_spclty_cd", # PRVDR_SPCLTY +} +_ITEM_EXT_NUMERIC = {"carr_line_mtus_cnt", "clm_line_ansthsa_unit_cnt"} + + +def _product_codes( + item: dict, +) -> tuple[str | None, str | None, str | None, Any, str | None]: + """(hcpcs, ndc, hipps, ndc_qty, ndc_unit) from item/detail productOrService.""" + pos = item.get("productOrService") or {} + hcpcs = None + for sys_ in HCPCS_SYSTEMS: + hcpcs = hcpcs or _coding(pos, sys_) + hipps = _coding(pos, HIPPS) + ndc = _coding(pos, NDC) + qty = item.get("quantity") or {} + ndc_qty, ndc_unit = (qty.get("value"), qty.get("unit")) if ndc else (None, None) + for det in item.get("detail", []) or []: + dndc = _coding(det.get("productOrService"), NDC) + if dndc: + ndc = ndc or dndc + dq = det.get("quantity") or {} + ndc_qty = dq.get("value", ndc_qty) + ndc_unit = dq.get("unit", ndc_unit) + break + return hcpcs, ndc, hipps, ndc_qty, ndc_unit + + +def _flatten_item( + item: dict, clm_id: str | None, dx_by_seq: dict[int, str | None] +) -> EobItem: + kwargs: dict[str, Any] = {} + adjs = item.get("adjudication", []) or [] + exts = item.get("extension", []) or [] + + for code, cols in _ITEM_ADJ.items(): + amt = _adj_amount(adjs, code) + if amt is not None: + for col in cols: + kwargs[col] = amt + for code, col in _ITEM_EXT.items(): + v = _ext(exts, code) + if v is not None: + kwargs[col] = _num(v) if col in _ITEM_EXT_NUMERIC else str(v) + + # ANSI reason / group codes in adjudication order (CLM_n_REV_CNTR_ANSI_*) + n = 0 + for adj in adjs: + rsn = _coding(adj.get("reason"), CS_ANSI_RSN) + grp = _coding(adj.get("reason"), CS_ANSI_GRP) + if rsn is None and grp is None: + continue + n += 1 + if n > 4: + break + kwargs[f"clm_{n}_rev_cntr_ansi_rsn_cd"] = rsn + kwargs[f"clm_{n}_rev_cntr_ansi_grp_cd"] = grp + + serviced = item.get("servicedPeriod") or {} + from_dt = _date(item.get("servicedDate") or serviced.get("start")) + thru_dt = _date(serviced.get("end") or item.get("servicedDate")) + hcpcs, ndc, hipps, ndc_qty, ndc_unit = _product_codes(item) + mods = [_coding(m) for m in (item.get("modifier") or [])[:5]] + mods += [None] * (5 - len(mods)) + qty = _num(_get(item, "quantity", "value")) + dx_seq = (item.get("diagnosisSequence") or [None])[0] + + return EobItem( + clm_id=clm_id, + clm_line_num=_num(item.get("sequence")), # CLM_LINE_NUM + clm_line_from_dt=from_dt, # CLM_LINE_FROM_DT + clm_line_thru_dt=thru_dt, # CLM_LINE_THRU_DT + clm_line_instnl_rev_ctr_dt=from_dt, # CLM_LINE_INSTNL_REV_CTR_DT + clm_line_prod_rev_ctr_cd=_coding(item.get("revenue"), CS + "CLM-REV-CNTR-CD") + or _coding(item.get("revenue")), # CLM_LINE_REV_CTR_CD + clm_rev_apc_hipps_cd=hipps, # CLM_REV_APC_HIPPS_CD + clm_line_hcpcs_cd=hcpcs, # CLM_LINE_HCPCS_CD + clm_line_ndc_cd=ndc, # CLM_LINE_NDC_CD + clm_line_ndc_qty=_num(ndc_qty), + clm_line_ndc_qty_qlfyr_cd=ndc_unit, + hcpcs_1_mdfr_cd=mods[0], + hcpcs_2_mdfr_cd=mods[1], + hcpcs_3_mdfr_cd=mods[2], + hcpcs_4_mdfr_cd=mods[3], + hcpcs_5_mdfr_cd=mods[4], + hcpcs_betos_cd=None, # removed in v3 + clm_pos_cd=_coding(item.get("locationCodeableConcept"), POS) + or _coding(item.get("locationCodeableConcept")), # CLM_POS_CD + clm_line_srvc_unit_qty=qty, # CLM_LINE_SRVC_UNIT_QTY + rev_cntr_unit_cnt=qty, # REV_CNTR_UNIT_CNT (same dictionary row) + clm_line_dgns_cd=dx_by_seq.get(dx_seq) + if dx_seq is not None + else None, # CLM_LINE_DGNS_CD + **kwargs, + ) + + +def flatten_eob(resource: dict) -> tuple[ExplanationOfBenefit, list[EobItem]]: + """Flatten a v3 EOB (any C4BB 2.2.0 EOB profile) into header + items.""" + identifiers = resource.get("identifier", []) or [] + infos = resource.get("supportingInfo", []) or [] + adjs = resource.get("adjudication", []) or [] + totals = resource.get("total", []) or [] + items = resource.get("item", []) or [] + kwargs: dict[str, Any] = {} + + clm_id = _identifier_typed(identifiers, C4BB_ID_TYPE, "uc") # CLM_UNIQ_ID + clm_cntl_num = _identifier(identifiers, ID_CLM_CNTL) # CLM_CNTL_NUM + eob_type = _coding(resource.get("type"), CS_EOB_TYPE) + + # Original control number: related claim reference under CLM-CNTL-NUM + orig = None + for rel in resource.get("related", []) or []: + ref = rel.get("reference") or {} + if ref.get("system") == ID_CLM_CNTL: + orig = ref.get("value") + kwargs["clm_orig_cntl_num"] = orig + + # supportingInfo + for code, col in _SI_CODES.items(): + v = _si_code(infos, code) + if v is not None: + kwargs[col] = v + for code, col in _SI_DATES.items(): + v = _si_date(infos, code) + if v is not None: + kwargs[col] = v + for code, col in _SI_STRINGS.items(): + v = _si_value(infos, code) + if v is not None: + kwargs[col] = str(v) + for code, (col, code_sys) in _SI_C4BB_CODES.items(): + v = _si_code(infos, code, system=C4BB_SI, code_system=code_sys) + if v is None and code_sys: + v = _si_code(infos, code, system=C4BB_SI) + if v is not None: + kwargs[col] = v + for code, col in _SI_C4BB_QUANTITIES.items(): + v = _si_value(infos, code, system=C4BB_SI) + if v is not None: + kwargs[col] = _num(v) + tob = _si(infos, "typeofbill", C4BB_SI) + if tob: + for code_sys, cols in _TYPEOFBILL.items(): + v = _coding(tob.get("code"), CS + code_sys) + if v is not None: + for col in cols: + kwargs[col] = v + if kwargs.get("bene_ptnt_stus_cd") is None: + kwargs["bene_ptnt_stus_cd"] = _si_code_by_system( + infos, CS + "CLM-MDCR-NCH-PTNT-STUS-IND-CD" + ) + if kwargs.get("clm_op_srvc_type_cd") is None: + kwargs["clm_op_srvc_type_cd"] = _si_code_by_system( + infos, CS + "CLM-OP-SRVC-TYPE-CD" + ) + if kwargs.get("nch_near_line_rec_ident_cd") is None: + kwargs["nch_near_line_rec_ident_cd"] = _si_code_by_system( + infos, CS + "CLM-NRLN-RIC-CD" + ) + blood = _si_value(infos, "CLM_BLOOD_PT_FRNSH_QTY") # NCH_BLOOD_PNTS_FRNSHD_QTY + if blood is not None: + kwargs["nch_blood_pnts_frnshd_qty"] = _num(blood) + admit_start, admit_end = _si_period(infos, "admissionperiod", C4BB_SI) + kwargs["clm_actv_care_from_dt"] = ( + admit_start # CLM_ACTV_CARE_FROM_DT / CLM_ADMSN_DT + ) + kwargs["clm_dschrg_dt"] = admit_end # CLM_DSCHRG_DT + kwargs["clm_submsn_dt"] = _si_date(infos, "clmrecvddate", C4BB_SI) # CLM_SUBMSN_DT + cntrctr = _si_code(infos, "CLM_CNTRCTR_NUM") # FI_NUM / CARR_NUM + kwargs["fi_num"] = cntrctr + kwargs["carr_num"] = cntrctr + kwargs["carr_clm_entry_cd"] = kwargs.get( + "clm_query_cd" + ) # ccwMapping CARR_CLM_ENTRY_CD + + # totals / header adjudication + for code, cols in _TOTALS.items(): + amt = _adj_amount(totals, code) + if amt is not None: + for col in cols: + kwargs[col] = amt + for code, cols in _ADJ_HEADER.items(): + v = ( + _adj_quantity(adjs, code) + if code in _ADJ_VALUES + else _adj_amount(adjs, code) + ) + if v is not None: + for col in cols: + kwargs[col] = v + bps = _adj_by_code(adjs, "benefitpaymentstatus", C4BB_ADJ_DISC) + kwargs["clm_prcng_excptn_cd"] = ( + _coding(bps.get("reason")) if bps else None + ) # PRCNG_EXCPTN_CD + + # billing provider (contained Organization referenced by provider) + org = _contained(resource, _get(resource, "provider", "reference")) + if org: + ids = org.get("identifier", []) or [] + npi = _identifier( + ids, US_NPI + ) # PRVDR_BLG_PRVDR_NPI_NUM → ORG_NPI_NUM, CARR_CLM_BLG_NPI_NUM + kwargs["carr_clm_blg_npi_num"] = npi + kwargs["fac_prvdr_npi_num"] = npi + kwargs["prvdr_oscar_num"] = _identifier(ids, CCN) # CLM_BLG_PRVDR_OSCAR_NUM + kwargs["clm_blg_prvdr_tax_num"] = _identifier(ids, TAX_ID) + kwargs["clm_srvc_prvdr_gnrc_id_num"] = _identifier( + ids, GENERIC_ID + ) # CLM_SRVC_PRVDR_GNRC_ID_NUM + kwargs["clm_blg_prvdr_name"] = _display_name(org.get("name")) + kwargs["clm_blg_prvdr_zip5_cd"] = _get( + (org.get("address") or [{}])[0], "postalCode" + ) + + # care team + ct = _care_team(resource.get("careTeam")) + kwargs["atndg_prvdr_npi_num"] = _npi( + (ct.get("attending") or [None])[0] + ) # AT_PHYSN_NPI + kwargs["oprtg_prvdr_npi_num"] = _npi( + (ct.get("operating") or [None])[0] + ) # OP_PHYSN_NPI + kwargs["othr_prvdr_npi_num"] = _npi( + (ct.get("otheroperating") or ct.get("supervisor") or [None])[0] + ) # OT_PHYSN_NPI + rendering = (ct.get("rendering") or [None])[0] + kwargs["rndrg_prvdr_npi_num"] = _npi(rendering) # RNDRNG_PHYSN_NPI + kwargs["clm_prvdr_spclty_cd"] = ( + _coding(_get(rendering, "qualification"), CS + "CLM-PRVDR-SPCLTY-CD") + if rendering + else None + ) # PRVDR_SPCLTY + kwargs["ordrg_prvdr_npi_num"] = _npi( + (ct.get("referring") or [None])[0] + ) # RFR_PHYSN_NPI + prescribing = (ct.get("prescribing") or [None])[0] + kwargs["clm_prsbng_prvdr_gnrc_id_num"] = _provider_value(prescribing) # PRSCRBR_ID + service = (ct.get("service") or [None])[0] + if service and not kwargs.get("clm_srvc_prvdr_gnrc_id_num"): + kwargs["clm_srvc_prvdr_gnrc_id_num"] = _provider_value(service) + + # line-level provider attributes the v2 model kept on the header + first_ext = (items[0].get("extension") if items else None) or [] + kwargs["clm_rndrg_prvdr_tax_num"] = _ext_str(first_ext, "CLM-PRVDR-TAX-NUM") + kwargs["rndrg_prvdr_type_cd"] = _ext_str(first_ext, "CLM-PRVDR-TYPE-CD") + kwargs["carr_line_prvdr_type_cd"] = kwargs["rndrg_prvdr_type_cd"] + kwargs["prtcptng_ind_cd"] = _ext_str(first_ext, "CLM-RNDRG-PRVDR-PRTCPTG-CD") + + # diagnoses / first procedure + dx_list = resource.get("diagnosis", []) or [] + kwargs.update(_diagnoses(dx_list)) + procs = resource.get("procedure", []) or [] + if procs: + kwargs["clm_val_sqnc_num"] = str(procs[0].get("sequence", "")) or None + kwargs["clm_prcdr_cd"] = _coding( + procs[0].get("procedureCodeableConcept") + ) # CLM_PRCDR_CD + kwargs["clm_prcdr_prfrm_dt"] = _date(procs[0].get("date")) # CLM_PRCDR_PRFRM_DT + + billable = resource.get("billablePeriod") or {} + header = ExplanationOfBenefit( + id=resource.get("id"), + last_updated=_get(resource, "meta", "lastUpdated"), + bene_id=_ref_id(_get(resource, "patient", "reference")), # BENE_SK on claim + clm_id=clm_id, + clm_uniq_id=clm_id, + pde_id=clm_id if eob_type == "PDE" else None, # ccwMapping PDE_ID (Pharmacy) + clm_grp_id=None, + clm_sub_type=_coding(resource.get("subType")), + clm_type_cd=_coding(resource.get("type"), CS_CLM_TYPE), # CLM_TYPE_CD + eob_type=eob_type, + system_type=_tag(resource, CS_SYSTEM_TYPE), + final_action=_tag(resource, CS_FINAL_ACTION), # CLM_FINL_ACTN_IND + clm_src_id=_get(resource, "meta", "source"), # CLM_SRC_ID + outcome=resource.get("outcome"), + clm_from_dt=_date(billable.get("start")), # CLM_FROM_DT + clm_thru_dt=_date(billable.get("end")), # CLM_THRU_DT + clm_efctv_dt=_date(resource.get("created")), # CLM_EFCTV_DT + clm_pmt_amt=_num(_get(resource, "payment", "amount", "value")), # CLM_PMT_AMT + clm_pd_dt=_date(_get(resource, "payment", "date")), # CLM_PD_DT + clm_cntl_num=clm_cntl_num, + fi_doc_clm_cntl_num=clm_cntl_num, # ccwMapping FI_DOC_CLM_CNTL_NUM + carr_clm_cntl_num=clm_cntl_num, # ccwMapping CARR_CLM_CNTL_NUM + **kwargs, + ) + dx_by_seq = _diagnosis_by_sequence(dx_list) + return header, [_flatten_item(it, clm_id, dx_by_seq) for it in items] diff --git a/src/bcda/pipe/cclf.py b/src/bcda/pipe/cclf.py index 4a2d23c..e4e4481 100644 --- a/src/bcda/pipe/cclf.py +++ b/src/bcda/pipe/cclf.py @@ -21,7 +21,7 @@ Pipeline expressions (12 total):: stg_beneficiary_demographics Patient + Coverage eligibility demographics → eligibility -Source data — BCDA v2 bulk FHIR export: +Source data — BCDA bulk FHIR export (v2 or v3, flattened to bcda.*): - ExplanationOfBenefit (header + items) - Patient - Coverage diff --git a/src/bcda/store.py b/src/bcda/store.py index 6e7200b..dd2dc51 100644 --- a/src/bcda/store.py +++ b/src/bcda/store.py @@ -19,7 +19,7 @@ the appropriate extra (``s3fs``, ``adlfs``) installed by the user. Usage:: - from bcda.client import Client + from bcda.client import Client, JobCancelledError from bcda.store import Store store = Store() @@ -51,7 +51,7 @@ import fsspec import pyarrow as pa import pyarrow.parquet as pq -from bcda.client import Client +from bcda.client import Client, JobCancelledError log = logging.getLogger(__name__) @@ -264,7 +264,9 @@ class Store: endpoint: str = "Group/all", types: list[str] | None = None, since: str | None = None, + type_filter: str | None = None, poll_interval: float = 10.0, + cancel_retries: int = 0, ) -> str: """Run a full export with persistent state tracking. @@ -283,11 +285,22 @@ class Store: FHIR resource types to request. since : str | None ``_since`` timestamp filter. + type_filter : str | None + v3 ``_typeFilter`` subquery (recorded alongside ``types``). poll_interval : float Seconds between status polls. + cancel_retries : int + How many times to restart the export after the server + reports it cancelled. On the public sandbox every caller + of a model entity can ``DELETE`` any job, and sweeps that + cancel every in-progress job happen several times an hour; + each cancellation is recorded as a ``cancelled`` transition + under the same key before the next attempt. """ key = _generate_key() types_str = ",".join(types) if types else "" + if type_filter: + types_str = f"{types_str}|_typeFilter={type_filter}" log.info("Starting export %s", key, extra={"job_key": key}) # 1. Record pending. @@ -300,23 +313,47 @@ class Store: ) try: - # 2. Start export. - job_url = client.start_export( - endpoint=endpoint, - types=types, - since=since, - ) - self._append_job( - key=key, - job_url=job_url, - endpoint=endpoint, - types=types_str, - since=since or "", - status="polling", - ) + for attempt in range(cancel_retries + 1): + # 2. Start export. + job_url = client.start_export( + endpoint=endpoint, + types=types, + since=since, + type_filter=type_filter, + ) + self._append_job( + key=key, + job_url=job_url, + endpoint=endpoint, + types=types_str, + since=since or "", + status="polling", + ) - # 3. Poll until complete. - result = client.poll(job_url, interval=poll_interval) + # 3. Poll until complete. + try: + result = client.poll(job_url, interval=poll_interval) + break + except JobCancelledError as exc: + self._append_job( + key=key, + job_url=job_url, + endpoint=endpoint, + types=types_str, + since=since or "", + status="cancelled", + error=str(exc), + ) + if attempt >= cancel_retries: + raise + log.warning( + "Export %s cancelled server-side (%s); restarting (%d/%d)", + key, + job_url, + attempt + 1, + cancel_retries, + extra={"job_key": key}, + ) self._append_job( key=key, job_url=job_url, @@ -349,6 +386,60 @@ class Store: log.info("Export %s finished", key, extra={"job_key": key}) return key + def adopt( + self, + client: Client, + job_url: str, + *, + endpoint: str = "Group/all", + types: list[str] | None = None, + poll_interval: float = 10.0, + ) -> str: + """Record and download a job this store did not start. + + BCDA keeps a completed job's files for 24 hours, and on the + public sandbox every caller of a model entity sees the same + job list — so a fresh export (6 min and 2 GB for the Large + entity's EOBs) is wasted when ``client.completed_jobs()`` + already lists one. ``adopt`` polls *job_url* (returns at once + when complete), appends the job under a new key with status + ``complete`` and downloads its outputs exactly as + :meth:`export` would. Returns the job key. + """ + key = _generate_key() + types_str = ",".join(types) if types else "" + log.info("Adopting job %s as %s", job_url, key, extra={"job_key": key}) + self._append_job( + key=key, + job_url=job_url, + endpoint=endpoint, + types=types_str, + status="polling", + ) + try: + result = client.poll(job_url, interval=poll_interval) + self._append_job( + key=key, + job_url=job_url, + endpoint=endpoint, + types=types_str, + status="complete", + ) + self._download_outputs(client, key, result) + except Exception as exc: + log.error("Adopt %s failed: %s", key, exc, extra={"job_key": key}) + self._append_job( + key=key, + job_url=job_url, + endpoint=endpoint, + types=types_str, + status="failed", + error=str(exc), + ) + raise + log.info("Adopted %s finished", key, extra={"job_key": key}) + return key + def _download_outputs( self, client: Client, diff --git a/src/bcda/table/coverage.py b/src/bcda/table/coverage.py index 7f9397d..ff63bac 100644 --- a/src/bcda/table/coverage.py +++ b/src/bcda/table/coverage.py @@ -1,6 +1,6 @@ """FHIR R4 Coverage resource — beneficiary enrollment and entitlement. -Flattened from the BCDA v2 Coverage ndjson export. Each beneficiary +Flattened from the BCDA Coverage ndjson export (v2 or v3). Each beneficiary may have multiple Coverage records (Part A, Part B, Part D). The Coverage resource carries enrollment status, dual eligibility, @@ -9,7 +9,14 @@ extensions for buy-in, HMO, cost sharing, Part C/D contracts, and Retiree Drug Subsidy. Each monthly series has 12 fields (01–12) representing January through December of the reference year. -Source: ``/api/v2/Coverage/$export`` bulk FHIR endpoint. +v3 delivers one Coverage per plan per period (``class_plan`` ∈ Part A, +Part B, Part C, Part D, Dual Medicare/Medicaid) with ``Coverage.period`` +instead of the v2 reference-year monthly series; the v3 extensions land +in the ``*_cd`` columns below and the legacy part A/B/D begin/end dates +are filled from the matching plan's period so downstream joins keep +working. + +Source: ``/api/{v2,v3}/Coverage/$export`` bulk FHIR endpoint. """ from __future__ import annotations @@ -563,3 +570,74 @@ class Coverage(SQLTable): ma_12: str | None = None """Medicare Advantage indicator — December.""" + + # ── v3-only (BCDA v3 Data Dictionary) ──────────────────── + period_start: date | None = None + """Coverage.period.start — enrollment / eligibility begin for this + plan row (IDR BENE_RNG_BGN_DT, BENE_ENRLMT_BGN_DT, BENE_MDCD_ELGBLTY_BGN_DT).""" + + period_end: date | None = None + """Coverage.period.end (IDR BENE_RNG_END_DT, BENE_ENRLMT_END_DT, + BENE_MDCD_ELGBLTY_END_DT).""" + + buyin_cd: str | None = None + """Buy-in code (extension BENE-BUYIN-CD; CCW MDCR_ENTLMT_BUYIN_*_IND).""" + + entlmt_stus_cd: str | None = None + """Medicare entitlement status (BENE-MDCR-ENTLMT-STUS-CD; CCW + BENE_PTA/PTB_TRMNTN_CD — also copied to ``a_trm_cd``/``b_trm_cd``).""" + + enrlmt_rsn_cd: str | None = None + """Enrollment reason (BENE-ENRLMT-RSN-CD).""" + + dsbld_stus_id: str | None = None + """Disabled status (BENE-DSBLD-STUS-ID).""" + + dual_stus_cd: str | None = None + """Dual status (BENE-DUAL-STUS-CD) on the Dual Medicare/Medicaid + row; CCW DUAL_STUS_CD_01..12.""" + + dual_type_cd: str | None = None + """Medicare/Medicaid dual type (BENE-DUAL-TYPE-CD).""" + + medicaid_state_cd: str | None = None + """Medicaid state, USPS (MEDICAID-STATE-CD).""" + + cntrct_num: str | None = None + """Part C/D contract number (BENE-CNTRCT-NUM; CCW PTC/PTD_CNTRCT_ID_*).""" + + pbp_num: str | None = None + """Part C/D plan benefit package (BENE-PBP-NUM; CCW PTC/PTD_PBP_ID_*).""" + + cvrg_type_cd: str | None = None + """Part C/D coverage type (BENE-CVRG-TYPE-CD; CCW PTC_PLAN_TYPE_CD_*).""" + + pbp_sgmt_num: str | None = None + """Part D PBP segment (CNTRCT-PBP-SGMT-NUM; CCW PTD_SGMT_ID_*).""" + + lis_efctv_cd: str | None = None + """Part D LIS effective code (BENE-LIS-EFCTV-CD; CCW CST_SHR_GRP_CD_*).""" + + lis_copmt_lvl_cd: str | None = None + """Part D LIS copayment level (BENE-LIS-COPMT-LVL-CD).""" + + lis_ptd_prm_pct: str | None = None + """Part D LIS premium subsidy percentage (BENE-LIS-PTD-PRM-PCT).""" + + rds_sw: str | None = None + """Retiree drug subsidy switch (BENE-ENRLMT-EMPLR-SBSDY-SW; CCW RDS_IND_*).""" + + rx_id: str | None = None + """Part D PDP member id (Coverage.class[rxid]).""" + + rx_group: str | None = None + """Part D PDP group (Coverage.class[rxgroup]).""" + + rx_pcn: str | None = None + """Part D PDP processor control number (Coverage.class[rxpcn]).""" + + rx_bin: str | None = None + """Part D PDP bank id (Coverage.class[rxbin]).""" + + cntrct_pbp_name: str | None = None + """Part C/D contract plan name (contained Organization.name).""" diff --git a/src/bcda/table/eob.py b/src/bcda/table/eob.py index 8e6171f..9508f5f 100644 --- a/src/bcda/table/eob.py +++ b/src/bcda/table/eob.py @@ -1,6 +1,6 @@ """FHIR R4 ExplanationOfBenefit resource — adjudicated claims. -Flattened from the BCDA v2 ExplanationOfBenefit ndjson export. +Flattened from the BCDA ExplanationOfBenefit ndjson export (v2 or v3). Covers Part A (institutional), Part B (professional/DME), and Part D (pharmacy) claims in a single resource type. @@ -10,7 +10,14 @@ fields (line items with revenue codes, HCPCS, adjudication amounts). This model represents the header-level flattened fields; item-level arrays are modeled separately in ``EobItem``. -Source: ``/api/v2/ExplanationOfBenefit/$export`` bulk FHIR endpoint. +v3 represents partially *and* fully adjudicated claims as EOBs (the +``system_type`` tag says which: SharedSystem, NationalClaimsHistory, +DDPS) and consolidates the v2 benefitBalance/extension amounts into +``adjudication``/``total``/``item.adjudication`` under the +``https://bluebutton.cms.gov/fhir/CodeSystem/Adjudication`` system. +The v3-only columns below carry those amounts under their IDR names. + +Source: ``/api/{v2,v3}/ExplanationOfBenefit/$export`` bulk FHIR endpoint. """ from __future__ import annotations @@ -360,6 +367,244 @@ class ExplanationOfBenefit(SQLTable): """Pharmacy service type code (Eob.facility extension valueCoding).""" + # ── v3-only: identity and provenance (BCDA v3 Data Dictionary) ── + clm_uniq_id: str | None = None + """IDR claim surrogate key (identifier[C4BB uc]; IDR CLM_UNIQ_ID). + Same value as ``clm_id`` on v3 rows.""" + + clm_orig_cntl_num: str | None = None + """Original claim control number for adjusted claims + (related[].reference where system CLM-CNTL-NUM).""" + + system_type: str | None = None + """meta.tag System-Type: SharedSystem (partially adjudicated), + NationalClaimsHistory, DDPS.""" + + final_action: str | None = None + """meta.tag Final-Action: FinalAction / NotFinalAction.""" + + eob_type: str | None = None + """BlueButton EOB-TYPE coding: INPATIENT, OUTPATIENT, SNF, HHA, + HOSPICE, CARRIER, DME, PDE.""" + + clm_src_id: str | None = None + """meta.source (IDR CLM_SRC_ID / META_SRC_SK): NCH, DDPS, FISS, MCS, VMS.""" + + outcome: str | None = None + """ExplanationOfBenefit.outcome: complete / partial.""" + + clm_adjstmt_type_cd: str | None = None + """Claim adjustment type (supportingInfo CLM_ADJSTMT_TYPE_CD).""" + + clm_idr_ld_dt: date | None = None + """IDR load date (supportingInfo CLM_IDR_LD_DT).""" + + clm_cms_proc_dt: date | None = None + """CMS process date (supportingInfo CLM_CMS_PROC_DT).""" + + clm_nch_wkly_proc_dt: date | None = None + """NCH weekly process date (supportingInfo CLM_NCH_WKLY_PROC_DT).""" + + clm_submsn_dt: date | None = None + """Claim received date (supportingInfo clmrecvddate).""" + + clm_pd_dt: date | None = None + """Payment date (payment.date; IDR CLM_PD_DT).""" + + clm_sbmt_frmt_cd: str | None = None + """Submission format (supportingInfo CLM_SBMT_FRMT_CD).""" + + clm_blg_prvdr_name: str | None = None + """Billing provider name (contained Organization.name).""" + + clm_blg_prvdr_tax_num: str | None = None + """Billing provider tax number (contained Organization identifier + urn:oid:2.16.840.1.113883.4.4).""" + + clm_blg_prvdr_zip5_cd: str | None = None + """Billing provider ZIP (contained Organization.address.postalCode).""" + + clm_dschrg_dt: date | None = None + """Discharge date (supportingInfo admissionperiod end).""" + + clm_actv_care_thru_dt: date | None = None + """Covered care through date (supportingInfo CLM_ACTV_CARE_THRU_DT).""" + + clm_mdcr_exhstd_dt: date | None = None + """Medicare benefits exhausted date (supportingInfo CLM_MDCR_EXHSTD_DT).""" + + clm_ncvrd_from_dt: date | None = None + """Noncovered stay from (supportingInfo CLM_NCVRD_FROM_DT).""" + + clm_ncvrd_thru_dt: date | None = None + """Noncovered stay through (supportingInfo CLM_NCVRD_THRU_DT).""" + + clm_qlfy_stay_from_dt: date | None = None + """Qualified stay from (supportingInfo CLM_QLFY_STAY_FROM_DT).""" + + clm_qlfy_stay_thru_dt: date | None = None + """Qualified stay through (supportingInfo CLM_QLFY_STAY_THRU_DT).""" + + clm_pps_ind_cd: str | None = None + """PPS indicator (supportingInfo CLM_PPS_IND_CD).""" + + clm_hha_lup_ind_cd: str | None = None + """HHA LUPA indicator (supportingInfo CLM_HHA_LUP_IND_CD).""" + + clm_hha_rfrl_cd: str | None = None + """HHA referral code (supportingInfo CLM_HHA_RFRL_CD).""" + + clm_clncl_tril_num: str | None = None + """Clinical trial number (supportingInfo CLM_CLNCL_TRIL_NUM).""" + + clm_prcng_excptn_cd: str | None = None + """Pricing exception / benefit payment status reason + (adjudication benefitpaymentstatus reason).""" + + # ── v3-only: header totals (ExplanationOfBenefit.total) ── + clm_sbmt_chrg_amt: float | None = None + """Total submitted charge (total CLM_SBMT_CHRG_AMT).""" + + clm_alowd_chrg_amt: float | None = None + """Allowed charge (total CLM_ALOWD_CHRG_AMT).""" + + clm_bene_pmt_amt: float | None = None + """Paid by beneficiary (total CLM_BENE_PMT_AMT).""" + + clm_prvdr_pmt_amt: float | None = None + """Paid to provider (total CLM_PRVDR_PMT_AMT).""" + + clm_mdcr_ddctbl_amt: float | None = None + """Deductible (total CLM_MDCR_DDCTBL_AMT).""" + + clm_mdcr_coinsrnc_amt: float | None = None + """Coinsurance liability (total CLM_MDCR_COINSRNC_AMT).""" + + clm_ncvrd_chrg_amt: float | None = None + """Noncovered charge (total CLM_NCVRD_CHRG_AMT).""" + + clm_othr_tp_pd_amt: float | None = None + """Other third party paid (total CLM_OTHR_TP_PD_AMT).""" + + clm_mdcr_instnl_bene_pd_amt: float | None = None + """Paid to patient, institutional (total CLM_MDCR_INSTNL_BENE_PD_AMT).""" + + tot_rx_cst_amt: float | None = None + """Total drug cost, Part D (total TOT_RX_CST_AMT).""" + + # ── v3-only: header adjudication (ExplanationOfBenefit.adjudication) ── + clm_instnl_cvrd_day_cnt: float | None = None + """Utilization day count (CLM_INSTNL_CVRD_DAY_CNT).""" + + clm_mdcr_ip_lrd_use_cnt: float | None = None + """Lifetime reserve days used (CLM_MDCR_IP_LRD_USE_CNT).""" + + clm_instnl_mdcr_coins_day_cnt: float | None = None + """Coinsurance day count (CLM_INSTNL_MDCR_COINS_DAY_CNT).""" + + clm_instnl_ncvrd_day_cnt: float | None = None + """Non-utilization day count (CLM_INSTNL_NCVRD_DAY_CNT).""" + + clm_instnl_per_diem_amt: float | None = None + """Pass-through per diem (CLM_INSTNL_PER_DIEM_AMT).""" + + clm_mdcr_ip_pps_drg_wt_num: float | None = None + """PPS capital DRG weight (CLM_MDCR_IP_PPS_DRG_WT_NUM).""" + + clm_mdcr_ip_pps_cptl_fsp_amt: float | None = None + """PPS capital FSP amount (CLM_MDCR_IP_PPS_CPTL_FSP_AMT).""" + + clm_mdcr_ip_pps_outlier_amt: float | None = None + """PPS capital outlier amount (CLM_MDCR_IP_PPS_OUTLIER_AMT).""" + + clm_mdcr_ip_pps_excptn_amt: float | None = None + """PPS capital exception amount (CLM_MDCR_IP_PPS_EXCPTN_AMT).""" + + clm_mdcr_ip_pps_cptl_hrmls_amt: float | None = None + """PPS old capital hold harmless (CLM_MDCR_IP_PPS_CPTL_HRMLS_AMT).""" + + clm_mdcr_ip_pps_cptl_tot_amt: float | None = None + """PPS total capital (CLM_MDCR_IP_PPS_CPTL_TOT_AMT).""" + + clm_instnl_drg_outlier_amt: float | None = None + """DRG outlier approved payment (CLM_INSTNL_DRG_OUTLIER_AMT).""" + + clm_mdcr_instnl_prmry_pyr_amt: float | None = None + """Institutional primary payer paid (CLM_MDCR_INSTNL_PRMRY_PYR_AMT).""" + + clm_mdcr_prfnl_prmry_pyr_amt: float | None = None + """Carrier primary payer paid (CLM_MDCR_PRFNL_PRMRY_PYR_AMT).""" + + clm_instnl_prfnl_amt: float | None = None + """Professional component charge (CLM_INSTNL_PRFNL_AMT).""" + + clm_blood_lblty_amt: float | None = None + """Blood deductible liability (CLM_BLOOD_LBLTY_AMT).""" + + clm_mdcr_ip_bene_ddctbl_amt: float | None = None + """Inpatient deductible (CLM_MDCR_IP_BENE_DDCTBL_AMT).""" + + clm_blood_chrg_amt: float | None = None + """Blood charge (CLM_BLOOD_CHRG_AMT).""" + + clm_blood_ncvrd_chrg_amt: float | None = None + """Blood noncovered charge (CLM_BLOOD_NCVRD_CHRG_AMT).""" + + clm_mdcr_hospc_prd_cnt: float | None = None + """Hospice period count (CLM_MDCR_HOSPC_PRD_CNT).""" + + clm_mdcr_hha_tot_visit_cnt: float | None = None + """HHA total visit count (CLM_MDCR_HHA_TOT_VISIT_CNT).""" + + clm_finl_stdzd_pymt_amt: float | None = None + """Final standardized payment (CLM_FINL_STDZD_PYMT_AMT).""" + + clm_hac_rdctn_pymt_amt: float | None = None + """HAC reduction (CLM_HAC_RDCTN_PYMT_AMT).""" + + clm_hipps_readmsn_rdctn_amt: float | None = None + """Readmission reduction (CLM_HIPPS_READMSN_RDCTN_AMT).""" + + clm_hipps_vbp_amt: float | None = None + """Value-based purchasing (CLM_HIPPS_VBP_AMT).""" + + clm_instnl_low_vol_pmt_amt: float | None = None + """Low volume payment (CLM_INSTNL_LOW_VOL_PMT_AMT).""" + + # ── v3-only: Part D supporting info ── + clm_brnd_gnrc_cd: str | None = None + """Brand/generic (supportingInfo brandgenericindicator).""" + + clm_cmpnd_cd: str | None = None + """Compound code (supportingInfo compoundcode).""" + + clm_line_rx_orgn_cd: str | None = None + """Rx origin (supportingInfo rxorigincode).""" + + clm_ptnt_rsdnc_cd: str | None = None + """Patient residence (supportingInfo CLM_PTNT_RSDNC_CD).""" + + clm_ltc_dspnsng_mthd_cd: str | None = None + """LTC dispensing method (supportingInfo CLM_LTC_DSPNSNG_MTHD_CD).""" + + clm_drug_cvrg_stus_cd: str | None = None + """Drug coverage status (supportingInfo CLM_DRUG_CVRG_STUS_CD).""" + + clm_ctstrphc_cvrg_ind_cd: str | None = None + """Catastrophic coverage indicator (supportingInfo CLM_CTSTRPHC_CVRG_IND_CD).""" + + clm_dspnsng_stus_cd: str | None = None + """Dispensing status (supportingInfo CLM_DSPNSNG_STUS_CD).""" + + clm_sbmtr_cntrct_num: str | None = None + """Submitter contract (supportingInfo CLM_SBMTR_CNTRCT_NUM).""" + + clm_sbmtr_cntrct_pbp_num: str | None = None + """Submitter PBP (supportingInfo CLM_SBMTR_CNTRCT_PBP_NUM).""" + + clm_line_authrzd_fill_num: float | None = None + """Authorized refills (supportingInfo refillsauthorized).""" + class EobItem(SQLTable): """FHIR R4 ExplanationOfBenefit.item — claim line items. @@ -571,3 +816,119 @@ class EobItem(SQLTable): tax_num: str | None = None """Line-level provider tax number (Eob.item.extension[tax_num] valueCoding).""" + + # ── v3-only (BCDA v3 Data Dictionary, item level) ──────── + clm_line_ndc_qty: float | None = None + """NDC quantity (item/detail quantity.value; IDR CLM_LINE_NDC_QTY).""" + + clm_line_ndc_qty_qlfyr_cd: str | None = None + """NDC quantity unit (item/detail quantity.unit).""" + + clm_line_dgns_cd: str | None = None + """Line diagnosis — header diagnosis at item.diagnosisSequence[0].""" + + clm_rev_dscnt_ind_cd: str | None = None + """Revenue center discount indicator (extension CLM-REV-DSCNT-IND-CD).""" + + clm_otaf_one_ind_cd: str | None = None + """Obligation to accept as full payment (extension CLM-OTAF-ONE-IND-CD).""" + + clm_rev_packg_ind_cd: str | None = None + """Packaging indicator (extension CLM-REV-PACKG-IND-CD).""" + + clm_rev_pmt_mthd_cd: str | None = None + """Payment method indicator (extension CLM-REV-PMT-MTHD-CD).""" + + clm_line_ansthsa_unit_cnt: float | None = None + """Anesthesia units (extension CLM-LINE-ANSTHSA-UNIT-CNT).""" + + clm_physn_astnt_cd: str | None = None + """Reduced payment physician assistant code (extension CLM-PHYSN-ASTNT-CD).""" + + clm_line_carr_hpsa_scrcty_cd: str | None = None + """HPSA scarcity (extension CLM-LINE-CARR-HPSA-SCRCTY-CD).""" + + clm_rndrg_prvdr_prtcptg_cd: str | None = None + """Rendering provider participating (extension CLM-RNDRG-PRVDR-PRTCPTG-CD).""" + + clm_rndrg_prvdr_type_cd: str | None = None + """Rendering provider type (extension CLM-PRVDR-TYPE-CD).""" + + clm_prvdr_spclty_cd: str | None = None + """Line provider specialty (extension CLM-PRVDR-SPCLTY-CD).""" + + clm_1_rev_cntr_ansi_rsn_cd: str | None = None + """Adjustment reason 1 (adjudication reason ANSI-RSN-CD).""" + + clm_1_rev_cntr_ansi_grp_cd: str | None = None + """Adjustment group 1 (adjudication reason ANSI-GRP-CD).""" + + clm_2_rev_cntr_ansi_rsn_cd: str | None = None + """Adjustment reason 2.""" + + clm_2_rev_cntr_ansi_grp_cd: str | None = None + """Adjustment group 2.""" + + clm_3_rev_cntr_ansi_rsn_cd: str | None = None + """Adjustment reason 3.""" + + clm_3_rev_cntr_ansi_grp_cd: str | None = None + """Adjustment group 3.""" + + clm_4_rev_cntr_ansi_rsn_cd: str | None = None + """Adjustment reason 4.""" + + clm_4_rev_cntr_ansi_grp_cd: str | None = None + """Adjustment group 4.""" + + # ── v3-only: item adjudication amounts (CodeSystem/Adjudication) ── + tot_rx_cst_amt: float | None = None + """Total drug cost, Part D (TOT_RX_CST_AMT).""" + + clm_line_grs_above_thrshld_amt: float | None = None + """Gross drug cost above OOP threshold (CLM_LINE_GRS_ABOVE_THRSHLD_AMT).""" + + clm_line_grs_blw_thrshld_amt: float | None = None + """Gross drug cost below OOP threshold (CLM_LINE_GRS_BLW_THRSHLD_AMT).""" + + clm_line_lis_amt: float | None = None + """Low income cost sharing subsidy (CLM_LINE_LIS_AMT).""" + + clm_line_plro_amt: float | None = None + """Patient liability reduction other paid (CLM_LINE_PLRO_AMT).""" + + clm_line_rptd_gap_dscnt_amt: float | None = None + """Reported gap discount (CLM_LINE_RPTD_GAP_DSCNT_AMT).""" + + clm_line_ncvrd_pd_amt: float | None = None + """Noncovered paid (CLM_LINE_NCVRD_PD_AMT).""" + + clm_line_bene_pd_amt: float | None = None + """Paid to beneficiary (CLM_LINE_BENE_PD_AMT).""" + + clm_line_mdcr_ddctbl_amt: float | None = None + """Part B / cash deductible (CLM_LINE_MDCR_DDCTBL_AMT).""" + + clm_line_prfnl_dme_price_amt: float | None = None + """DME purchase price (CLM_LINE_PRFNL_DME_PRICE_AMT).""" + + clm_line_dmerc_scrn_svgs_amt: float | None = None + """DMERC screen savings (CLM_LINE_DMERC_SCRN_SVGS_AMT).""" + + clm_line_add_on_pymt_amt: float | None = None + """Add-on payment (CLM_LINE_ADD_ON_PYMT_AMT).""" + + clm_rev_cntr_tdapa_amt: float | None = None + """Transitional drug add-on payment adjustment (CLM_REV_CNTR_TDAPA_AMT).""" + + clm_line_carr_clncl_chrg_amt: float | None = None + """Carrier clinical charge (CLM_LINE_CARR_CLNCL_CHRG_AMT).""" + + clm_line_carr_psych_ot_lmt_amt: float | None = None + """Therapy amount applied to limit (CLM_LINE_CARR_PSYCH_OT_LMT_AMT).""" + + clm_line_prfnl_intrst_amt: float | None = None + """Professional interest (CLM_LINE_PRFNL_INTRST_AMT).""" + + clm_mdcr_prmry_pyr_alowd_amt: float | None = None + """Line primary payer allowed (CLM_MDCR_PRMRY_PYR_ALOWD_AMT).""" diff --git a/src/bcda/table/patient.py b/src/bcda/table/patient.py index 9c0ede5..03797e2 100644 --- a/src/bcda/table/patient.py +++ b/src/bcda/table/patient.py @@ -1,10 +1,18 @@ """FHIR R4 Patient resource — beneficiary demographics and identifiers. -Flattened from the BCDA v2 Patient ndjson export. One row per +Flattened from the BCDA Patient ndjson export (v2 or v3 — the +flattener detects the dialect per resource). One row per beneficiary. Maps CCLF8 (demographics) and CCLF9 (XREF) fields to their FHIR R4 element paths. -Source: ``/api/v2/Patient/$export`` bulk FHIR endpoint. +v3 differences (BCDA v3 Data Dictionary, ``ccwMapping`` column): +``id`` is the IDR beneficiary surrogate key (BENE_SK), not the CCW +BENE_ID; race moves to the US Core extension (OMB codes, crosswalked +back into ``bene_race_cd``); the ``rfrnc_yr``/``dual_*`` extensions +are gone (dual status lives on its own Coverage row); the contact +language and historical source keys are new. + +Source: ``/api/{v2,v3}/Patient/$export`` bulk FHIR endpoint. """ from __future__ import annotations @@ -166,3 +174,21 @@ class Patient(SQLTable): dual_12: str | None = None """Dual eligibility status — December.""" + + # ── v3-only (BCDA v3 Data Dictionary) ──────────────────── + cntct_lang_cd: str | None = None + """Contact language, BCP-47 (Patient.communication.language; + IDR CNTCT_LANG_CD).""" + + xref_efctv_sk: str | None = None + """Current Patient FHIR id after cross-reference — the + ``replaced-by`` link target when present, else ``id`` + (IDR BENE_XREF_EFCTV_SK).""" + + bene_sk_list: str | None = None + """Semicolon-joined historical source keys + (Patient.link.other.display ∪ Patient.id).""" + + mbi_efctv_dt: date | None = None + """Current MBI effective date (Patient.identifier[us-mbi].period.start; + IDR BENE_MBI_EFCTV_DT).""" diff --git a/src/cli/load.py b/src/cli/load.py index 8cdb11f..35ed28c 100644 --- a/src/cli/load.py +++ b/src/cli/load.py @@ -55,8 +55,10 @@ def bcda( ) -> None: """Flatten BCDA FHIR exports and load into DuckDB. - Reads Patient/EOB/Coverage ndjson files, flattens them to Parquet, - and loads into the ``bcda`` schema. + Reads Patient/EOB/Coverage ndjson files (v2 or v3 payloads, any + number of parts), flattens them to Parquet, and loads into the + ``bcda`` schema. Without --ndjson-dir, uses the newest export per + resource type under the BCDA store (a v3 pull is three jobs). """ from aco.load.bcda import load_bcda diff --git a/tests/aco/test_coverage_gaps.py b/tests/aco/test_coverage_gaps.py index 377c7f1..21da359 100644 --- a/tests/aco/test_coverage_gaps.py +++ b/tests/aco/test_coverage_gaps.py @@ -490,7 +490,7 @@ class TestLoadBcdaFindLatest: (exports / "job1").mkdir() result = _find_latest_export(tmp_path) - assert result.name == "job1" + assert [d.name for d in result] == ["job1"] def test_find_latest_no_exports_dir(self, tmp_path): from aco.load.bcda import _find_latest_export diff --git a/tests/aco/test_load_bcda.py b/tests/aco/test_load_bcda.py index 793a634..7021a33 100644 --- a/tests/aco/test_load_bcda.py +++ b/tests/aco/test_load_bcda.py @@ -29,4 +29,96 @@ class TestFindLatestExport: # Set old_job mtime to past so new_job is clearly newer os.utime(exports / "old_job", (1000000, 1000000)) result = _find_latest_export(tmp_path) - assert result.name == "new_job" + assert [d.name for d in result] == ["new_job"] + + def test_newest_per_resource_type(self, tmp_path: Path) -> None: + """Three single-type v3 jobs: each type comes from its own dir.""" + import os + + exports = tmp_path / "exports" + exports.mkdir() + for i, (name, rtype) in enumerate( + [ + ("j_eob", "ExplanationOfBenefit"), + ("j_pat", "Patient"), + ("j_cov", "Coverage"), + ] + ): + d = exports / name + d.mkdir() + (d / f"{rtype}.ndjson").write_text("{}\n") + os.utime(d, (1000000 + i, 1000000 + i)) + # a newer Patient-only job supersedes j_pat + d = exports / "j_pat2" + d.mkdir() + (d / "Patient.ndjson").write_text("{}\n") + os.utime(d, (2000000, 2000000)) + result = _find_latest_export(tmp_path) + assert [d.name for d in result] == ["j_eob", "j_cov", "j_pat2"] + + +class TestLoadV3EndToEnd: + """Official v3 samples, split like three single-type jobs → DuckDB.""" + + def test_load_from_three_export_dirs(self, tmp_path: Path) -> None: + import shutil + + import duckdb + + from aco.load.bcda import load_bcda + from conf import path as conf_path + + samples = Path(__file__).resolve().parents[2] / "dev" / "vendor" / "bcda-site" + store = tmp_path / "store" + for job, rtype in ( + ("j1", "Patient"), + ("j2", "Coverage"), + ("j3", "ExplanationOfBenefit"), + ): + d = store / "exports" / job + d.mkdir(parents=True) + shutil.copy(samples / f"v3_{rtype}.ndjson", d / f"{rtype}.ndjson") + # a second EOB part in the same job dir + shutil.copy( + samples / "v3_ExplanationOfBenefit.ndjson", + store / "exports" / "j3" / "ExplanationOfBenefit.1.ndjson", + ) + db = tmp_path / "aco.duckdb" + + from unittest.mock import patch + + real_path = conf_path + + def fake_path(key: str): + if key == "storage.bcda": + return store + return real_path(key) + + with patch("conf.path", side_effect=fake_path): + stats = load_bcda(database=str(db)) + assert stats == { + "bcda.patient": 94, + "bcda.explanation_of_benefit": 538, + "bcda.eob_item": 2464, + "bcda.coverage": 175, + } + con = duckdb.connect(str(db), read_only=True) + plans = dict( + con.execute( + "select class_plan, count(*) from bcda.coverage group by 1" + ).fetchall() + ) + assert set(plans) == {"Part A", "Part B", "Part C", "Dual Medicare/Medicaid"} + sys_types = dict( + con.execute( + "select system_type, count(*) from bcda.explanation_of_benefit group by 1" + ).fetchall() + ) + assert sys_types == {"NationalClaimsHistory": 282, "DDPS": 256} + assert ( + con.execute( + "select count(*) from bcda.patient where bene_mbi_id is null" + ).fetchone()[0] + == 0 + ) + con.close() diff --git a/tests/aco/test_load_modules.py b/tests/aco/test_load_modules.py index 6be041f..cdcd516 100644 --- a/tests/aco/test_load_modules.py +++ b/tests/aco/test_load_modules.py @@ -49,7 +49,7 @@ class TestLoadBcda: from aco.load.bcda import _find_latest_export result = _find_latest_export(tmp_path) - assert result.is_dir() + assert len(result) == 1 and result[0].is_dir() class TestStageAll: diff --git a/tests/bcda/test_client.py b/tests/bcda/test_client.py index 80fdc7c..6d74fa1 100644 --- a/tests/bcda/test_client.py +++ b/tests/bcda/test_client.py @@ -846,7 +846,7 @@ class TestClientDefaults: def test_defaults(self) -> None: c = Client("cid", "csec") assert c.base_url == SANDBOX - assert c.version == "v2" + assert c.version == "v3" assert c.max_retries == 3 from conf import path diff --git a/tests/bcda/test_client_v3.py b/tests/bcda/test_client_v3.py new file mode 100644 index 0000000..d63b1aa --- /dev/null +++ b/tests/bcda/test_client_v3.py @@ -0,0 +1,388 @@ +"""bcda.client — v3 additions: typeFilter, OperationOutcome errors, +job history Bundle, completed-job reuse, cancellation, version/health.""" + +from __future__ import annotations + +import gzip +from unittest.mock import patch + +import httpx +import pytest + +from bcda.client import ( + ASYNC_HEADERS, + SANDBOX, + SYSTEM_TYPES, + BcdaHttpError, + Client, + JobCancelledError, + _requested_types, + operation_outcome, + raise_for_status, + type_filter_tag, +) + + +def _token_response(tok: str = "tok") -> httpx.Response: + return httpx.Response(200, json={"access_token": tok, "expires_in": 1200}) + + +def _oo(text: str, *, code: int = 400, details: bool = False) -> httpx.Response: + issue: dict = {"severity": "error", "code": "structure"} + if details: + issue["details"] = {"text": text} + else: + issue["diagnostics"] = text + return httpx.Response( + code, json={"resourceType": "OperationOutcome", "issue": [issue]} + ) + + +def _client(handler, **kw) -> Client: + kw.setdefault("output_dir", "/tmp/bcda-test") + c = Client("cid", "csec", **kw) + c._http = httpx.Client(transport=httpx.MockTransport(handler)) + return c + + +class TestHelpers: + def test_type_filter_tag(self) -> None: + assert type_filter_tag("DDPS") == ( + "ExplanationOfBenefit?_tag=https://bluebutton.cms.gov/fhir/CodeSystem/System-Type|DDPS" + ) + assert type_filter_tag(*SYSTEM_TYPES).count("|") == 3 + with pytest.raises(ValueError): + type_filter_tag("Bogus") + with pytest.raises(ValueError): + type_filter_tag() + + def test_operation_outcome(self) -> None: + assert ( + operation_outcome(_oo("Prefer header is required")) + == "Prefer header is required" + ) + assert operation_outcome(_oo("via details", details=True)) == "via details" + assert operation_outcome(httpx.Response(500, content=b"")) == "" + assert ( + operation_outcome(httpx.Response(404, json={"resourceType": "Bundle"})) + == "" + ) + assert operation_outcome(httpx.Response(404, json=[1, 2])) == "" + assert ( + operation_outcome( + httpx.Response( + 400, + json={ + "resourceType": "OperationOutcome", + "issue": [{"severity": "error"}], + }, + ) + ) + == "" + ) + + def test_raise_for_status(self) -> None: + raise_for_status(httpx.Response(200)) # no-op + resp = _oo("bad", code=404) + resp.request = httpx.Request("GET", "https://x/y") + with pytest.raises(BcdaHttpError) as ei: + raise_for_status(resp) + assert ei.value.status_code == 404 + assert ei.value.diagnostics == "bad" + assert "bad" in str(ei.value) + assert isinstance(ei.value, httpx.HTTPStatusError) + plain = httpx.Response(500, content=b"") + plain.request = httpx.Request("GET", "https://x/y") + with pytest.raises(BcdaHttpError) as ei: + raise_for_status(plain) + assert ei.value.diagnostics == "" and "500" in str(ei.value) + + def test_requested_types(self) -> None: + req = "GET https://sandbox.bcda.cms.gov/api/v3/Group/all/$export?_type=Patient%2CCoverage%2CExplanationOfBenefit" + assert _requested_types(req) == ["Patient", "Coverage", "ExplanationOfBenefit"] + assert _requested_types("GET https://x/api/v3/Group/all/$export") == [] + assert _requested_types("https://x/api/v3/Patient/$export?_type=Patient") == [ + "Patient" + ] + + +class TestExportTypeFilter: + def test_params_sent(self) -> None: + captured: dict = {} + + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + captured["url"] = str(req.url) + captured["prefer"] = req.headers.get("prefer") + return httpx.Response( + 202, headers={"Content-Location": f"{SANDBOX}/api/v3/jobs/1"} + ) + + c = _client(handler) + url = c.start_export( + types=["ExplanationOfBenefit"], + since="2026-01-01T00:00:00Z", + type_filter=type_filter_tag("SharedSystem"), + ) + assert url.endswith("/api/v3/jobs/1") + assert "_typeFilter=ExplanationOfBenefit%3F_tag%3D" in captured["url"] + assert "_since=2026-01-01" in captured["url"] + assert captured["prefer"] == "respond-async" + c.close() + + def test_export_passes_type_filter(self, tmp_path) -> None: + calls: list[str] = [] + + def handler(req: httpx.Request) -> httpx.Response: + url = str(req.url) + calls.append(url) + if "/auth/token" in url: + return _token_response() + if "$export" in url: + return httpx.Response( + 202, headers={"Content-Location": f"{SANDBOX}/api/v3/jobs/9"} + ) + if url.endswith("/jobs/9"): + return httpx.Response( + 200, + json={ + "output": [ + {"type": "Patient", "url": f"{SANDBOX}/data/9/a.ndjson"} + ], + "error": [], + }, + ) + return httpx.Response(200, content=b'{"resourceType":"Patient"}\n') + + c = _client(handler, output_dir=tmp_path) + paths = c.export( + types=["Patient"], + type_filter="ExplanationOfBenefit?outcome=partial", + poll_interval=0, + ) + assert [p.name for p in paths] == ["Patient.ndjson"] + assert any("_typeFilter=" in u for u in calls) + c.close() + + +class TestErrors: + def test_client_error_carries_diagnostics(self) -> None: + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + return _oo("application/fhir+json is the only supported response format") + + c = _client(handler) + with pytest.raises(BcdaHttpError, match="only supported response format"): + c._request("GET", f"{SANDBOX}/api/v3/jobs") + c.close() + + def test_auth_failure_diagnostics(self) -> None: + def handler(req: httpx.Request) -> httpx.Response: + return _oo("invalid client", code=401) + + c = _client(handler) + with pytest.raises(BcdaHttpError, match="invalid client"): + c.authenticate() + c.close() + + @patch("time.sleep") + def test_5xx_exhausted_is_bcda_error(self, _sleep) -> None: + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + return httpx.Response(503, content=b"down") + + c = _client(handler, max_retries=2) + with pytest.raises(BcdaHttpError) as ei: + c._request("GET", f"{SANDBOX}/x") + assert ei.value.status_code == 503 + c.close() + + def test_poll_cancelled(self) -> None: + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + return _oo("Job has been cancelled.", code=404) + + c = _client(handler) + with pytest.raises(JobCancelledError, match="cancelled"): + c.poll(f"{SANDBOX}/api/v3/jobs/5", interval=0) + c.close() + + def test_poll_other_404_propagates(self) -> None: + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + return _oo("Job not found", code=404) + + c = _client(handler) + with pytest.raises(BcdaHttpError, match="not found"): + c.poll(f"{SANDBOX}/api/v3/jobs/5", interval=0) + c.close() + + def test_poll_sends_bulk_headers(self) -> None: + seen: dict = {} + + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + seen.update({k.lower(): v for k, v in req.headers.items()}) + return httpx.Response(200, json={"output": []}) + + c = _client(handler) + c.poll(f"{SANDBOX}/api/v3/jobs/5", interval=0) + assert ( + seen["prefer"] == "respond-async" + and seen["accept"] == ASYNC_HEADERS["Accept"] + ) + c.close() + + def test_download_error_has_diagnostics(self, tmp_path) -> None: + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + return _oo("file expired", code=410) + + c = _client(handler, output_dir=tmp_path) + with pytest.raises(BcdaHttpError, match="file expired"): + c._download_file(f"{SANDBOX}/data/1/x.ndjson", tmp_path / "x.ndjson") + c.close() + + def test_download_gzip_fixture(self, tmp_path) -> None: + """The upstream gzip fixture decodes to the same bytes as the plain one.""" + from pathlib import Path + + fixtures = ( + Path(__file__).resolve().parents[2] + / "dev" + / "vendor" + / "bcda-app" + / "shared_files" + / "gzip_feature_test" + ) + gz = (fixtures / "test_gzip_encoded.ndjson").read_bytes() + plain = (fixtures / "test_no_encoding.ndjson").read_bytes() + assert gzip.decompress(gz).strip() == plain.strip() + + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + return httpx.Response(200, content=gz, headers={"Content-Encoding": "gzip"}) + + c = _client(handler, output_dir=tmp_path) + c._download_file(f"{SANDBOX}/data/1/x.ndjson", tmp_path / "x.ndjson") + assert (tmp_path / "x.ndjson").read_bytes().strip() == plain.strip() + c.close() + + +def _task(job_id: str, status: str, request: str, end: str) -> dict: + return { + "resourceType": "Task", + "identifier": [{"system": f"{SANDBOX}/api/v3/jobs", "value": job_id}], + "status": status, + "input": [{"type": {"text": "BULK FHIR Export"}, "valueString": request}], + "executionPeriod": {"start": "2026-10-25T00:00:00Z", "end": end}, + } + + +class TestJobs: + def test_bundle_unwrapped_with_headers(self) -> None: + seen: dict = {} + + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + seen["prefer"] = req.headers.get("prefer") + seen["accept"] = req.headers.get("accept") + return httpx.Response( + 200, + json={ + "resourceType": "Bundle", + "type": "searchset", + "entry": [ + {"resource": _task("1", "completed", "GET x", "2026")}, + {"fullUrl": "no-resource"}, + ], + }, + ) + + c = _client(handler) + jobs = c.jobs() + assert len(jobs) == 1 and jobs[0]["status"] == "completed" + assert seen == {"prefer": "respond-async", "accept": "application/fhir+json"} + c.close() + + def test_status_404_means_empty(self) -> None: + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + return _oo("no jobs", code=404) + + c = _client(handler) + assert c.jobs(status="Completed") == [] + with pytest.raises(BcdaHttpError): + c.jobs() # unfiltered 404 is a real error + c.close() + + def test_completed_jobs(self) -> None: + tasks = [ + _task( + "10", + "completed", + f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Patient", + "2026-10-25T10:00:00Z", + ), + _task( + "11", + "completed", + f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Coverage%2CPatient", + "2026-10-25T12:00:00Z", + ), + _task( + "12", + "cancelled", + f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Patient", + "2026-10-25T13:00:00Z", + ), + _task( + "13", + "completed", + f"GET {SANDBOX}/api/v2/Group/all/$export?_type=Patient", + "2026-10-25T14:00:00Z", + ), + {"resourceType": "Task", "status": "completed"}, + ] + + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + return httpx.Response( + 200, + json={ + "resourceType": "Bundle", + "entry": [{"resource": t} for t in tasks], + }, + ) + + c = _client(handler) + got = c.completed_jobs() + assert [j["job_id"] for j in got] == ["11", "10"] # newest first, v3 only + assert got[0]["types"] == ["Coverage", "Patient"] + assert got[0]["job_url"].endswith("/api/v3/jobs/11") + c.close() + + +class TestInfoEndpoints: + def test_version_and_health_unauthenticated(self) -> None: + def handler(req: httpx.Request) -> httpx.Response: + assert "Authorization" not in req.headers + if str(req.url).endswith("/_version"): + return httpx.Response(200, json={"version": "r302"}) + return httpx.Response(200, json={"database": "ok"}) + + c = _client(handler) + assert c.server_version()["version"] == "r302" + assert c.health()["database"] == "ok" + c.close() diff --git a/tests/bcda/test_express.py b/tests/bcda/test_express.py index 9e86a51..813bf01 100644 --- a/tests/bcda/test_express.py +++ b/tests/bcda/test_express.py @@ -2727,3 +2727,81 @@ class TestFlattenExport: assert "bcda.patient" in counts assert "bcda.explanation_of_benefit" not in counts assert "bcda.coverage" not in counts + + +# ── v3 coverage shape through the CCLF demographics stage ─────── + + +class TestDemographicsV3Coverage: + """v3 Coverage: one row per plan; dual status on its own row.""" + + def _frames(self): + import polars as pl + + patient = pl.DataFrame( + { + "bene_mbi_id": ["M1", "M2", "M3"], + "bene_sex_cd": ["1", "2", "1"], + "bene_race_cd": ["1", "2", None], + "bene_dob": [None, None, None], + "bene_death_dt": [None, None, None], + "bene_1st_name": ["a", "b", "c"], + "bene_midl_name": [None, None, None], + "bene_last_name": ["x", "y", "z"], + "bene_line_1_adr": [None, None, None], + "geo_zip_plc_name": [None, None, None], + "geo_usps_state_cd": [None, None, None], + "geo_zip5_cd": [None, None, None], + } + ) + coverage = pl.DataFrame( + { + "subscriber_id": ["M1", "M1", "M1", "M2", "M2", "M3"], + "class_plan": [ + "Part C", + "Part B", + "Part A", + "Part B", + "Dual Medicare/Medicaid", + "Part A", + ], + "ms_cd": [None, "10", "20", "10", None, "30"], + "orec": [None, "0", "1", "0", None, "2"], + "dual_01": [None, None, None, None, None, "04"], + "dual_stus_cd": [None, None, None, None, "08", None], + "bene_part_a_enrlmt_bgn_dt": [None, None, None, None, None, None], + "mdcr_covg_a_end_dt": [None, None, None, None, None, None], + } + ) + return patient, coverage + + def test_part_a_preferred_and_dual_joined(self) -> None: + from bcda.express.cclf import eligibility, stg_beneficiary_demographics + + patient, coverage = self._frames() + demo = stg_beneficiary_demographics(patient, coverage).sort("bene_mbi_id") + assert demo["ms_cd"].to_list() == [ + "20", + "10", + "30", + ] # M1 → Part A row, not Part C/B + assert demo["dual_stus_cd_dual"].to_list() == [None, "08", None] + elig = eligibility(demo).sort("person_id") + assert elig["dual_status_code"].to_list() == [None, "08", "04"] + assert elig["medicare_status_code"].to_list() == ["20", "10", "30"] + assert elig["original_reason_entitlement_code"].to_list() == ["1", "0", "2"] + + def test_v2_frames_without_plan_columns_still_work(self) -> None: + import polars as pl + + from bcda.express.cclf import eligibility, stg_beneficiary_demographics + + patient, coverage = self._frames() + v2cov = coverage.drop("class_plan", "dual_stus_cd").unique( + subset=["subscriber_id"], keep="first" + ) + demo = stg_beneficiary_demographics(patient, v2cov) + assert "dual_stus_cd_dual" not in demo.columns + elig = eligibility(demo).sort("person_id") + assert elig["dual_status_code"].to_list() == [None, None, "04"] + assert isinstance(elig, pl.DataFrame) diff --git a/tests/bcda/test_flatten_stream.py b/tests/bcda/test_flatten_stream.py new file mode 100644 index 0000000..904b5ba --- /dev/null +++ b/tests/bcda/test_flatten_stream.py @@ -0,0 +1,148 @@ +"""bcda.express.flatten — multi-part resolution and streaming Parquet output.""" + +from __future__ import annotations + +import json +import os +from datetime import date +from pathlib import Path + +import fsspec +import pyarrow as pa +import pyarrow.parquet as pq + +from bcda.express import flatten +from bcda.table.coverage import Coverage +from bcda.table.eob import EobItem, ExplanationOfBenefit +from bcda.table.patient import Patient + +ROOT = Path(__file__).resolve().parents[2] +SAMPLES = ROOT / "dev" / "vendor" / "bcda-site" + + +class TestResolveNdjson: + def test_parts_sorted_and_later_dirs_win(self, tmp_path: Path) -> None: + a, b = tmp_path / "a", tmp_path / "b" + a.mkdir() + b.mkdir() + for name in ( + "ExplanationOfBenefit.10.ndjson", + "ExplanationOfBenefit.ndjson", + "ExplanationOfBenefit.2.ndjson", + "ExplanationOfBenefitEmpty.ndjson", + "Patient.ndjson", + ): + (a / name).write_text("{}\n") + (b / "Patient.ndjson").write_text("{}\n") + (b / "Coverage.ndjson").write_text("{}\n") + found = flatten.resolve_ndjson([a, b]) + assert [p.name for p in found["ExplanationOfBenefit"]] == [ + "ExplanationOfBenefit.ndjson", + "ExplanationOfBenefit.2.ndjson", + "ExplanationOfBenefit.10.ndjson", + ] + assert found["Patient"] == [b / "Patient.ndjson"] + assert found["Coverage"] == [b / "Coverage.ndjson"] + assert flatten.resolve_ndjson(str(a))["Patient"] == [a / "Patient.ndjson"] + assert flatten.resolve_ndjson(tmp_path / "missing") == {} + + +class TestArrowSchema: + def test_types(self) -> None: + s = flatten.arrow_schema(EobItem) + assert s.field("clm_id").type == pa.string() + assert s.field("clm_line_num").type == pa.float64() + assert s.field("clm_line_from_dt").type == pa.date32() + assert set(s.names) == set(EobItem.column_names()) + for model in (Patient, Coverage, ExplanationOfBenefit): + assert flatten.arrow_schema(model).names == model.column_names() + + def test_unknown_annotation_falls_back_to_string(self) -> None: + from conf.table_base import SQLTable + + class Odd(SQLTable): + weird: dict | None = None + plain: int = 0 + + s = flatten.arrow_schema(Odd) + assert s.field("weird").type == pa.string() + assert s.field("plain").type == pa.int64() + + +class TestParquetSink: + def test_writes_batches_and_skips_empty(self, tmp_path: Path) -> None: + fs = fsspec.filesystem("file") + path = str(tmp_path / "flat" / "p.parquet") + sink = flatten._ParquetSink(fs, path, flatten.arrow_schema(Patient)) + sink.write([]) + sink.write([{"id": "1", "bene_dob": date(2000, 1, 1)}]) + sink.write([{"id": "2"}]) + sink.close() + assert sink.rows == 2 + t = pq.read_table(path) + assert t.num_rows == 2 and t.schema.field("bene_dob").type == pa.date32() + assert t.column("id").to_pylist() == ["1", "2"] + + +class TestFlattenExportStreaming: + def test_official_samples_in_small_batches( + self, tmp_path: Path, monkeypatch + ) -> None: + """Batch size 7 forces many Arrow batches; counts must not change.""" + import shutil + + monkeypatch.setattr(flatten, "BATCH_ROWS", 7) + src = tmp_path / "ndjson" + src.mkdir() + shutil.copy(SAMPLES / "v3_Patient.ndjson", src / "Patient.ndjson") + shutil.copy(SAMPLES / "v3_Coverage.ndjson", src / "Coverage.ndjson") + # split the EOB sample into two parts to exercise multi-part reading + lines = (SAMPLES / "v3_ExplanationOfBenefit.ndjson").read_text().splitlines() + (src / "ExplanationOfBenefit.ndjson").write_text( + "\n".join(lines[:100]) + "\n\n" + ) + (src / "ExplanationOfBenefit.1.ndjson").write_text( + "\n".join(lines[100:]) + "\n" + ) + counts = flatten.flatten_export(src, tmp_path / "store") + assert counts == { + "bcda.patient": 94, + "bcda.explanation_of_benefit": 269, + "bcda.eob_item": 1232, + "bcda.coverage": 175, + } + eob = pq.read_table( + tmp_path / "store" / "flat" / "explanation_of_benefit.parquet" + ) + assert eob.num_rows == 269 + assert set(eob.column("system_type").to_pylist()) == { + "NationalClaimsHistory", + "DDPS", + } + assert eob.schema.field("clm_from_dt").type == pa.date32() + + def test_iter_ndjson_skips_blank_lines(self, tmp_path: Path) -> None: + p = tmp_path / "x.ndjson" + p.write_text('{"a":1}\n\n \n{"a":2}\n') + assert [r["a"] for r in flatten._iter_ndjson([p])] == [1, 2] + assert flatten._read_ndjson(p) == [{"a": 1}, {"a": 2}] + + def test_missing_types_write_nothing(self, tmp_path: Path) -> None: + src = tmp_path / "ndjson" + src.mkdir() + (src / "Patient.ndjson").write_text( + json.dumps( + { + "id": "P", + "meta": { + "profile": [ + "http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Patient|2.2.0" + ] + }, + } + ) + + "\n" + ) + counts = flatten.flatten_export(src, tmp_path / "store") + assert counts == {"bcda.patient": 1} + assert sorted(os.listdir(tmp_path / "store" / "flat")) == ["patient.parquet"] diff --git a/tests/bcda/test_store.py b/tests/bcda/test_store.py index 91c3235..46edddb 100644 --- a/tests/bcda/test_store.py +++ b/tests/bcda/test_store.py @@ -2,6 +2,7 @@ from __future__ import annotations +from pathlib import Path from unittest.mock import MagicMock import pyarrow as pa @@ -656,3 +657,177 @@ class TestStorageOptions: def test_root_trailing_slash(self) -> None: store = Store("data/bcda/") assert store._root == "data/bcda" + + +# ── v3: adopt completed jobs, type_filter passthrough ─────────── + + +class TestAdoptAndTypeFilter: + def _client(self, handler): + import httpx + + from bcda.client import Client + + c = Client("cid", "csec", output_dir="/tmp/bcda-store-test") + c._http = httpx.Client(transport=httpx.MockTransport(handler)) + return c + + def test_adopt_downloads_completed_job(self, tmp_path) -> None: + import httpx + + from bcda.store import Store + + def handler(req: httpx.Request) -> httpx.Response: + url = str(req.url) + if "/auth/token" in url: + return httpx.Response( + 200, json={"access_token": "t", "expires_in": 1200} + ) + if url.endswith("/jobs/77"): + return httpx.Response( + 200, + json={ + "output": [ + {"type": "Patient", "url": "https://s/data/77/p.ndjson"}, + {"type": "Patient", "url": "https://s/data/77/p2.ndjson"}, + ] + }, + ) + return httpx.Response(200, content=b'{"resourceType":"Patient"}\n') + + store = Store(path=str(tmp_path)) + client = self._client(handler) + key = store.adopt( + client, "https://s/api/v3/jobs/77", types=["Patient"], poll_interval=0 + ) + job = store.get_job(key) + assert job["status"] == "complete" + assert job["job_url"].endswith("/jobs/77") + assert job["types"] == "Patient" + files = store.get_files(key) + assert sorted(Path(f["storage_path"]).name for f in files) == [ + "Patient.1.ndjson", + "Patient.ndjson", + ] + assert all(f["status"] == "complete" for f in files) + client.close() + + def test_adopt_failure_recorded(self, tmp_path) -> None: + import httpx + import pytest + + from bcda.client import BcdaHttpError + from bcda.store import Store + + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return httpx.Response( + 200, json={"access_token": "t", "expires_in": 1200} + ) + return httpx.Response( + 404, + json={ + "resourceType": "OperationOutcome", + "issue": [{"diagnostics": "Job has expired"}], + }, + ) + + store = Store(path=str(tmp_path)) + client = self._client(handler) + with pytest.raises(BcdaHttpError, match="expired"): + store.adopt(client, "https://s/api/v3/jobs/78", poll_interval=0) + failed = store.list_jobs(status="failed") + assert len(failed) == 1 and "expired" in failed[0]["error"] + client.close() + + def test_export_records_type_filter(self, tmp_path) -> None: + import httpx + + from bcda.store import Store + + seen: dict = {} + + def handler(req: httpx.Request) -> httpx.Response: + url = str(req.url) + if "/auth/token" in url: + return httpx.Response( + 200, json={"access_token": "t", "expires_in": 1200} + ) + if "$export" in url: + seen["url"] = url + return httpx.Response( + 202, headers={"Content-Location": "https://s/api/v3/jobs/1"} + ) + if url.endswith("/jobs/1"): + return httpx.Response(200, json={"output": []}) + return httpx.Response(200, content=b"") + + store = Store(path=str(tmp_path)) + client = self._client(handler) + key = store.export( + client, + types=["ExplanationOfBenefit"], + type_filter="ExplanationOfBenefit?outcome=partial", + poll_interval=0, + ) + assert "_typeFilter=" in seen["url"] + assert ( + store.get_job(key)["types"] + == "ExplanationOfBenefit|_typeFilter=ExplanationOfBenefit?outcome=partial" + ) + client.close() + + def test_export_retries_after_server_cancellation(self, tmp_path) -> None: + import httpx + import pytest + + from bcda.client import JobCancelledError + from bcda.store import Store + + polls = {"n": 0} + + def handler(req: httpx.Request) -> httpx.Response: + url = str(req.url) + if "/auth/token" in url: + return httpx.Response( + 200, json={"access_token": "t", "expires_in": 1200} + ) + if "$export" in url: + polls["n"] += 1 + return httpx.Response( + 202, + headers={"Content-Location": f"https://s/api/v3/jobs/{polls['n']}"}, + ) + if url.endswith("/jobs/1") or url.endswith("/jobs/2"): + return httpx.Response( + 404, + json={ + "resourceType": "OperationOutcome", + "issue": [{"diagnostics": "Job has been cancelled."}], + }, + ) + if url.endswith("/jobs/3"): + return httpx.Response(200, json={"output": []}) + return httpx.Response(200, content=b"") + + store = Store(path=str(tmp_path)) + client = self._client(handler) + key = store.export(client, types=["Patient"], poll_interval=0, cancel_retries=2) + job = store.get_job(key) + assert job["status"] == "complete" and job["job_url"].endswith("/jobs/3") + history = store._read_parquet( + store._jobs_path, store._read_parquet.__globals__["JOBS_SCHEMA"] + ) + statuses = [r["status"] for r in history.to_pylist() if r["key"] == key] + assert statuses.count("cancelled") == 2 and statuses[-1] == "complete" + client.close() + + # no retries left → the cancellation propagates and is recorded + polls["n"] = 0 + client = self._client(handler) + with pytest.raises(JobCancelledError): + store.export(client, types=["Patient"], poll_interval=0) + assert store.list_jobs(status="failed")[-1]["error"].startswith( + "https://s/api/v3/jobs/1" + ) + client.close() diff --git a/tests/bcda/test_v3.py b/tests/bcda/test_v3.py new file mode 100644 index 0000000..27cd093 --- /dev/null +++ b/tests/bcda/test_v3.py @@ -0,0 +1,1321 @@ +"""bcda.express.v3 — BCDA v3 (C4BB 2.2.0) flatteners. + +Two layers: unit tests on hand-built resources that exercise every +branch of the extraction primitives, and invariants over CMS's +official v3 sample files vendored under ``dev/vendor/bcda-site``. +""" + +from __future__ import annotations + +import json +from datetime import date +from pathlib import Path + +import pytest + +from bcda.express import v3 +from bcda.express.flatten import flatten_coverage, flatten_eob, flatten_patient + +ROOT = Path(__file__).resolve().parents[2] +SAMPLES = ROOT / "dev" / "vendor" / "bcda-site" + +SD, CS = v3.SD, v3.CS + + +def _ndjson(name: str) -> list[dict]: + path = SAMPLES / name + with path.open() as f: + return [json.loads(line) for line in f if line.strip()] + + +@pytest.fixture(scope="module") +def sample_patients() -> list[dict]: + return _ndjson("v3_Patient.ndjson") + + +@pytest.fixture(scope="module") +def sample_coverages() -> list[dict]: + return _ndjson("v3_Coverage.ndjson") + + +@pytest.fixture(scope="module") +def sample_eobs() -> list[dict]: + return _ndjson("v3_ExplanationOfBenefit.ndjson") + + +# ── primitives ─────────────────────────────────────────────── + + +class TestPrimitives: + def test_get_and_scalars(self) -> None: + assert v3._get({"a": {"b": 1}}, "a", "b") == 1 + assert v3._get({"a": 1}, "a", "b") is None + assert v3._get(None, "a") is None + assert v3._date("2024-01-02T00:00:00Z") == date(2024, 1, 2) + assert v3._date("not a date") is None + assert v3._date(None) is None + assert v3._date(42) is None + assert v3._num("1.5") == 1.5 + assert v3._num(True) is None + assert v3._num("x") is None + assert v3._num(None) is None + assert v3._ref_id("Patient/-1") == "-1" + assert v3._ref_id("#org") == "org" + assert v3._ref_id("") is None + assert v3._ref_id(None) is None + + def test_coding_and_has_coding(self) -> None: + cc = { + "coding": [ + {"system": "http://x/A", "code": "a"}, + {"system": "http://y/B", "code": "b"}, + ] + } + assert v3._coding(cc) == "a" + assert v3._coding(cc, "http://y/B") == "b" + assert v3._coding(cc, "/B") == "b" # suffix match + assert v3._coding(cc, "nope") is None + assert v3._coding(None) is None + assert v3._coding("string") is None + assert v3._has_coding(cc, "http://x/A", "a") + assert not v3._has_coding(cc, "http://x/A", "b") + assert not v3._has_coding(None, "s", "c") + + @pytest.mark.parametrize( + "ext, expected", + [ + ({"valueCoding": {"code": "Z"}}, "Z"), + ({"valueCode": "c"}, "c"), + ({"valueString": "s"}, "s"), + ({"valueDate": "2020-01-01"}, "2020-01-01"), + ({"valueDateTime": "2020-01-01T00:00:00Z"}, "2020-01-01T00:00:00Z"), + ({"valueBoolean": True}, True), + ({"valueDecimal": 1.5}, 1.5), + ({"valueInteger": 3}, 3), + ({"valueUnsignedInt": 4}, 4), + ({"valuePositiveInt": 5}, 5), + ({"valueQuantity": {"value": 7}}, 7), + ({"valueMoney": {"value": 9.5}}, 9.5), + ({"valueIdentifier": {"value": "id"}}, "id"), + ({"valueReference": {"reference": "x"}}, None), + ], + ) + def test_ext_value(self, ext: dict, expected: object) -> None: + assert v3._ext_value(ext) == expected + + def test_ext_lookup(self) -> None: + exts = [ + {"url": SD + "X", "valueCoding": {"code": "1"}}, + {"url": "other", "valueString": "n"}, + ] + assert v3._ext(exts, "X") == "1" + assert v3._ext(exts, "Y") is None + assert v3._ext(None, "X") is None + assert v3._ext_str(exts, "X") == "1" + assert v3._ext_str(exts, "Y") is None + + def test_identifiers(self) -> None: + ids = [ + {"system": "http://a/one", "value": "1"}, + {"type": {"coding": [{"system": "sys", "code": "uc"}]}, "value": "UC"}, + ] + assert v3._identifier(ids, "http://a/one") == "1" + assert v3._identifier(ids, "one") == "1" + assert v3._identifier(ids, "zzz") is None + assert v3._identifier(None, "x") is None + assert v3._identifier_typed(ids, "sys", "uc") == "UC" + assert v3._identifier_typed(ids, "sys", "xx") is None + assert v3._identifier_typed(None, "sys", "uc") is None + + def test_tag(self) -> None: + r = {"meta": {"tag": [{"system": "s", "code": "c"}]}} + assert v3._tag(r, "s") == "c" + assert v3._tag(r, "t") is None + assert v3._tag({}, "s") is None + + def test_supporting_info(self) -> None: + infos = [ + { + "category": { + "coding": [{"system": CS + "Supporting-Information", "code": "A"}] + }, + "code": { + "coding": [ + {"system": CS + "X", "code": "ax"}, + {"system": "o", "code": "ao"}, + ] + }, + "timingPeriod": {"start": "2020-01-01", "end": "2020-02-01"}, + "valueString": "vs", + }, + { + "category": { + "coding": [{"system": CS + "Supporting-Information", "code": "B"}] + }, + "timingDate": "2021-03-04", + "valueQuantity": {"value": 3}, + }, + { + "category": { + "coding": [{"system": CS + "Supporting-Information", "code": "C"}] + }, + "valueBoolean": True, + }, + { + "category": { + "coding": [{"system": CS + "Supporting-Information", "code": "D"}] + }, + }, + ] + assert v3._si(infos, "A")["valueString"] == "vs" + assert v3._si(infos, "nope") is None + assert v3._si(None, "A") is None + assert len(v3._si_all(infos, "A")) == 1 + assert v3._si_code(infos, "A") == "ax" + assert v3._si_code(infos, "A", code_system="o") == "ao" + assert v3._si_code(infos, "B") is None + assert v3._si_code(infos, "Q") is None + assert v3._si_code_by_system(infos, CS + "X") == "ax" + assert v3._si_code_by_system(infos, CS + "Nope") is None + assert v3._si_date(infos, "A") == date(2020, 1, 1) + assert v3._si_date(infos, "B") == date(2021, 3, 4) + assert v3._si_date(infos, "Q") is None + assert v3._si_period(infos, "A") == (date(2020, 1, 1), date(2020, 2, 1)) + assert v3._si_period(infos, "B") == (date(2021, 3, 4), None) + assert v3._si_period(infos, "Q") == (None, None) + assert v3._si_value(infos, "A") == "vs" + assert v3._si_value(infos, "B") == 3 + assert v3._si_value(infos, "C") is True + assert v3._si_value(infos, "D") is None + assert v3._si_value(infos, "Q") is None + + def test_adjudication(self) -> None: + adjs = [ + { + "category": { + "coding": [{"system": CS + "Adjudication", "code": "AMT"}] + }, + "amount": {"value": 1.5}, + }, + { + "category": { + "coding": [{"system": CS + "Adjudication", "code": "CNT"}] + }, + "value": 3, + }, + { + "category": { + "coding": [{"system": CS + "Adjudication", "code": "QTY"}] + }, + "quantity": {"value": 4}, + }, + ] + assert v3._adj_amount(adjs, "AMT") == 1.5 + assert v3._adj_amount(adjs, "CNT") is None + assert v3._adj_amount(adjs, "NOPE") is None + assert v3._adj_value(adjs, "CNT") == 3 + assert v3._adj_value(adjs, "NOPE") is None + assert v3._adj_quantity(adjs, "CNT") == 3 + assert v3._adj_quantity(adjs, "QTY") == 4 + assert v3._adj_quantity(adjs, "NOPE") is None + assert v3._adj_by_code(None, "AMT") is None + + def test_contained_and_names(self) -> None: + r = {"contained": [{"id": "org", "name": "ORG"}]} + assert v3._contained(r, "#org")["name"] == "ORG" + assert v3._contained(r, "#missing") is None + assert v3._contained(r, None) is None + assert v3._display_name(None) is None + assert v3._display_name("Acme") == "Acme" + assert ( + v3._display_name([{"family": "Doe", "given": ["Jane", "Q"]}]) + == "Doe, Jane Q" + ) + assert v3._display_name([{"text": "Dr Who"}]) == "Dr Who" + assert v3._display_name([{}]) is None + assert v3._display_name(["x"]) is None + assert v3._display_name([]) is None + assert v3._display_name(42) == "42" + + def test_care_team_and_npi(self) -> None: + members = [ + { + "role": {"coding": [{"code": "rendering"}, {"code": "rendering"}]}, + "provider": {"identifier": {"system": v3.US_NPI, "value": "111"}}, + }, + { + "role": {"coding": [{"code": "attending"}]}, + "provider": { + "identifier": { + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "NPI", + } + ] + }, + "value": "222", + } + }, + }, + { + "role": {"coding": [{"code": "referring"}]}, + "provider": {"identifier": {"system": "upin", "value": "U"}}, + }, + {"role": {}, "provider": {}}, + ] + ct = v3._care_team(members) + assert set(ct) == {"rendering", "attending", "referring"} + assert v3._npi(ct["rendering"][0]) == "111" + assert v3._npi(ct["attending"][0]) == "222" + assert v3._npi(ct["referring"][0]) is None + assert v3._npi(None) is None + assert v3._npi({"provider": {"identifier": "oops"}}) is None + assert v3._provider_value(ct["referring"][0]) == "U" + assert v3._care_team(None) == {} + + +class TestIsV3: + def test_profile(self) -> None: + assert v3.is_v3( + { + "meta": { + "profile": [ + "http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Patient|2.2.0" + ] + } + } + ) + assert not v3.is_v3( + { + "meta": { + "profile": [ + "http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Patient|1.0.0" + ] + } + } + ) + assert not v3.is_v3({"meta": {"profile": [None]}}) + + def test_tag_extension_identifier(self) -> None: + assert v3.is_v3( + {"meta": {"tag": [{"system": v3.CS_SYSTEM_TYPE, "code": "DDPS"}]}} + ) + assert v3.is_v3({"extension": [{"url": SD + "X"}]}) + assert v3.is_v3({"identifier": [{"system": v3.ID_CLM_CNTL}]}) + assert not v3.is_v3( + { + "extension": [ + {"url": "https://bluebutton.cms.gov/resources/variables/race"} + ] + } + ) + assert not v3.is_v3({}) + + +# ── Patient ────────────────────────────────────────────────── + + +def _patient(**over: object) -> dict: + base = { + "resourceType": "Patient", + "id": "-7", + "meta": { + "lastUpdated": "2026-01-01T00:00:00Z", + "profile": [ + "http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Patient|2.2.0" + ], + }, + "identifier": [ + { + "system": v3.US_MBI, + "value": "OLD1", + "period": {"start": "2010-01-01", "end": "2015-01-01"}, + }, + {"system": v3.US_MBI, "value": "CUR", "period": {"start": "2020-01-01"}}, + { + "system": v3.US_MBI, + "value": "OLD2", + "period": {"start": "2015-01-01", "end": "2020-01-01"}, + }, + {"system": "other", "value": "x"}, + ], + "name": [{"family": "Doe", "given": ["Jane", "Q"]}], + "birthDate": "1950-02-03", + "deceasedDateTime": "2025-01-01T00:00:00Z", + "gender": "female", + "address": [ + { + "line": ["1 A St", "Apt 2", "L3", "L4", "L5", "L6"], + "city": "Town", + "state": "TX", + "postalCode": "775501234", + } + ], + "extension": [ + {"url": v3.US_CORE_SEX, "valueCode": "248152002"}, + { + "url": v3.US_CORE_RACE, + "extension": [ + {"url": "ombCategory", "valueCoding": {"code": "2054-5"}}, + {"url": "text", "valueString": "Black"}, + ], + }, + { + "url": v3.US_CORE_ETHNICITY, + "extension": [ + {"url": "ombCategory", "valueCoding": {"code": "2186-5"}} + ], + }, + ], + "communication": [ + {"language": {"coding": [{"system": "urn:ietf:bcp:47", "code": "es"}]}} + ], + "link": [ + { + "type": "replaced-by", + "other": {"reference": "Patient/-9", "display": "-9"}, + }, + {"type": "seealso", "other": {"display": "-8"}}, + ], + } + base.update(over) + return base + + +class TestFlattenPatientV3: + def test_dispatch_and_identity(self) -> None: + p = flatten_patient(_patient()) + assert p.id == "-7" + assert p.bene_id == "-7" # BENE_SK + assert p.bene_mbi_id == "CUR" + assert p.identifier_currency == "current" + assert p.crnt_num == "CUR" + assert p.mbi_efctv_dt == date(2020, 1, 1) + assert p.prvs_num == "OLD2" # most recent historic + assert p.efctv_bgn_dt == date(2015, 1, 1) + assert p.efctv_end_dt == date(2020, 1, 1) + assert p.xref_efctv_sk == "-9" + assert p.bene_sk_list == "-9;-8;-7" + assert p.cntct_lang_cd == "es" + + def test_demographics(self) -> None: + p = flatten_patient(_patient()) + assert (p.bene_1st_name, p.bene_midl_name, p.bene_last_name) == ( + "Jane", + "Q", + "Doe", + ) + assert p.bene_dob == date(1950, 2, 3) + assert p.bene_death_dt == date(2025, 1, 1) + assert p.bene_sex_cd == "2" + assert p.us_core_sex == "248152002" + assert (p.us_core_race, p.bene_race_cd) == ("2054-5", "2") + assert p.geo_zip5_cd == "77550" + assert p.geo_zip4_cd == "1234" + assert p.bene_zip_cd == "775501234" + assert p.bene_line_6_adr == "L6" + assert p.geo_zip_plc_name == "Town" + assert p.geo_usps_state_cd == "TX" + assert p.rfrnc_yr is None and p.dual_01 is None + + def test_hispanic_wins_and_unknowns(self) -> None: + r = _patient( + extension=[ + { + "url": v3.US_CORE_ETHNICITY, + "extension": [ + {"url": "ombCategory", "valueCoding": {"code": v3.HISPANIC_OMB}} + ], + } + ] + ) + p = flatten_patient(r) + assert (p.us_core_race, p.bene_race_cd) == (None, "5") + r = _patient( + extension=[ + { + "url": v3.US_CORE_RACE, + "extension": [ + {"url": "ombCategory", "valueCoding": {"code": "9999-9"}} + ], + } + ] + ) + assert flatten_patient(r).bene_race_cd == "3" # unmapped OMB → Other + r = _patient( + extension=[ + { + "url": v3.US_CORE_RACE, + "extension": [ + {"url": "ombCategory", "valueCoding": {"code": "UNK"}} + ], + } + ] + ) + assert flatten_patient(r).bene_race_cd == "0" + assert flatten_patient(_patient(extension=[])).bene_race_cd is None + + def test_sex_fallbacks(self) -> None: + assert flatten_patient(_patient(extension=[], gender="male")).bene_sex_cd == "1" + assert ( + flatten_patient(_patient(extension=[], gender="other")).bene_sex_cd == "0" + ) + assert flatten_patient(_patient(extension=[], gender=None)).bene_sex_cd is None + + def test_mbi_edge_cases(self) -> None: + # all MBIs closed → latest end is "current" (historic flag), next is previous + ids = [ + { + "system": v3.US_MBI, + "value": "A", + "period": {"start": "2010-01-01", "end": "2012-01-01"}, + }, + { + "system": v3.US_MBI, + "value": "B", + "period": {"start": "2012-01-01", "end": "2014-01-01"}, + }, + ] + p = flatten_patient(_patient(identifier=ids)) + assert (p.bene_mbi_id, p.identifier_currency, p.prvs_num) == ( + "B", + "historic", + "A", + ) + p = flatten_patient(_patient(identifier=ids[:1])) + assert (p.bene_mbi_id, p.prvs_num) == ("A", None) + p = flatten_patient(_patient(identifier=[])) + assert ( + p.bene_mbi_id is None + and p.identifier_currency is None + and p.mbi_efctv_dt is None + ) + p = flatten_patient( + _patient(identifier=[{"system": v3.US_MBI, "value": "ONLY"}]) + ) + assert (p.bene_mbi_id, p.prvs_num, p.efctv_bgn_dt) == ("ONLY", None, None) + + def test_minimal(self) -> None: + p = flatten_patient( + { + "meta": { + "profile": [ + "http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Patient|2.2.0" + ] + } + } + ) + assert p.id is None and p.xref_efctv_sk is None and p.bene_sk_list is None + assert p.geo_zip5_cd is None and p.bene_1st_name is None + + def test_v2_payload_still_routes_to_v2(self) -> None: + r = { + "id": "P1", + "identifier": [{"system": "http://ccw/bene_id", "value": "B1"}], + "extension": [ + { + "url": "https://bluebutton.cms.gov/resources/variables/race", + "valueCoding": {"code": "1"}, + } + ], + } + p = flatten_patient(r) + assert p.bene_id == "B1" and p.bene_race_cd == "1" + + +# ── Coverage ───────────────────────────────────────────────── + + +def _coverage(plan: str, **over: object) -> dict: + base = { + "resourceType": "Coverage", + "id": f"{plan}-1", + "meta": { + "lastUpdated": "2026-01-01T00:00:00Z", + "profile": [ + "http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Coverage|2.2.0" + ], + }, + "status": "active", + "subscriberId": "MBI1", + "beneficiary": {"reference": "Patient/-7"}, + "type": {"coding": [{"system": "https://nahdo.org/sopt", "code": "1"}]}, + "class": [ + {"type": {"coding": [{"code": "group"}]}, "value": "Medicare"}, + {"type": {"coding": [{"code": "plan"}]}, "value": plan}, + ], + "period": {"start": "2024-01-01", "end": "2024-12-31"}, + "extension": [ + {"url": SD + "BENE-MDCR-STUS-CD", "valueCoding": {"code": "10"}}, + {"url": SD + "BENE-MDCR-ENTLMT-RSN-CD", "valueCoding": {"code": "0"}}, + {"url": SD + "BENE-ESRD-STUS-ID", "valueCoding": {"code": "N"}}, + {"url": SD + "BENE-MDCR-ENTLMT-STUS-CD", "valueCoding": {"code": "Y"}}, + {"url": SD + "BENE-BUYIN-CD", "valueCoding": {"code": "B"}}, + ], + } + base.update(over) + return base + + +class TestFlattenCoverageV3: + def test_part_a(self) -> None: + c = flatten_coverage(_coverage("Part A")) + assert c.class_plan == "Part A" and c.class_group == "Medicare" + assert c.beneficiary_reference == "-7" and c.subscriber_id == "MBI1" + assert c.coverage_type == "1" + assert (c.period_start, c.period_end) == (date(2024, 1, 1), date(2024, 12, 31)) + assert (c.bene_part_a_enrlmt_bgn_dt, c.mdcr_covg_a_end_dt) == ( + date(2024, 1, 1), + date(2024, 12, 31), + ) + assert c.bene_part_b_enrlmt_bgn_dt is None + assert (c.ms_cd, c.orec, c.esrd_ind, c.buyin_cd) == ("10", "0", "N", "B") + assert (c.entlmt_stus_cd, c.a_trm_cd, c.b_trm_cd) == ("Y", "Y", None) + + def test_part_b_and_d(self) -> None: + b = flatten_coverage(_coverage("Part B")) + assert (b.bene_part_b_enrlmt_bgn_dt, b.mdcr_covg_b_end_dt) == ( + date(2024, 1, 1), + date(2024, 12, 31), + ) + assert (b.a_trm_cd, b.b_trm_cd) == (None, "Y") + d = flatten_coverage( + _coverage( + "Part D", + extension=[ + {"url": SD + "BENE-CNTRCT-NUM", "valueString": "S0001"}, + {"url": SD + "BENE-PBP-NUM", "valueString": "001"}, + {"url": SD + "CNTRCT-PBP-SGMT-NUM", "valueString": "7"}, + {"url": SD + "BENE-LIS-EFCTV-CD", "valueCoding": {"code": "1"}}, + {"url": SD + "BENE-LIS-COPMT-LVL-CD", "valueCoding": {"code": "2"}}, + {"url": SD + "BENE-LIS-PTD-PRM-PCT", "valueDecimal": 100}, + {"url": SD + "BENE-ENRLMT-EMPLR-SBSDY-SW", "valueBoolean": False}, + ], + contained=[ + { + "resourceType": "Organization", + "id": "plan-org", + "name": "Big PDP", + }, + {"resourceType": "Organization", "id": "cms-org", "name": "CMS"}, + ], + ) + ) + assert (d.mdcr_covg_d_bgn_dt, d.mdcr_covg_d_end_dt) == ( + date(2024, 1, 1), + date(2024, 12, 31), + ) + assert (d.cntrct_num, d.pbp_num, d.pbp_sgmt_num) == ("S0001", "001", "7") + assert (d.lis_efctv_cd, d.lis_copmt_lvl_cd, d.lis_ptd_prm_pct, d.rds_sw) == ( + "1", + "2", + "100", + "False", + ) + assert d.cntrct_pbp_name == "Big PDP" + + def test_part_c_rx_classes_and_dual(self) -> None: + c = flatten_coverage( + _coverage( + "Part C", + **{ + "class": [ + {"type": {"coding": [{"code": "plan"}]}, "value": "Part C"}, + {"type": {"coding": [{"code": "rxid"}]}, "value": "RID"}, + {"type": {"coding": [{"code": "rxgroup"}]}, "value": "RG"}, + {"type": {"coding": [{"code": "rxpcn"}]}, "value": "PCN"}, + {"type": {"coding": [{"code": "rxbin"}]}, "value": "BIN"}, + {"type": {"coding": [{"code": "weird"}]}, "value": "?"}, + ], + "extension": [ + {"url": SD + "BENE-CVRG-TYPE-CD", "valueCoding": {"code": "11"}} + ], + "contained": [ + {"resourceType": "Organization", "id": "cms-org", "name": "CMS"} + ], + }, + ) + ) + assert (c.rx_id, c.rx_group, c.rx_pcn, c.rx_bin) == ("RID", "RG", "PCN", "BIN") + assert ( + c.cvrg_type_cd == "11" + and c.cntrct_pbp_name is None + and c.class_group is None + ) + assert c.bene_part_a_enrlmt_bgn_dt is None and c.mdcr_covg_d_bgn_dt is None + dual = flatten_coverage( + _coverage( + "Dual Medicare/Medicaid", + extension=[ + {"url": SD + "BENE-DUAL-STUS-CD", "valueCoding": {"code": "08"}}, + {"url": SD + "BENE-DUAL-TYPE-CD", "valueCoding": {"code": "N"}}, + {"url": SD + "MEDICAID-STATE-CD", "valueCoding": {"code": "VA"}}, + ], + ) + ) + assert (dual.dual_stus_cd, dual.dual_type_cd, dual.medicaid_state_cd) == ( + "08", + "N", + "VA", + ) + + def test_minimal_uses_mbi_identifier(self) -> None: + c = flatten_coverage( + { + "meta": { + "profile": [ + "http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-Coverage|2.2.0" + ] + }, + "identifier": [{"system": v3.US_MBI, "value": "M"}], + } + ) + assert ( + c.subscriber_id == "M" and c.class_plan is None and c.period_start is None + ) + + +# ── ExplanationOfBenefit ───────────────────────────────────── + + +def _si_bb(category: str, **body: object) -> dict: + d = { + "category": { + "coding": [ + {"system": v3.CS_SI, "code": category}, + { + "system": "http://terminology.hl7.org/CodeSystem/claiminformationcategory", + "code": "info", + }, + ] + } + } + d.update(body) + return d + + +def _si_c4bb(category: str, **body: object) -> dict: + d = {"category": {"coding": [{"system": v3.C4BB_SI, "code": category}]}} + d.update(body) + return d + + +def _coded(system: str, value: str) -> dict: + return {"code": {"coding": [{"system": system, "code": value}]}} + + +def _adj( + code: str, + *, + amount: float | None = None, + value: float | None = None, + system: str = v3.CS_ADJ, +) -> dict: + d: dict = {"category": {"coding": [{"system": system, "code": code}]}} + if amount is not None: + d["amount"] = {"value": amount, "currency": "USD"} + if value is not None: + d["value"] = value + return d + + +def _eob(**over: object) -> dict: + base = { + "resourceType": "ExplanationOfBenefit", + "id": "-100", + "meta": { + "lastUpdated": "2026-02-02T00:00:00Z", + "source": "NCH", + "profile": [ + "http://hl7.org/fhir/us/carin-bb/StructureDefinition/C4BB-ExplanationOfBenefit-Inpatient-Institutional|2.2.0" + ], + "tag": [ + {"system": v3.CS_SYSTEM_TYPE, "code": "NationalClaimsHistory"}, + {"system": v3.CS_FINAL_ACTION, "code": "FinalAction"}, + ], + }, + "identifier": [ + { + "type": {"coding": [{"system": v3.C4BB_ID_TYPE, "code": "uc"}]}, + "value": "-100", + }, + {"system": v3.ID_CLM_CNTL, "value": "CCN-1"}, + ], + "related": [ + {"reference": {"system": v3.ID_CLM_CNTL, "value": "CCN-0"}}, + {"reference": {"system": "x", "value": "no"}}, + ], + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/claim-type", + "code": "institutional", + }, + {"system": v3.CS_CLM_TYPE, "code": "60"}, + {"system": v3.CS_EOB_TYPE, "code": "INPATIENT"}, + ] + }, + "subType": {"coding": [{"code": "inpatient"}]}, + "outcome": "complete", + "patient": {"reference": "Patient/-7"}, + "billablePeriod": {"start": "2024-03-01", "end": "2024-03-10"}, + "created": "2024-04-01T00:00:00Z", + "payment": {"amount": {"value": 1000.5}, "date": "2024-04-02"}, + "provider": {"reference": "#billing"}, + "contained": [ + { + "resourceType": "Organization", + "id": "billing", + "name": "General Hospital", + "identifier": [ + {"system": v3.US_NPI, "value": "1234567893"}, + {"system": v3.CCN, "value": "450001"}, + {"system": v3.TAX_ID, "value": "99-1234567"}, + {"system": "http://" + v3.GENERIC_ID, "value": "GEN1"}, + ], + "address": [{"postalCode": "77550"}], + } + ], + "careTeam": [ + { + "role": {"coding": [{"code": "attending"}]}, + "provider": {"identifier": {"system": v3.US_NPI, "value": "ATT"}}, + }, + { + "role": {"coding": [{"code": "operating"}]}, + "provider": {"identifier": {"system": v3.US_NPI, "value": "OPR"}}, + }, + { + "role": {"coding": [{"code": "supervisor"}]}, + "provider": {"identifier": {"system": v3.US_NPI, "value": "SUP"}}, + }, + { + "role": {"coding": [{"code": "rendering"}]}, + "provider": {"identifier": {"system": v3.US_NPI, "value": "RND"}}, + "qualification": { + "coding": [{"system": CS + "CLM-PRVDR-SPCLTY-CD", "code": "61"}] + }, + }, + { + "role": {"coding": [{"code": "referring"}]}, + "provider": {"identifier": {"system": v3.US_NPI, "value": "REF"}}, + }, + { + "role": {"coding": [{"code": "prescribing"}]}, + "provider": {"identifier": {"system": v3.US_NPI, "value": "PRS"}}, + }, + { + "role": {"coding": [{"code": "service"}]}, + "provider": {"identifier": {"system": "other", "value": "SVC"}}, + }, + ], + "diagnosis": [ + { + "sequence": 1, + "diagnosisCodeableConcept": { + "coding": [ + {"system": "http://hl7.org/fhir/sid/icd-10-cm", "code": "A00"} + ] + }, + "type": [{"coding": [{"code": "principal"}]}], + "onAdmission": {"coding": [{"code": "Y"}]}, + }, + { + "sequence": 2, + "diagnosisCodeableConcept": { + "coding": [ + {"system": "http://hl7.org/fhir/sid/icd-10-cm", "code": "B00"} + ] + }, + "type": [{"coding": [{"code": "admitting"}]}], + }, + {"sequence": 13, "diagnosisCodeableConcept": {"coding": [{"code": "Z99"}]}}, + { + "diagnosisCodeableConcept": { + "coding": [ + {"system": "http://hl7.org/fhir/sid/icd-9-cm", "code": "9"} + ] + } + }, + ], + "procedure": [ + { + "sequence": 1, + "date": "2024-03-02", + "procedureCodeableConcept": {"coding": [{"code": "5A1945Z"}]}, + } + ], + "supportingInfo": [ + _si_bb("CLM_NRLN_RIC_CD", **_coded(CS + "CLM-NRLN-RIC-CD", "V")), + _si_bb("CLM_QUERY_CD", **_coded(CS + "CLM-QUERY-CD", "3")), + _si_bb("CLM_IDR_LD_DT", timingDate="2024-05-01"), + _si_bb("CLM_SBMTR_CNTRCT_NUM", valueString="H1234"), + _si_bb("CLM_BLOOD_PT_FRNSH_QTY", valueQuantity={"value": 2}), + _si_bb("CLM_CNTRCTR_NUM", **_coded(CS + "CLM-CNTRCTR-NUM", "10311")), + _si_c4bb( + "admissionperiod", + timingPeriod={"start": "2024-03-01", "end": "2024-03-10"}, + ), + _si_c4bb("clmrecvddate", timingDate="2024-03-12"), + _si_c4bb("discharge-status", **_coded(CS + "BENE-PTNT-STUS-CD", "01")), + _si_c4bb("drg", **_coded("https://www.cms.gov/.../MS-DRG", "470")), + _si_c4bb("dayssupply", valueQuantity={"value": 30}), + _si_c4bb( + "typeofbill", + code={ + "coding": [ + { + "system": "https://www.nubc.org/CodeSystem/TypeOfBill", + "code": "0111", + }, + {"system": CS + "CLM-BILL-FAC-TYPE-CD", "code": "1"}, + {"system": CS + "CLM-BILL-CLSFCTN-CD", "code": "1"}, + {"system": CS + "CLM-BILL-FREQ-CD", "code": "1"}, + ] + }, + ), + ], + "total": [ + _adj("CLM_SBMT_CHRG_AMT", amount=5000), + _adj("CLM_BENE_PMT_AMT", amount=20), + ], + "adjudication": [ + { + "category": { + "coding": [ + {"system": v3.C4BB_ADJ_DISC, "code": "benefitpaymentstatus"} + ] + }, + "reason": {"coding": [{"code": "innetwork"}]}, + }, + _adj("CLM_OPRTNL_IME_AMT", amount=132), + _adj("CLM_INSTNL_CVRD_DAY_CNT", value=9), + ], + "item": [ + { + "sequence": 1, + "diagnosisSequence": [2], + "servicedPeriod": {"start": "2024-03-01", "end": "2024-03-02"}, + "productOrService": { + "coding": [ + { + "system": "https://www.cms.gov/Medicare/Coding/HCPCSReleaseCodeSets", + "code": "G0001", + }, + { + "system": "https://www.cms.gov/x/HIPPSCodes", + "code": "HIPPS1", + }, + ] + }, + "modifier": [ + {"coding": [{"code": "25"}]}, + {"coding": [{"code": "59"}]}, + ], + "revenue": { + "coding": [{"system": CS + "CLM-REV-CNTR-CD", "code": "0450"}] + }, + "locationCodeableConcept": { + "coding": [ + { + "system": "https://www.cms.gov/Medicare/Coding/place-of-service-codes/Place_of_Service_Code_Set", + "code": "11", + } + ] + }, + "quantity": {"value": 3}, + "extension": [ + {"url": SD + "CLM-REV-CNTR-STUS-CD", "valueCoding": {"code": "S"}}, + {"url": SD + "CLM-LINE-PRFNL-MTUS-CNT", "valueDecimal": 2.5}, + {"url": SD + "CLM-PRVDR-TAX-NUM", "valueString": "TAX1"}, + {"url": SD + "CLM-PRVDR-TYPE-CD", "valueCoding": {"code": "7"}}, + { + "url": SD + "CLM-RNDRG-PRVDR-PRTCPTG-CD", + "valueCoding": {"code": "1"}, + }, + ], + "adjudication": [ + _adj("CLM_LINE_CVRD_PD_AMT", amount=10), + _adj("CLM_LINE_MDCR_DDCTBL_AMT", amount=1), + { + "category": {"coding": [{"code": "x"}]}, + "reason": { + "coding": [ + {"system": v3.CS_ANSI_RSN, "code": "45"}, + {"system": v3.CS_ANSI_GRP, "code": "CO"}, + ] + }, + }, + { + "category": {"coding": [{"code": "x"}]}, + "reason": { + "coding": [{"system": v3.CS_ANSI_GRP, "code": "PR"}] + }, + }, + { + "category": {"coding": [{"code": "x"}]}, + "reason": {"coding": [{"system": v3.CS_ANSI_RSN, "code": "1"}]}, + }, + { + "category": {"coding": [{"code": "x"}]}, + "reason": {"coding": [{"system": v3.CS_ANSI_RSN, "code": "2"}]}, + }, + { + "category": {"coding": [{"code": "x"}]}, + "reason": { + "coding": [{"system": v3.CS_ANSI_RSN, "code": "5th"}] + }, + }, + { + "category": {"coding": [{"code": "x"}]}, + "reason": {"coding": [{"code": "none"}]}, + }, + ], + }, + { + "sequence": 2, + "servicedDate": "2024-03-05", + "productOrService": { + "coding": [ + {"system": "http://www.ama-assn.org/go/cpt", "code": "99213"} + ] + }, + "detail": [ + { + "productOrService": { + "coding": [{"system": v3.NDC, "code": "00001"}] + }, + "quantity": {"value": 5, "unit": "ML"}, + } + ], + "revenue": { + "coding": [ + { + "system": "https://www.nubc.org/CodeSystem/RevenueCodes", + "code": "0250", + } + ] + }, + "locationCodeableConcept": {"coding": [{"code": "22"}]}, + }, + ], + } + base.update(over) + return base + + +class TestFlattenEobV3: + def test_identity_and_tags(self) -> None: + h, items = flatten_eob(_eob()) + assert (h.id, h.clm_id, h.clm_uniq_id, h.bene_id) == ( + "-100", + "-100", + "-100", + "-7", + ) + assert (h.clm_cntl_num, h.fi_doc_clm_cntl_num, h.carr_clm_cntl_num) == ( + "CCN-1", + ) * 3 + assert h.clm_orig_cntl_num == "CCN-0" + assert (h.system_type, h.final_action, h.eob_type, h.clm_type_cd) == ( + "NationalClaimsHistory", + "FinalAction", + "INPATIENT", + "60", + ) + assert (h.clm_src_id, h.outcome, h.clm_sub_type) == ( + "NCH", + "complete", + "inpatient", + ) + assert h.pde_id is None + assert (h.clm_from_dt, h.clm_thru_dt, h.clm_efctv_dt) == ( + date(2024, 3, 1), + date(2024, 3, 10), + date(2024, 4, 1), + ) + assert (h.clm_pmt_amt, h.clm_pd_dt) == (1000.5, date(2024, 4, 2)) + assert len(items) == 2 + + def test_supporting_info(self) -> None: + h, _ = flatten_eob(_eob()) + assert h.nch_near_line_rec_ident_cd == "V" + assert (h.clm_query_cd, h.carr_clm_entry_cd) == ("3", "3") + assert h.clm_idr_ld_dt == date(2024, 5, 1) + assert h.clm_sbmtr_cntrct_num == "H1234" + assert h.nch_blood_pnts_frnshd_qty == 2 + assert (h.fi_num, h.carr_num) == ("10311", "10311") + assert (h.clm_actv_care_from_dt, h.clm_dschrg_dt) == ( + date(2024, 3, 1), + date(2024, 3, 10), + ) + assert h.clm_submsn_dt == date(2024, 3, 12) + assert h.bene_ptnt_stus_cd == "01" + assert h.dgns_drg_cd == "470" + assert h.clm_line_days_suply_qty == 30 + assert ( + h.clm_bill_fac_type_cd, + h.clm_bill_clsfctn_cd, + h.clm_srvc_clsfctn_type_cd, + h.clm_bill_freq_cd, + ) == ("1", "1", "1", "1") + + def test_c4bb_code_falls_back_to_any_system(self) -> None: + r = _eob( + supportingInfo=[ + _si_c4bb("discharge-status", **_coded("urn:other", "30")), + { + "category": {}, + "code": { + "coding": [ + { + "system": CS + "CLM-MDCR-NCH-PTNT-STUS-IND-CD", + "code": "C", + } + ] + }, + }, + ] + ) + h, _ = flatten_eob(r) + assert h.bene_ptnt_stus_cd == "30" + r = _eob( + supportingInfo=[ + { + "category": {}, + "code": { + "coding": [ + { + "system": CS + "CLM-MDCR-NCH-PTNT-STUS-IND-CD", + "code": "C", + }, + {"system": CS + "CLM-OP-SRVC-TYPE-CD", "code": "3"}, + {"system": CS + "CLM-NRLN-RIC-CD", "code": "W"}, + ] + }, + } + ] + ) + h, _ = flatten_eob(r) + assert ( + h.bene_ptnt_stus_cd, + h.clm_op_srvc_type_cd, + h.nch_near_line_rec_ident_cd, + ) == ("C", "3", "W") + + def test_amounts(self) -> None: + h, _ = flatten_eob(_eob()) + assert (h.clm_sbmt_chrg_amt, h.clm_mdcr_instnl_tot_chrg_amt) == (5000, 5000) + assert h.clm_bene_pmt_amt == 20 + assert (h.clm_oprtnl_ime_amt, h.ime_op_clm_val_amt) == (132, 132) + assert h.clm_instnl_cvrd_day_cnt == 9 + assert h.clm_prcng_excptn_cd == "innetwork" + + def test_providers(self) -> None: + h, _ = flatten_eob(_eob()) + assert (h.carr_clm_blg_npi_num, h.fac_prvdr_npi_num) == ( + "1234567893", + "1234567893", + ) + assert h.prvdr_oscar_num == "450001" + assert h.clm_blg_prvdr_tax_num == "99-1234567" + assert h.clm_srvc_prvdr_gnrc_id_num == "GEN1" + assert (h.clm_blg_prvdr_name, h.clm_blg_prvdr_zip5_cd) == ( + "General Hospital", + "77550", + ) + assert (h.atndg_prvdr_npi_num, h.oprtg_prvdr_npi_num, h.othr_prvdr_npi_num) == ( + "ATT", + "OPR", + "SUP", + ) + assert (h.rndrg_prvdr_npi_num, h.clm_prvdr_spclty_cd) == ("RND", "61") + assert (h.ordrg_prvdr_npi_num, h.clm_prsbng_prvdr_gnrc_id_num) == ("REF", "PRS") + assert ( + h.clm_rndrg_prvdr_tax_num, + h.rndrg_prvdr_type_cd, + h.carr_line_prvdr_type_cd, + h.prtcptng_ind_cd, + ) == ("TAX1", "7", "7", "1") + + def test_service_provider_fallback_and_practitioner(self) -> None: + r = _eob( + contained=[ + { + "resourceType": "Practitioner", + "id": "billing", + "name": [{"family": "Jones", "given": ["C"]}], + "identifier": [], + } + ], + ) + h, _ = flatten_eob(r) + assert h.clm_blg_prvdr_name == "Jones, C" + assert ( + h.clm_srvc_prvdr_gnrc_id_num == "SVC" + ) # from the "service" care team role + assert h.prvdr_oscar_num is None + h, _ = flatten_eob( + _eob(provider={"extension": [{"url": "data-absent"}]}, careTeam=[]) + ) + assert h.carr_clm_blg_npi_num is None and h.rndrg_prvdr_npi_num is None + + def test_diagnoses_and_procedure(self) -> None: + h, _ = flatten_eob(_eob()) + assert (h.prncpl_dgns_cd, h.admtg_dgns_cd) == ("A00", "B00") + assert (h.clm_dgns_1_cd, h.clm_dgns_2_cd, h.clm_dgns_12_cd) == ( + "A00", + "B00", + None, + ) + assert h.dgns_prcdr_icd_ind == "9" # last diagnosis seen was ICD-9 + assert h.clm_poa_ind == "Y" + assert (h.clm_val_sqnc_num, h.clm_prcdr_cd, h.clm_prcdr_prfrm_dt) == ( + "1", + "5A1945Z", + date(2024, 3, 2), + ) + h, _ = flatten_eob(_eob(procedure=[], diagnosis=[])) + assert h.clm_prcdr_cd is None and h.prncpl_dgns_cd is None + + def test_items(self) -> None: + _, items = flatten_eob(_eob()) + a, b = items + assert a.clm_id == "-100" and a.clm_line_num == 1 + assert ( + a.clm_line_from_dt, + a.clm_line_thru_dt, + a.clm_line_instnl_rev_ctr_dt, + ) == (date(2024, 3, 1), date(2024, 3, 2), date(2024, 3, 1)) + assert a.clm_line_hcpcs_cd == "G0001" and a.clm_rev_apc_hipps_cd == "HIPPS1" + assert (a.hcpcs_1_mdfr_cd, a.hcpcs_2_mdfr_cd, a.hcpcs_3_mdfr_cd) == ( + "25", + "59", + None, + ) + assert a.clm_line_prod_rev_ctr_cd == "0450" and a.clm_pos_cd == "11" + assert (a.clm_line_srvc_unit_qty, a.rev_cntr_unit_cnt) == (3, 3) + assert a.clm_line_dgns_cd == "B00" + assert ( + a.rev_cntr_stus_ind_cd == "S" + and a.carr_line_mtus_cnt == 2.5 + and a.tax_num == "TAX1" + ) + assert (a.clm_rndrg_prvdr_type_cd, a.clm_rndrg_prvdr_prtcptg_cd) == ("7", "1") + assert (a.clm_line_cvrd_pd_amt, a.rev_cntr_pmt_amt) == (10, 10) + assert ( + a.clm_line_mdcr_ddctbl_amt, + a.clm_line_bene_ptb_ddctbl_amt, + a.rev_cntr_cash_ddctbl_amt, + ) == (1, 1, 1) + assert (a.clm_1_rev_cntr_ansi_rsn_cd, a.clm_1_rev_cntr_ansi_grp_cd) == ( + "45", + "CO", + ) + assert (a.clm_2_rev_cntr_ansi_rsn_cd, a.clm_2_rev_cntr_ansi_grp_cd) == ( + None, + "PR", + ) + assert (a.clm_3_rev_cntr_ansi_rsn_cd, a.clm_4_rev_cntr_ansi_rsn_cd) == ( + "1", + "2", + ) + assert a.hcpcs_betos_cd is None + # second item: servicedDate, CPT, NDC from detail, revenue fallback, POS fallback + assert (b.clm_line_from_dt, b.clm_line_thru_dt) == ( + date(2024, 3, 5), + date(2024, 3, 5), + ) + assert b.clm_line_hcpcs_cd == "99213" and b.clm_line_ndc_cd == "00001" + assert (b.clm_line_ndc_qty, b.clm_line_ndc_qty_qlfyr_cd) == (5, "ML") + assert b.clm_line_prod_rev_ctr_cd == "0250" and b.clm_pos_cd == "22" + assert b.clm_line_dgns_cd is None and b.clm_line_cvrd_pd_amt is None + + def test_pharmacy_identity(self) -> None: + r = _eob( + type={ + "coding": [ + {"system": v3.CS_EOB_TYPE, "code": "PDE"}, + {"system": v3.CS_CLM_TYPE, "code": "1"}, + ] + }, + item=[ + { + "sequence": 1, + "productOrService": {"coding": [{"system": v3.NDC, "code": "123"}]}, + "quantity": {"value": 30, "unit": "EA"}, + "servicedDate": "2024-01-01", + } + ], + ) + h, items = flatten_eob(r) + assert h.pde_id == "-100" and h.eob_type == "PDE" + assert items[0].clm_line_ndc_cd == "123" + assert (items[0].clm_line_ndc_qty, items[0].clm_line_ndc_qty_qlfyr_cd) == ( + 30, + "EA", + ) + + def test_minimal(self) -> None: + h, items = flatten_eob( + {"meta": {"tag": [{"system": v3.CS_SYSTEM_TYPE, "code": "DDPS"}]}} + ) + assert h.system_type == "DDPS" and h.clm_id is None and items == [] + assert h.clm_prcng_excptn_cd is None and h.clm_orig_cntl_num is None + + +# ── official sample invariants ─────────────────────────────── + + +class TestOfficialSamples: + def test_every_sample_is_v3( + self, sample_patients, sample_coverages, sample_eobs + ) -> None: + assert all( + v3.is_v3(r) for r in sample_patients + sample_coverages + sample_eobs + ) + + def test_patients(self, sample_patients) -> None: + rows = [flatten_patient(r) for r in sample_patients] + assert len(rows) == 94 + assert all(p.bene_mbi_id for p in rows) + assert all(p.bene_id == p.id for p in rows) + assert {p.bene_sex_cd for p in rows} == {"1", "2"} + assert {p.bene_race_cd for p in rows} <= set("0123456") + assert all(p.cntct_lang_cd for p in rows) + + def test_coverages(self, sample_coverages) -> None: + rows = [flatten_coverage(r) for r in sample_coverages] + plans = {c.class_plan for c in rows} + assert plans == {"Part A", "Part B", "Part C", "Dual Medicare/Medicaid"} + assert all(c.period_start for c in rows) + part_a = [c for c in rows if c.class_plan == "Part A"] + assert all(c.ms_cd and c.bene_part_a_enrlmt_bgn_dt for c in part_a) + dual = [c for c in rows if c.class_plan == "Dual Medicare/Medicaid"] + assert all(c.dual_stus_cd for c in dual) + + def test_eobs(self, sample_eobs) -> None: + heads, items = [], [] + for r in sample_eobs: + h, lines = flatten_eob(r) + heads.append(h) + items.extend(lines) + assert len(heads) == 269 and len(items) == 1232 + assert all( + h.clm_id and h.clm_cntl_num and h.system_type and h.eob_type for h in heads + ) + assert {h.system_type for h in heads} == {"NationalClaimsHistory", "DDPS"} + pde = [h for h in heads if h.eob_type == "PDE"] + assert pde and all( + h.pde_id == h.clm_id and h.clm_line_days_suply_qty is not None for h in pde + ) + inst = [ + h + for h in heads + if h.eob_type in ("INPATIENT", "SNF", "HOSPICE", "HHA", "OUTPATIENT") + ] + assert inst and all(h.clm_bill_fac_type_cd for h in inst) + assert all(i.clm_id and i.clm_line_from_dt for i in items) + assert sum(1 for i in items if i.clm_line_hcpcs_cd) > 1000 diff --git a/tests/test_final_coverage_gaps.py b/tests/test_final_coverage_gaps.py index 5f1b6ca..e2d2c76 100644 --- a/tests/test_final_coverage_gaps.py +++ b/tests/test_final_coverage_gaps.py @@ -1060,8 +1060,9 @@ class TestAcoLoadBcdaFindLatest: d1.mkdir() d2 = exports_dir / "export-002" d2.mkdir() + (d2 / "Patient.ndjson").write_text("{}\n") result = _find_latest_export(tmp_path) - assert result.is_dir() # line 51 + assert result == [d2] # only the dir that holds a resource file class TestAcoLoadSeedParquet: