From 63e9c0029a518a72eec6865d94851d09aea694f7 Mon Sep 17 00:00:00 2001 From: kert Date: Sun, 25 Oct 2026 17:33:13 -0400 Subject: [PATCH] =?UTF-8?q?feat(cms):=20vendor=20the=20institutional=20pro?= =?UTF-8?q?vider=20universe=20(CCN=20=E2=86=94=20NPI=20=E2=86=94=20TIN)=20?= =?UTF-8?q?from=20data.cms.gov;=20give=20carrier=20lines=20their=20billed?= =?UTF-8?q?=20TIN?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "No TIN available" is a fact about the data, not a bug in ours. Institutional claims never carry a TIN: the real Large-entity v3 pull has 0 billing TINs on every claim type and 0 CCNs on every institutional claim (the dictionary's NamingSystem/CCN identifier is not emitted and half the institutional claims have no billing provider at all), and CCLF1 has an OSCAR/CCN and NPIs but no TIN either. The only TIN on claims is the carrier/DME CLM_RNDRG_PRVDR_TAX_NUM, the TIN the line was billed under, so the connector now gives professional and DME lines that value as billing_tin too (institutional stays NULL; the facility is its NPI and CCN). CMS publishes no provider TIN or EIN on data.cms.gov — checked every enrollment, ownership, change-of-ownership, Provider of Services and cost-report family in the catalog. What can be vendored is CCN ↔ NPI ↔ organization. cms.datacms resolves a dataset family to its newest version in the data.cms.gov catalog, downloads the CSV (or pages the API) and names columns like the generated cms.table models. `stack load cms` (aco.load.cms) loads the institutional universe — hospital, SNF, HHA, hospice, FQHC and RHC enrollments, the Provider of Services file and the three cost reports — as cms.* with provenance columns and ingest_log rows, then derives cms.institutional_provider, cms.ccn_crosswalk (PECOS ∪ POS-only CCNs, 65,161 rows today), cms.provider_tin_observed (NPI → TIN as seen on our own claims) and cms.ccn_tin (CCN → NPI → TIN, with cms.aco_participant_tin as the authoritative hook when a participant list is loaded). The command prints facility-NPI → CCN → TIN coverage; the Data page gets the same query. On the sandbox nothing resolves — its 8 facility NPIs are synthetic — which is the expected answer there. refs #1139 --- docs/docs/cli/api-serve.md | 2 +- docs/docs/cli/api.md | 2 +- docs/docs/cli/bcda-adopt.md | 2 +- docs/docs/cli/bcda-auth.md | 2 +- docs/docs/cli/bcda-cancel.md | 2 +- docs/docs/cli/bcda-errors.md | 2 +- docs/docs/cli/bcda-export.md | 2 +- docs/docs/cli/bcda-jobs.md | 2 +- docs/docs/cli/bcda-load.md | 2 +- docs/docs/cli/bcda-preflight.md | 2 +- docs/docs/cli/bcda-resume.md | 2 +- docs/docs/cli/bcda-store.md | 2 +- docs/docs/cli/bcda-sync.md | 2 +- docs/docs/cli/bcda-ui.md | 2 +- docs/docs/cli/bcda-vendor.md | 2 +- docs/docs/cli/bcda.md | 2 +- docs/docs/cli/bib-attach-iom.md | 2 +- docs/docs/cli/bib-attach-oig.md | 2 +- docs/docs/cli/bib-backfill-comments.md | 2 +- docs/docs/cli/bib-cfr-jump.md | 2 +- docs/docs/cli/bib-code-tags.md | 2 +- docs/docs/cli/bib-discover-pfs-rules.md | 2 +- docs/docs/cli/bib-fetch-docket-comments.md | 2 +- docs/docs/cli/bib-fetch-pfs-comments.md | 2 +- docs/docs/cli/bib-fr-grab.md | 2 +- docs/docs/cli/bib-fr-jump.md | 2 +- docs/docs/cli/bib-import-zotero.md | 2 +- docs/docs/cli/bib-ingest-iom.md | 2 +- docs/docs/cli/bib-ingest-mail.md | 2 +- docs/docs/cli/bib-ingest-oig.md | 2 +- docs/docs/cli/bib-query.md | 2 +- docs/docs/cli/bib-refresh-iom.md | 2 +- docs/docs/cli/bib-refresh-oig.md | 2 +- docs/docs/cli/bib-sync-zotero.md | 2 +- docs/docs/cli/bib-sync.md | 2 +- docs/docs/cli/bib-tag.md | 2 +- docs/docs/cli/bib-watch-iom.md | 2 +- docs/docs/cli/bib.md | 2 +- docs/docs/cli/bls-ces.md | 2 +- docs/docs/cli/bls-oews.md | 2 +- docs/docs/cli/bls.md | 2 +- docs/docs/cli/comments-dockets.md | 2 +- docs/docs/cli/comments-extract-ocr.md | 2 +- docs/docs/cli/comments-extract.md | 2 +- docs/docs/cli/comments-seal.md | 2 +- docs/docs/cli/comments-stats.md | 2 +- docs/docs/cli/comments-unseal.md | 2 +- docs/docs/cli/comments.md | 2 +- docs/docs/cli/db-comment.md | 2 +- docs/docs/cli/db-inspect.md | 2 +- docs/docs/cli/db.md | 2 +- docs/docs/cli/docs-build.md | 2 +- docs/docs/cli/docs-generate.md | 2 +- docs/docs/cli/docs-serve.md | 2 +- docs/docs/cli/docs.md | 2 +- docs/docs/cli/generate-models.md | 2 +- docs/docs/cli/generate.md | 2 +- docs/docs/cli/lake-deploy.md | 2 +- docs/docs/cli/lake-load.md | 2 +- docs/docs/cli/lake-parity.md | 2 +- docs/docs/cli/lake-validate.md | 2 +- docs/docs/cli/lake.md | 2 +- docs/docs/cli/llm-eval-tags.md | 2 +- docs/docs/cli/llm-hosts.md | 2 +- docs/docs/cli/llm-index.md | 2 +- docs/docs/cli/llm-prune-rules.md | 2 +- docs/docs/cli/llm-restamp.md | 2 +- docs/docs/cli/llm-serve.md | 2 +- docs/docs/cli/llm-stamp-themes.md | 2 +- docs/docs/cli/llm-tag.md | 2 +- docs/docs/cli/llm-vocab.md | 2 +- docs/docs/cli/llm.md | 2 +- docs/docs/cli/load-cms.md | 30 ++ docs/docs/cli/load-seed.md | 2 +- docs/docs/cli/load.md | 2 + docs/docs/cli/mail-attach-smarthost.md | 2 +- docs/docs/cli/mail-dkim-export.md | 2 +- docs/docs/cli/mail-dns.md | 2 +- docs/docs/cli/mail-down.md | 2 +- docs/docs/cli/mail-provision.md | 2 +- docs/docs/cli/mail-rotate-creds.md | 2 +- docs/docs/cli/mail-seed-mailboxes.md | 2 +- docs/docs/cli/mail-status.md | 2 +- docs/docs/cli/mail-up.md | 2 +- docs/docs/cli/mail-wire-git.md | 2 +- docs/docs/cli/mail.md | 2 +- docs/docs/cli/perf-show.md | 2 +- docs/docs/cli/perf.md | 2 +- docs/docs/cli/pfs-cpt-ingest.md | 2 +- docs/docs/cli/pfs-elements.md | 2 +- docs/docs/cli/pfs-exposure.md | 2 +- docs/docs/cli/pfs-families.md | 2 +- docs/docs/cli/pfs-guidance.md | 2 +- docs/docs/cli/pfs-lineage.md | 2 +- docs/docs/cli/pfs-reaction.md | 2 +- docs/docs/cli/pfs-review.md | 2 +- docs/docs/cli/pfs-utilization.md | 2 +- docs/docs/cli/pfs.md | 2 +- docs/docs/cli/prisma-eligible.md | 2 +- docs/docs/cli/prisma-export.md | 2 +- docs/docs/cli/prisma-extract.md | 2 +- docs/docs/cli/prisma-fetch.md | 2 +- docs/docs/cli/prisma-flow.md | 2 +- docs/docs/cli/prisma-init.md | 2 +- docs/docs/cli/prisma-ping-llm.md | 2 +- docs/docs/cli/prisma-run.md | 2 +- docs/docs/cli/prisma-screen.md | 2 +- docs/docs/cli/prisma-vpn.md | 2 +- docs/docs/cli/prisma.md | 2 +- docs/docs/cli/rec-list.md | 2 +- docs/docs/cli/rec-opps.md | 2 +- docs/docs/cli/rec-pfs.md | 2 +- docs/docs/cli/rec.md | 2 +- docs/docs/cli/zot-dump-schema.md | 2 +- docs/docs/cli/zot-fix-dates.md | 2 +- docs/docs/cli/zot-fix-fields.md | 2 +- docs/docs/cli/zot-fix-keys.md | 2 +- docs/docs/cli/zot-verify-parity.md | 2 +- docs/docs/cli/zot.md | 2 +- .../specs/2026-10-25-bcda-v3-design.md | 6 + src/aco/load/cms.py | 381 +++++++++++++++++ src/bcda/express/cclf.py | 23 +- src/bcda/ui/service.py | 8 + src/cli/load.py | 48 +++ src/cms/datacms.py | 287 +++++++++++++ tests/aco/test_load_cms.py | 398 ++++++++++++++++++ tests/bcda/test_cclf_v3.py | 2 + tests/bcda/test_ui_service.py | 6 +- tests/cms/test_datacms.py | 242 +++++++++++ 129 files changed, 1548 insertions(+), 119 deletions(-) create mode 100644 docs/docs/cli/load-cms.md create mode 100644 src/aco/load/cms.py create mode 100644 src/cms/datacms.py create mode 100644 tests/aco/test_load_cms.py create mode 100644 tests/cms/test_datacms.py diff --git a/docs/docs/cli/api-serve.md b/docs/docs/cli/api-serve.md index deb7179..0c67627 100644 --- a/docs/docs/cli/api-serve.md +++ b/docs/docs/cli/api-serve.md @@ -1,6 +1,6 @@ --- title: stack api serve -sidebar_position: 80 +sidebar_position: 81 --- # `stack api serve` diff --git a/docs/docs/cli/api.md b/docs/docs/cli/api.md index c2de7f9..69df372 100644 --- a/docs/docs/cli/api.md +++ b/docs/docs/cli/api.md @@ -1,6 +1,6 @@ --- title: stack api -sidebar_position: 79 +sidebar_position: 80 --- # `stack api` diff --git a/docs/docs/cli/bcda-adopt.md b/docs/docs/cli/bcda-adopt.md index 6141d2e..ea01c5c 100644 --- a/docs/docs/cli/bcda-adopt.md +++ b/docs/docs/cli/bcda-adopt.md @@ -1,6 +1,6 @@ --- title: stack bcda adopt -sidebar_position: 15 +sidebar_position: 16 --- # `stack bcda adopt` diff --git a/docs/docs/cli/bcda-auth.md b/docs/docs/cli/bcda-auth.md index 491c7cb..3fc839c 100644 --- a/docs/docs/cli/bcda-auth.md +++ b/docs/docs/cli/bcda-auth.md @@ -1,6 +1,6 @@ --- title: stack bcda auth -sidebar_position: 12 +sidebar_position: 13 --- # `stack bcda auth` diff --git a/docs/docs/cli/bcda-cancel.md b/docs/docs/cli/bcda-cancel.md index 8d92a82..b69a25a 100644 --- a/docs/docs/cli/bcda-cancel.md +++ b/docs/docs/cli/bcda-cancel.md @@ -1,6 +1,6 @@ --- title: stack bcda cancel -sidebar_position: 17 +sidebar_position: 18 --- # `stack bcda cancel` diff --git a/docs/docs/cli/bcda-errors.md b/docs/docs/cli/bcda-errors.md index 105f81d..d9ecc96 100644 --- a/docs/docs/cli/bcda-errors.md +++ b/docs/docs/cli/bcda-errors.md @@ -1,6 +1,6 @@ --- title: stack bcda errors -sidebar_position: 19 +sidebar_position: 20 --- # `stack bcda errors` diff --git a/docs/docs/cli/bcda-export.md b/docs/docs/cli/bcda-export.md index 769f973..e7ebbd7 100644 --- a/docs/docs/cli/bcda-export.md +++ b/docs/docs/cli/bcda-export.md @@ -1,6 +1,6 @@ --- title: stack bcda export -sidebar_position: 14 +sidebar_position: 15 --- # `stack bcda export` diff --git a/docs/docs/cli/bcda-jobs.md b/docs/docs/cli/bcda-jobs.md index 28ff632..21f95c8 100644 --- a/docs/docs/cli/bcda-jobs.md +++ b/docs/docs/cli/bcda-jobs.md @@ -1,6 +1,6 @@ --- title: stack bcda jobs -sidebar_position: 13 +sidebar_position: 14 --- # `stack bcda jobs` diff --git a/docs/docs/cli/bcda-load.md b/docs/docs/cli/bcda-load.md index 37d5f06..2ec7af9 100644 --- a/docs/docs/cli/bcda-load.md +++ b/docs/docs/cli/bcda-load.md @@ -1,6 +1,6 @@ --- title: stack bcda load -sidebar_position: 20 +sidebar_position: 21 --- # `stack bcda load` diff --git a/docs/docs/cli/bcda-preflight.md b/docs/docs/cli/bcda-preflight.md index 50ac8ef..16c5c93 100644 --- a/docs/docs/cli/bcda-preflight.md +++ b/docs/docs/cli/bcda-preflight.md @@ -1,6 +1,6 @@ --- title: stack bcda preflight -sidebar_position: 11 +sidebar_position: 12 --- # `stack bcda preflight` diff --git a/docs/docs/cli/bcda-resume.md b/docs/docs/cli/bcda-resume.md index cf571e4..e4b1593 100644 --- a/docs/docs/cli/bcda-resume.md +++ b/docs/docs/cli/bcda-resume.md @@ -1,6 +1,6 @@ --- title: stack bcda resume -sidebar_position: 16 +sidebar_position: 17 --- # `stack bcda resume` diff --git a/docs/docs/cli/bcda-store.md b/docs/docs/cli/bcda-store.md index 04ad7a2..7dfa0b6 100644 --- a/docs/docs/cli/bcda-store.md +++ b/docs/docs/cli/bcda-store.md @@ -1,6 +1,6 @@ --- title: stack bcda store -sidebar_position: 18 +sidebar_position: 19 --- # `stack bcda store` diff --git a/docs/docs/cli/bcda-sync.md b/docs/docs/cli/bcda-sync.md index 377ae33..b6f2966 100644 --- a/docs/docs/cli/bcda-sync.md +++ b/docs/docs/cli/bcda-sync.md @@ -1,6 +1,6 @@ --- title: stack bcda sync -sidebar_position: 22 +sidebar_position: 23 --- # `stack bcda sync` diff --git a/docs/docs/cli/bcda-ui.md b/docs/docs/cli/bcda-ui.md index b2fcadb..7eb7344 100644 --- a/docs/docs/cli/bcda-ui.md +++ b/docs/docs/cli/bcda-ui.md @@ -1,6 +1,6 @@ --- title: stack bcda ui -sidebar_position: 10 +sidebar_position: 11 --- # `stack bcda ui` diff --git a/docs/docs/cli/bcda-vendor.md b/docs/docs/cli/bcda-vendor.md index d6312fd..b47ae1c 100644 --- a/docs/docs/cli/bcda-vendor.md +++ b/docs/docs/cli/bcda-vendor.md @@ -1,6 +1,6 @@ --- title: stack bcda vendor -sidebar_position: 21 +sidebar_position: 22 --- # `stack bcda vendor` diff --git a/docs/docs/cli/bcda.md b/docs/docs/cli/bcda.md index a73ae1b..4fec3ac 100644 --- a/docs/docs/cli/bcda.md +++ b/docs/docs/cli/bcda.md @@ -1,6 +1,6 @@ --- title: stack bcda -sidebar_position: 9 +sidebar_position: 10 --- # `stack bcda` diff --git a/docs/docs/cli/bib-attach-iom.md b/docs/docs/cli/bib-attach-iom.md index 7742dcc..52f8070 100644 --- a/docs/docs/cli/bib-attach-iom.md +++ b/docs/docs/cli/bib-attach-iom.md @@ -1,6 +1,6 @@ --- title: stack bib attach-iom -sidebar_position: 35 +sidebar_position: 36 --- # `stack bib attach-iom` diff --git a/docs/docs/cli/bib-attach-oig.md b/docs/docs/cli/bib-attach-oig.md index 9d56ef4..43663cc 100644 --- a/docs/docs/cli/bib-attach-oig.md +++ b/docs/docs/cli/bib-attach-oig.md @@ -1,6 +1,6 @@ --- title: stack bib attach-oig -sidebar_position: 40 +sidebar_position: 41 --- # `stack bib attach-oig` diff --git a/docs/docs/cli/bib-backfill-comments.md b/docs/docs/cli/bib-backfill-comments.md index d80ebf5..e2cd687 100644 --- a/docs/docs/cli/bib-backfill-comments.md +++ b/docs/docs/cli/bib-backfill-comments.md @@ -1,6 +1,6 @@ --- title: stack bib backfill-comments -sidebar_position: 33 +sidebar_position: 34 --- # `stack bib backfill-comments` diff --git a/docs/docs/cli/bib-cfr-jump.md b/docs/docs/cli/bib-cfr-jump.md index 87edf68..5c68ff4 100644 --- a/docs/docs/cli/bib-cfr-jump.md +++ b/docs/docs/cli/bib-cfr-jump.md @@ -1,6 +1,6 @@ --- title: stack bib cfr-jump -sidebar_position: 45 +sidebar_position: 46 --- # `stack bib cfr-jump` diff --git a/docs/docs/cli/bib-code-tags.md b/docs/docs/cli/bib-code-tags.md index eccf715..13cf021 100644 --- a/docs/docs/cli/bib-code-tags.md +++ b/docs/docs/cli/bib-code-tags.md @@ -1,6 +1,6 @@ --- title: stack bib code-tags -sidebar_position: 46 +sidebar_position: 47 --- # `stack bib code-tags` diff --git a/docs/docs/cli/bib-discover-pfs-rules.md b/docs/docs/cli/bib-discover-pfs-rules.md index 00d443f..a6aa291 100644 --- a/docs/docs/cli/bib-discover-pfs-rules.md +++ b/docs/docs/cli/bib-discover-pfs-rules.md @@ -1,6 +1,6 @@ --- title: stack bib discover-pfs-rules -sidebar_position: 29 +sidebar_position: 30 --- # `stack bib discover-pfs-rules` diff --git a/docs/docs/cli/bib-fetch-docket-comments.md b/docs/docs/cli/bib-fetch-docket-comments.md index 0903bad..1570acf 100644 --- a/docs/docs/cli/bib-fetch-docket-comments.md +++ b/docs/docs/cli/bib-fetch-docket-comments.md @@ -1,6 +1,6 @@ --- title: stack bib fetch-docket-comments -sidebar_position: 30 +sidebar_position: 31 --- # `stack bib fetch-docket-comments` diff --git a/docs/docs/cli/bib-fetch-pfs-comments.md b/docs/docs/cli/bib-fetch-pfs-comments.md index 62cd3f8..ba9666c 100644 --- a/docs/docs/cli/bib-fetch-pfs-comments.md +++ b/docs/docs/cli/bib-fetch-pfs-comments.md @@ -1,6 +1,6 @@ --- title: stack bib fetch-pfs-comments -sidebar_position: 31 +sidebar_position: 32 --- # `stack bib fetch-pfs-comments` diff --git a/docs/docs/cli/bib-fr-grab.md b/docs/docs/cli/bib-fr-grab.md index 2ca1534..10922aa 100644 --- a/docs/docs/cli/bib-fr-grab.md +++ b/docs/docs/cli/bib-fr-grab.md @@ -1,6 +1,6 @@ --- title: stack bib fr-grab -sidebar_position: 43 +sidebar_position: 44 --- # `stack bib fr-grab` diff --git a/docs/docs/cli/bib-fr-jump.md b/docs/docs/cli/bib-fr-jump.md index 8dafaa9..e786a3b 100644 --- a/docs/docs/cli/bib-fr-jump.md +++ b/docs/docs/cli/bib-fr-jump.md @@ -1,6 +1,6 @@ --- title: stack bib fr-jump -sidebar_position: 44 +sidebar_position: 45 --- # `stack bib fr-jump` diff --git a/docs/docs/cli/bib-import-zotero.md b/docs/docs/cli/bib-import-zotero.md index e2873a0..10f565b 100644 --- a/docs/docs/cli/bib-import-zotero.md +++ b/docs/docs/cli/bib-import-zotero.md @@ -1,6 +1,6 @@ --- title: stack bib import-zotero -sidebar_position: 37 +sidebar_position: 38 --- # `stack bib import-zotero` diff --git a/docs/docs/cli/bib-ingest-iom.md b/docs/docs/cli/bib-ingest-iom.md index deeb2b8..782eef4 100644 --- a/docs/docs/cli/bib-ingest-iom.md +++ b/docs/docs/cli/bib-ingest-iom.md @@ -1,6 +1,6 @@ --- title: stack bib ingest-iom -sidebar_position: 34 +sidebar_position: 35 --- # `stack bib ingest-iom` diff --git a/docs/docs/cli/bib-ingest-mail.md b/docs/docs/cli/bib-ingest-mail.md index 2e1ec78..6718736 100644 --- a/docs/docs/cli/bib-ingest-mail.md +++ b/docs/docs/cli/bib-ingest-mail.md @@ -1,6 +1,6 @@ --- title: stack bib ingest-mail -sidebar_position: 32 +sidebar_position: 33 --- # `stack bib ingest-mail` diff --git a/docs/docs/cli/bib-ingest-oig.md b/docs/docs/cli/bib-ingest-oig.md index fddff7d..caa38d6 100644 --- a/docs/docs/cli/bib-ingest-oig.md +++ b/docs/docs/cli/bib-ingest-oig.md @@ -1,6 +1,6 @@ --- title: stack bib ingest-oig -sidebar_position: 39 +sidebar_position: 40 --- # `stack bib ingest-oig` diff --git a/docs/docs/cli/bib-query.md b/docs/docs/cli/bib-query.md index ac85327..2d8c480 100644 --- a/docs/docs/cli/bib-query.md +++ b/docs/docs/cli/bib-query.md @@ -1,6 +1,6 @@ --- title: stack bib query -sidebar_position: 28 +sidebar_position: 29 --- # `stack bib query` diff --git a/docs/docs/cli/bib-refresh-iom.md b/docs/docs/cli/bib-refresh-iom.md index 83c3342..74f47d8 100644 --- a/docs/docs/cli/bib-refresh-iom.md +++ b/docs/docs/cli/bib-refresh-iom.md @@ -1,6 +1,6 @@ --- title: stack bib refresh-iom -sidebar_position: 42 +sidebar_position: 43 --- # `stack bib refresh-iom` diff --git a/docs/docs/cli/bib-refresh-oig.md b/docs/docs/cli/bib-refresh-oig.md index 00ca821..0846147 100644 --- a/docs/docs/cli/bib-refresh-oig.md +++ b/docs/docs/cli/bib-refresh-oig.md @@ -1,6 +1,6 @@ --- title: stack bib refresh-oig -sidebar_position: 41 +sidebar_position: 42 --- # `stack bib refresh-oig` diff --git a/docs/docs/cli/bib-sync-zotero.md b/docs/docs/cli/bib-sync-zotero.md index 76b9993..6b596c6 100644 --- a/docs/docs/cli/bib-sync-zotero.md +++ b/docs/docs/cli/bib-sync-zotero.md @@ -1,6 +1,6 @@ --- title: stack bib sync-zotero -sidebar_position: 38 +sidebar_position: 39 --- # `stack bib sync-zotero` diff --git a/docs/docs/cli/bib-sync.md b/docs/docs/cli/bib-sync.md index ab54893..28ef45b 100644 --- a/docs/docs/cli/bib-sync.md +++ b/docs/docs/cli/bib-sync.md @@ -1,6 +1,6 @@ --- title: stack bib sync -sidebar_position: 26 +sidebar_position: 27 --- # `stack bib sync` diff --git a/docs/docs/cli/bib-tag.md b/docs/docs/cli/bib-tag.md index 7f1b9b9..95f833e 100644 --- a/docs/docs/cli/bib-tag.md +++ b/docs/docs/cli/bib-tag.md @@ -1,6 +1,6 @@ --- title: stack bib tag -sidebar_position: 27 +sidebar_position: 28 --- # `stack bib tag` diff --git a/docs/docs/cli/bib-watch-iom.md b/docs/docs/cli/bib-watch-iom.md index b80b28a..bd1d543 100644 --- a/docs/docs/cli/bib-watch-iom.md +++ b/docs/docs/cli/bib-watch-iom.md @@ -1,6 +1,6 @@ --- title: stack bib watch-iom -sidebar_position: 36 +sidebar_position: 37 --- # `stack bib watch-iom` diff --git a/docs/docs/cli/bib.md b/docs/docs/cli/bib.md index 286f03f..2a9dfcd 100644 --- a/docs/docs/cli/bib.md +++ b/docs/docs/cli/bib.md @@ -1,6 +1,6 @@ --- title: stack bib -sidebar_position: 25 +sidebar_position: 26 --- # `stack bib` diff --git a/docs/docs/cli/bls-ces.md b/docs/docs/cli/bls-ces.md index cdabc92..e4e64d4 100644 --- a/docs/docs/cli/bls-ces.md +++ b/docs/docs/cli/bls-ces.md @@ -1,6 +1,6 @@ --- title: stack bls ces -sidebar_position: 49 +sidebar_position: 50 --- # `stack bls ces` diff --git a/docs/docs/cli/bls-oews.md b/docs/docs/cli/bls-oews.md index 55820cf..f377066 100644 --- a/docs/docs/cli/bls-oews.md +++ b/docs/docs/cli/bls-oews.md @@ -1,6 +1,6 @@ --- title: stack bls oews -sidebar_position: 48 +sidebar_position: 49 --- # `stack bls oews` diff --git a/docs/docs/cli/bls.md b/docs/docs/cli/bls.md index 14a0ea9..29be165 100644 --- a/docs/docs/cli/bls.md +++ b/docs/docs/cli/bls.md @@ -1,6 +1,6 @@ --- title: stack bls -sidebar_position: 47 +sidebar_position: 48 --- # `stack bls` diff --git a/docs/docs/cli/comments-dockets.md b/docs/docs/cli/comments-dockets.md index 35624d1..a2dd72f 100644 --- a/docs/docs/cli/comments-dockets.md +++ b/docs/docs/cli/comments-dockets.md @@ -1,6 +1,6 @@ --- title: stack comments dockets -sidebar_position: 54 +sidebar_position: 55 --- # `stack comments dockets` diff --git a/docs/docs/cli/comments-extract-ocr.md b/docs/docs/cli/comments-extract-ocr.md index 8fc3592..9d5285e 100644 --- a/docs/docs/cli/comments-extract-ocr.md +++ b/docs/docs/cli/comments-extract-ocr.md @@ -1,6 +1,6 @@ --- title: stack comments extract-ocr -sidebar_position: 52 +sidebar_position: 53 --- # `stack comments extract-ocr` diff --git a/docs/docs/cli/comments-extract.md b/docs/docs/cli/comments-extract.md index e97b2bc..2a03b59 100644 --- a/docs/docs/cli/comments-extract.md +++ b/docs/docs/cli/comments-extract.md @@ -1,6 +1,6 @@ --- title: stack comments extract -sidebar_position: 51 +sidebar_position: 52 --- # `stack comments extract` diff --git a/docs/docs/cli/comments-seal.md b/docs/docs/cli/comments-seal.md index 8151f5f..ff5fe35 100644 --- a/docs/docs/cli/comments-seal.md +++ b/docs/docs/cli/comments-seal.md @@ -1,6 +1,6 @@ --- title: stack comments seal -sidebar_position: 55 +sidebar_position: 56 --- # `stack comments seal` diff --git a/docs/docs/cli/comments-stats.md b/docs/docs/cli/comments-stats.md index 32f994f..b1a6998 100644 --- a/docs/docs/cli/comments-stats.md +++ b/docs/docs/cli/comments-stats.md @@ -1,6 +1,6 @@ --- title: stack comments stats -sidebar_position: 53 +sidebar_position: 54 --- # `stack comments stats` diff --git a/docs/docs/cli/comments-unseal.md b/docs/docs/cli/comments-unseal.md index 1cc4ca5..fab72b0 100644 --- a/docs/docs/cli/comments-unseal.md +++ b/docs/docs/cli/comments-unseal.md @@ -1,6 +1,6 @@ --- title: stack comments unseal -sidebar_position: 56 +sidebar_position: 57 --- # `stack comments unseal` diff --git a/docs/docs/cli/comments.md b/docs/docs/cli/comments.md index 5c6f7cd..5df41a1 100644 --- a/docs/docs/cli/comments.md +++ b/docs/docs/cli/comments.md @@ -1,6 +1,6 @@ --- title: stack comments -sidebar_position: 50 +sidebar_position: 51 --- # `stack comments` diff --git a/docs/docs/cli/db-comment.md b/docs/docs/cli/db-comment.md index a3ac5a7..8cf803f 100644 --- a/docs/docs/cli/db-comment.md +++ b/docs/docs/cli/db-comment.md @@ -1,6 +1,6 @@ --- title: stack db comment -sidebar_position: 73 +sidebar_position: 74 --- # `stack db comment` diff --git a/docs/docs/cli/db-inspect.md b/docs/docs/cli/db-inspect.md index 28d5b2d..f9a2780 100644 --- a/docs/docs/cli/db-inspect.md +++ b/docs/docs/cli/db-inspect.md @@ -1,6 +1,6 @@ --- title: stack db inspect -sidebar_position: 74 +sidebar_position: 75 --- # `stack db inspect` diff --git a/docs/docs/cli/db.md b/docs/docs/cli/db.md index 309e7bb..0315be5 100644 --- a/docs/docs/cli/db.md +++ b/docs/docs/cli/db.md @@ -1,6 +1,6 @@ --- title: stack db -sidebar_position: 72 +sidebar_position: 73 --- # `stack db` diff --git a/docs/docs/cli/docs-build.md b/docs/docs/cli/docs-build.md index 07c6175..ce2ac24 100644 --- a/docs/docs/cli/docs-build.md +++ b/docs/docs/cli/docs-build.md @@ -1,6 +1,6 @@ --- title: stack docs build -sidebar_position: 76 +sidebar_position: 77 --- # `stack docs build` diff --git a/docs/docs/cli/docs-generate.md b/docs/docs/cli/docs-generate.md index a7ea1a5..f5ab425 100644 --- a/docs/docs/cli/docs-generate.md +++ b/docs/docs/cli/docs-generate.md @@ -1,6 +1,6 @@ --- title: stack docs generate -sidebar_position: 78 +sidebar_position: 79 --- # `stack docs generate` diff --git a/docs/docs/cli/docs-serve.md b/docs/docs/cli/docs-serve.md index 4fede99..c096693 100644 --- a/docs/docs/cli/docs-serve.md +++ b/docs/docs/cli/docs-serve.md @@ -1,6 +1,6 @@ --- title: stack docs serve -sidebar_position: 77 +sidebar_position: 78 --- # `stack docs serve` diff --git a/docs/docs/cli/docs.md b/docs/docs/cli/docs.md index a05afb4..f0eca86 100644 --- a/docs/docs/cli/docs.md +++ b/docs/docs/cli/docs.md @@ -1,6 +1,6 @@ --- title: stack docs -sidebar_position: 75 +sidebar_position: 76 --- # `stack docs` diff --git a/docs/docs/cli/generate-models.md b/docs/docs/cli/generate-models.md index 5794474..d0e3bed 100644 --- a/docs/docs/cli/generate-models.md +++ b/docs/docs/cli/generate-models.md @@ -1,6 +1,6 @@ --- title: stack generate models -sidebar_position: 24 +sidebar_position: 25 --- # `stack generate models` diff --git a/docs/docs/cli/generate.md b/docs/docs/cli/generate.md index fbbb526..2d6153c 100644 --- a/docs/docs/cli/generate.md +++ b/docs/docs/cli/generate.md @@ -1,6 +1,6 @@ --- title: stack generate -sidebar_position: 23 +sidebar_position: 24 --- # `stack generate` diff --git a/docs/docs/cli/lake-deploy.md b/docs/docs/cli/lake-deploy.md index 796afb9..b758820 100644 --- a/docs/docs/cli/lake-deploy.md +++ b/docs/docs/cli/lake-deploy.md @@ -1,6 +1,6 @@ --- title: stack lake deploy -sidebar_position: 58 +sidebar_position: 59 --- # `stack lake deploy` diff --git a/docs/docs/cli/lake-load.md b/docs/docs/cli/lake-load.md index bfa4a06..caf1e15 100644 --- a/docs/docs/cli/lake-load.md +++ b/docs/docs/cli/lake-load.md @@ -1,6 +1,6 @@ --- title: stack lake load -sidebar_position: 59 +sidebar_position: 60 --- # `stack lake load` diff --git a/docs/docs/cli/lake-parity.md b/docs/docs/cli/lake-parity.md index 222d968..7ee9cfa 100644 --- a/docs/docs/cli/lake-parity.md +++ b/docs/docs/cli/lake-parity.md @@ -1,6 +1,6 @@ --- title: stack lake parity -sidebar_position: 61 +sidebar_position: 62 --- # `stack lake parity` diff --git a/docs/docs/cli/lake-validate.md b/docs/docs/cli/lake-validate.md index eddac98..d690b52 100644 --- a/docs/docs/cli/lake-validate.md +++ b/docs/docs/cli/lake-validate.md @@ -1,6 +1,6 @@ --- title: stack lake validate -sidebar_position: 60 +sidebar_position: 61 --- # `stack lake validate` diff --git a/docs/docs/cli/lake.md b/docs/docs/cli/lake.md index 2333909..26c4ad6 100644 --- a/docs/docs/cli/lake.md +++ b/docs/docs/cli/lake.md @@ -1,6 +1,6 @@ --- title: stack lake -sidebar_position: 57 +sidebar_position: 58 --- # `stack lake` diff --git a/docs/docs/cli/llm-eval-tags.md b/docs/docs/cli/llm-eval-tags.md index b0582af..7542e53 100644 --- a/docs/docs/cli/llm-eval-tags.md +++ b/docs/docs/cli/llm-eval-tags.md @@ -1,6 +1,6 @@ --- title: stack llm eval-tags -sidebar_position: 69 +sidebar_position: 70 --- # `stack llm eval-tags` diff --git a/docs/docs/cli/llm-hosts.md b/docs/docs/cli/llm-hosts.md index 9b00348..a8c1668 100644 --- a/docs/docs/cli/llm-hosts.md +++ b/docs/docs/cli/llm-hosts.md @@ -1,6 +1,6 @@ --- title: stack llm hosts -sidebar_position: 65 +sidebar_position: 66 --- # `stack llm hosts` diff --git a/docs/docs/cli/llm-index.md b/docs/docs/cli/llm-index.md index f038d0e..ae6588b 100644 --- a/docs/docs/cli/llm-index.md +++ b/docs/docs/cli/llm-index.md @@ -1,6 +1,6 @@ --- title: stack llm index -sidebar_position: 63 +sidebar_position: 64 --- # `stack llm index` diff --git a/docs/docs/cli/llm-prune-rules.md b/docs/docs/cli/llm-prune-rules.md index 1792b91..bff1dec 100644 --- a/docs/docs/cli/llm-prune-rules.md +++ b/docs/docs/cli/llm-prune-rules.md @@ -1,6 +1,6 @@ --- title: stack llm prune-rules -sidebar_position: 70 +sidebar_position: 71 --- # `stack llm prune-rules` diff --git a/docs/docs/cli/llm-restamp.md b/docs/docs/cli/llm-restamp.md index 15a3774..68890d7 100644 --- a/docs/docs/cli/llm-restamp.md +++ b/docs/docs/cli/llm-restamp.md @@ -1,6 +1,6 @@ --- title: stack llm restamp -sidebar_position: 64 +sidebar_position: 65 --- # `stack llm restamp` diff --git a/docs/docs/cli/llm-serve.md b/docs/docs/cli/llm-serve.md index 17b8fee..3a6a6fb 100644 --- a/docs/docs/cli/llm-serve.md +++ b/docs/docs/cli/llm-serve.md @@ -1,6 +1,6 @@ --- title: stack llm serve -sidebar_position: 66 +sidebar_position: 67 --- # `stack llm serve` diff --git a/docs/docs/cli/llm-stamp-themes.md b/docs/docs/cli/llm-stamp-themes.md index 5742bbe..ff7dbaf 100644 --- a/docs/docs/cli/llm-stamp-themes.md +++ b/docs/docs/cli/llm-stamp-themes.md @@ -1,6 +1,6 @@ --- title: stack llm stamp-themes -sidebar_position: 71 +sidebar_position: 72 --- # `stack llm stamp-themes` diff --git a/docs/docs/cli/llm-tag.md b/docs/docs/cli/llm-tag.md index 8235894..673cc71 100644 --- a/docs/docs/cli/llm-tag.md +++ b/docs/docs/cli/llm-tag.md @@ -1,6 +1,6 @@ --- title: stack llm tag -sidebar_position: 68 +sidebar_position: 69 --- # `stack llm tag` diff --git a/docs/docs/cli/llm-vocab.md b/docs/docs/cli/llm-vocab.md index 7ee5363..c02f11d 100644 --- a/docs/docs/cli/llm-vocab.md +++ b/docs/docs/cli/llm-vocab.md @@ -1,6 +1,6 @@ --- title: stack llm vocab -sidebar_position: 67 +sidebar_position: 68 --- # `stack llm vocab` diff --git a/docs/docs/cli/llm.md b/docs/docs/cli/llm.md index c66c4cd..69abd96 100644 --- a/docs/docs/cli/llm.md +++ b/docs/docs/cli/llm.md @@ -1,6 +1,6 @@ --- title: stack llm -sidebar_position: 62 +sidebar_position: 63 --- # `stack llm` diff --git a/docs/docs/cli/load-cms.md b/docs/docs/cli/load-cms.md new file mode 100644 index 0000000..4d32158 --- /dev/null +++ b/docs/docs/cli/load-cms.md @@ -0,0 +1,30 @@ +--- +title: stack load cms +sidebar_position: 8 +--- + +# `stack load cms` + +``` +Usage: stack load cms [OPTIONS] + + Vendor the institutional provider universe from data.cms.gov: CCN ↔ NPI ↔ TIN. + + PECOS enrollments (hospital, SNF, HHA, hospice, FQHC, RHC), the + Provider of Services file and the cost reports, loaded as ``cms.*`` + with their models' column names, then the crosswalk tables. CMS + publishes no TIN; ``cms.ccn_tin`` carries one only where our own + claims or ``cms.aco_participant_tin`` know it. + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --table -t TEXT cms. (default: the institutional │ +│ universe). │ +│ --database TEXT DuckDB database path (default: config). │ +│ --refresh Re-download even if the CSV is on disk. │ +│ --derive --no-derive Rebuild institutional_provider / │ +│ ccn_crosswalk / provider_tin_observed / │ +│ ccn_tin. │ +│ [default: derive] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +``` diff --git a/docs/docs/cli/load-seed.md b/docs/docs/cli/load-seed.md index 4a5c5e6..18a7501 100644 --- a/docs/docs/cli/load-seed.md +++ b/docs/docs/cli/load-seed.md @@ -1,6 +1,6 @@ --- title: stack load seed -sidebar_position: 8 +sidebar_position: 9 --- # `stack load seed` diff --git a/docs/docs/cli/load.md b/docs/docs/cli/load.md index 0772e4a..6447783 100644 --- a/docs/docs/cli/load.md +++ b/docs/docs/cli/load.md @@ -16,6 +16,8 @@ Usage: stack load [OPTIONS] COMMAND [ARGS]... ╭─ Commands ───────────────────────────────────────────────────────────────────╮ │ cclf Ingest CCLF claim files into DuckDB. │ │ bcda Flatten BCDA FHIR exports and load into DuckDB. │ +│ cms Vendor the institutional provider universe from data.cms.gov: CCN ↔ │ +│ NPI ↔ TIN. │ │ seed Load seed/reference data (CSV, Excel, Parquet) into DuckDB. │ ╰──────────────────────────────────────────────────────────────────────────────╯ ``` diff --git a/docs/docs/cli/mail-attach-smarthost.md b/docs/docs/cli/mail-attach-smarthost.md index f287be7..043b0ee 100644 --- a/docs/docs/cli/mail-attach-smarthost.md +++ b/docs/docs/cli/mail-attach-smarthost.md @@ -1,6 +1,6 @@ --- title: stack mail attach-smarthost -sidebar_position: 121 +sidebar_position: 122 --- # `stack mail attach-smarthost` diff --git a/docs/docs/cli/mail-dkim-export.md b/docs/docs/cli/mail-dkim-export.md index 40eb1e3..867d1da 100644 --- a/docs/docs/cli/mail-dkim-export.md +++ b/docs/docs/cli/mail-dkim-export.md @@ -1,6 +1,6 @@ --- title: stack mail dkim-export -sidebar_position: 120 +sidebar_position: 121 --- # `stack mail dkim-export` diff --git a/docs/docs/cli/mail-dns.md b/docs/docs/cli/mail-dns.md index 725950a..893943c 100644 --- a/docs/docs/cli/mail-dns.md +++ b/docs/docs/cli/mail-dns.md @@ -1,6 +1,6 @@ --- title: stack mail dns -sidebar_position: 119 +sidebar_position: 120 --- # `stack mail dns` diff --git a/docs/docs/cli/mail-down.md b/docs/docs/cli/mail-down.md index 9309ef8..27dc331 100644 --- a/docs/docs/cli/mail-down.md +++ b/docs/docs/cli/mail-down.md @@ -1,6 +1,6 @@ --- title: stack mail down -sidebar_position: 117 +sidebar_position: 118 --- # `stack mail down` diff --git a/docs/docs/cli/mail-provision.md b/docs/docs/cli/mail-provision.md index b108d73..11cf506 100644 --- a/docs/docs/cli/mail-provision.md +++ b/docs/docs/cli/mail-provision.md @@ -1,6 +1,6 @@ --- title: stack mail provision -sidebar_position: 115 +sidebar_position: 116 --- # `stack mail provision` diff --git a/docs/docs/cli/mail-rotate-creds.md b/docs/docs/cli/mail-rotate-creds.md index bf4e438..a6489e0 100644 --- a/docs/docs/cli/mail-rotate-creds.md +++ b/docs/docs/cli/mail-rotate-creds.md @@ -1,6 +1,6 @@ --- title: stack mail rotate-creds -sidebar_position: 122 +sidebar_position: 123 --- # `stack mail rotate-creds` diff --git a/docs/docs/cli/mail-seed-mailboxes.md b/docs/docs/cli/mail-seed-mailboxes.md index d583b96..8051a29 100644 --- a/docs/docs/cli/mail-seed-mailboxes.md +++ b/docs/docs/cli/mail-seed-mailboxes.md @@ -1,6 +1,6 @@ --- title: stack mail seed-mailboxes -sidebar_position: 123 +sidebar_position: 124 --- # `stack mail seed-mailboxes` diff --git a/docs/docs/cli/mail-status.md b/docs/docs/cli/mail-status.md index c5424de..e135886 100644 --- a/docs/docs/cli/mail-status.md +++ b/docs/docs/cli/mail-status.md @@ -1,6 +1,6 @@ --- title: stack mail status -sidebar_position: 118 +sidebar_position: 119 --- # `stack mail status` diff --git a/docs/docs/cli/mail-up.md b/docs/docs/cli/mail-up.md index 78a2e78..fcd5b90 100644 --- a/docs/docs/cli/mail-up.md +++ b/docs/docs/cli/mail-up.md @@ -1,6 +1,6 @@ --- title: stack mail up -sidebar_position: 116 +sidebar_position: 117 --- # `stack mail up` diff --git a/docs/docs/cli/mail-wire-git.md b/docs/docs/cli/mail-wire-git.md index 3b7a781..72df4c5 100644 --- a/docs/docs/cli/mail-wire-git.md +++ b/docs/docs/cli/mail-wire-git.md @@ -1,6 +1,6 @@ --- title: stack mail wire-git -sidebar_position: 124 +sidebar_position: 125 --- # `stack mail wire-git` diff --git a/docs/docs/cli/mail.md b/docs/docs/cli/mail.md index 510a133..a663dba 100644 --- a/docs/docs/cli/mail.md +++ b/docs/docs/cli/mail.md @@ -1,6 +1,6 @@ --- title: stack mail -sidebar_position: 114 +sidebar_position: 115 --- # `stack mail` diff --git a/docs/docs/cli/perf-show.md b/docs/docs/cli/perf-show.md index 54dac5c..5a65b6f 100644 --- a/docs/docs/cli/perf-show.md +++ b/docs/docs/cli/perf-show.md @@ -1,6 +1,6 @@ --- title: stack perf show -sidebar_position: 82 +sidebar_position: 83 --- # `stack perf show` diff --git a/docs/docs/cli/perf.md b/docs/docs/cli/perf.md index 7524bd4..e7fadc0 100644 --- a/docs/docs/cli/perf.md +++ b/docs/docs/cli/perf.md @@ -1,6 +1,6 @@ --- title: stack perf -sidebar_position: 81 +sidebar_position: 82 --- # `stack perf` diff --git a/docs/docs/cli/pfs-cpt-ingest.md b/docs/docs/cli/pfs-cpt-ingest.md index cc0eadc..27b76c8 100644 --- a/docs/docs/cli/pfs-cpt-ingest.md +++ b/docs/docs/cli/pfs-cpt-ingest.md @@ -1,6 +1,6 @@ --- title: stack pfs cpt-ingest -sidebar_position: 96 +sidebar_position: 97 --- # `stack pfs cpt-ingest` diff --git a/docs/docs/cli/pfs-elements.md b/docs/docs/cli/pfs-elements.md index 0702a65..f7b0cba 100644 --- a/docs/docs/cli/pfs-elements.md +++ b/docs/docs/cli/pfs-elements.md @@ -1,6 +1,6 @@ --- title: stack pfs elements -sidebar_position: 88 +sidebar_position: 89 --- # `stack pfs elements` diff --git a/docs/docs/cli/pfs-exposure.md b/docs/docs/cli/pfs-exposure.md index 59bb8f9..468634f 100644 --- a/docs/docs/cli/pfs-exposure.md +++ b/docs/docs/cli/pfs-exposure.md @@ -1,6 +1,6 @@ --- title: stack pfs exposure -sidebar_position: 93 +sidebar_position: 94 --- # `stack pfs exposure` diff --git a/docs/docs/cli/pfs-families.md b/docs/docs/cli/pfs-families.md index c7ea024..60faed0 100644 --- a/docs/docs/cli/pfs-families.md +++ b/docs/docs/cli/pfs-families.md @@ -1,6 +1,6 @@ --- title: stack pfs families -sidebar_position: 90 +sidebar_position: 91 --- # `stack pfs families` diff --git a/docs/docs/cli/pfs-guidance.md b/docs/docs/cli/pfs-guidance.md index 8796860..cb43089 100644 --- a/docs/docs/cli/pfs-guidance.md +++ b/docs/docs/cli/pfs-guidance.md @@ -1,6 +1,6 @@ --- title: stack pfs guidance -sidebar_position: 91 +sidebar_position: 92 --- # `stack pfs guidance` diff --git a/docs/docs/cli/pfs-lineage.md b/docs/docs/cli/pfs-lineage.md index 4293131..5e19ef9 100644 --- a/docs/docs/cli/pfs-lineage.md +++ b/docs/docs/cli/pfs-lineage.md @@ -1,6 +1,6 @@ --- title: stack pfs lineage -sidebar_position: 89 +sidebar_position: 90 --- # `stack pfs lineage` diff --git a/docs/docs/cli/pfs-reaction.md b/docs/docs/cli/pfs-reaction.md index 0ddcc2a..3c1d008 100644 --- a/docs/docs/cli/pfs-reaction.md +++ b/docs/docs/cli/pfs-reaction.md @@ -1,6 +1,6 @@ --- title: stack pfs reaction -sidebar_position: 92 +sidebar_position: 93 --- # `stack pfs reaction` diff --git a/docs/docs/cli/pfs-review.md b/docs/docs/cli/pfs-review.md index 4c5c6b1..5a57f76 100644 --- a/docs/docs/cli/pfs-review.md +++ b/docs/docs/cli/pfs-review.md @@ -1,6 +1,6 @@ --- title: stack pfs review -sidebar_position: 95 +sidebar_position: 96 --- # `stack pfs review` diff --git a/docs/docs/cli/pfs-utilization.md b/docs/docs/cli/pfs-utilization.md index 86adcf2..b4a42c0 100644 --- a/docs/docs/cli/pfs-utilization.md +++ b/docs/docs/cli/pfs-utilization.md @@ -1,6 +1,6 @@ --- title: stack pfs utilization -sidebar_position: 94 +sidebar_position: 95 --- # `stack pfs utilization` diff --git a/docs/docs/cli/pfs.md b/docs/docs/cli/pfs.md index adf65d5..f643e9c 100644 --- a/docs/docs/cli/pfs.md +++ b/docs/docs/cli/pfs.md @@ -1,6 +1,6 @@ --- title: stack pfs -sidebar_position: 87 +sidebar_position: 88 --- # `stack pfs` diff --git a/docs/docs/cli/prisma-eligible.md b/docs/docs/cli/prisma-eligible.md index c1f98d6..c906274 100644 --- a/docs/docs/cli/prisma-eligible.md +++ b/docs/docs/cli/prisma-eligible.md @@ -1,6 +1,6 @@ --- title: stack prisma eligible -sidebar_position: 108 +sidebar_position: 109 --- # `stack prisma eligible` diff --git a/docs/docs/cli/prisma-export.md b/docs/docs/cli/prisma-export.md index e496332..ca54166 100644 --- a/docs/docs/cli/prisma-export.md +++ b/docs/docs/cli/prisma-export.md @@ -1,6 +1,6 @@ --- title: stack prisma export -sidebar_position: 105 +sidebar_position: 106 --- # `stack prisma export` diff --git a/docs/docs/cli/prisma-extract.md b/docs/docs/cli/prisma-extract.md index 99f66bf..d58d124 100644 --- a/docs/docs/cli/prisma-extract.md +++ b/docs/docs/cli/prisma-extract.md @@ -1,6 +1,6 @@ --- title: stack prisma extract -sidebar_position: 109 +sidebar_position: 110 --- # `stack prisma extract` diff --git a/docs/docs/cli/prisma-fetch.md b/docs/docs/cli/prisma-fetch.md index c180b30..b7a1a03 100644 --- a/docs/docs/cli/prisma-fetch.md +++ b/docs/docs/cli/prisma-fetch.md @@ -1,6 +1,6 @@ --- title: stack prisma fetch -sidebar_position: 111 +sidebar_position: 112 --- # `stack prisma fetch` diff --git a/docs/docs/cli/prisma-flow.md b/docs/docs/cli/prisma-flow.md index ea4a1cd..79ceb38 100644 --- a/docs/docs/cli/prisma-flow.md +++ b/docs/docs/cli/prisma-flow.md @@ -1,6 +1,6 @@ --- title: stack prisma flow -sidebar_position: 110 +sidebar_position: 111 --- # `stack prisma flow` diff --git a/docs/docs/cli/prisma-init.md b/docs/docs/cli/prisma-init.md index 6db6ce4..574f139 100644 --- a/docs/docs/cli/prisma-init.md +++ b/docs/docs/cli/prisma-init.md @@ -1,6 +1,6 @@ --- title: stack prisma init -sidebar_position: 104 +sidebar_position: 105 --- # `stack prisma init` diff --git a/docs/docs/cli/prisma-ping-llm.md b/docs/docs/cli/prisma-ping-llm.md index 995c52c..0f6cbc5 100644 --- a/docs/docs/cli/prisma-ping-llm.md +++ b/docs/docs/cli/prisma-ping-llm.md @@ -1,6 +1,6 @@ --- title: stack prisma ping-llm -sidebar_position: 106 +sidebar_position: 107 --- # `stack prisma ping-llm` diff --git a/docs/docs/cli/prisma-run.md b/docs/docs/cli/prisma-run.md index 929c6cb..ce18493 100644 --- a/docs/docs/cli/prisma-run.md +++ b/docs/docs/cli/prisma-run.md @@ -1,6 +1,6 @@ --- title: stack prisma run -sidebar_position: 112 +sidebar_position: 113 --- # `stack prisma run` diff --git a/docs/docs/cli/prisma-screen.md b/docs/docs/cli/prisma-screen.md index b22d248..1e5d76f 100644 --- a/docs/docs/cli/prisma-screen.md +++ b/docs/docs/cli/prisma-screen.md @@ -1,6 +1,6 @@ --- title: stack prisma screen -sidebar_position: 107 +sidebar_position: 108 --- # `stack prisma screen` diff --git a/docs/docs/cli/prisma-vpn.md b/docs/docs/cli/prisma-vpn.md index 430eb0e..e9a6001 100644 --- a/docs/docs/cli/prisma-vpn.md +++ b/docs/docs/cli/prisma-vpn.md @@ -1,6 +1,6 @@ --- title: stack prisma vpn -sidebar_position: 113 +sidebar_position: 114 --- # `stack prisma vpn` diff --git a/docs/docs/cli/prisma.md b/docs/docs/cli/prisma.md index 98d0e08..076fab0 100644 --- a/docs/docs/cli/prisma.md +++ b/docs/docs/cli/prisma.md @@ -1,6 +1,6 @@ --- title: stack prisma -sidebar_position: 103 +sidebar_position: 104 --- # `stack prisma` diff --git a/docs/docs/cli/rec-list.md b/docs/docs/cli/rec-list.md index 1645b31..c97ab93 100644 --- a/docs/docs/cli/rec-list.md +++ b/docs/docs/cli/rec-list.md @@ -1,6 +1,6 @@ --- title: stack rec list -sidebar_position: 84 +sidebar_position: 85 --- # `stack rec list` diff --git a/docs/docs/cli/rec-opps.md b/docs/docs/cli/rec-opps.md index a8f40c1..89a270f 100644 --- a/docs/docs/cli/rec-opps.md +++ b/docs/docs/cli/rec-opps.md @@ -1,6 +1,6 @@ --- title: stack rec opps -sidebar_position: 86 +sidebar_position: 87 --- # `stack rec opps` diff --git a/docs/docs/cli/rec-pfs.md b/docs/docs/cli/rec-pfs.md index 42ccde5..ff85848 100644 --- a/docs/docs/cli/rec-pfs.md +++ b/docs/docs/cli/rec-pfs.md @@ -1,6 +1,6 @@ --- title: stack rec pfs -sidebar_position: 85 +sidebar_position: 86 --- # `stack rec pfs` diff --git a/docs/docs/cli/rec.md b/docs/docs/cli/rec.md index dc47b70..b612b72 100644 --- a/docs/docs/cli/rec.md +++ b/docs/docs/cli/rec.md @@ -1,6 +1,6 @@ --- title: stack rec -sidebar_position: 83 +sidebar_position: 84 --- # `stack rec` diff --git a/docs/docs/cli/zot-dump-schema.md b/docs/docs/cli/zot-dump-schema.md index 44e4001..0e1e5ee 100644 --- a/docs/docs/cli/zot-dump-schema.md +++ b/docs/docs/cli/zot-dump-schema.md @@ -1,6 +1,6 @@ --- title: stack zot dump-schema -sidebar_position: 98 +sidebar_position: 99 --- # `stack zot dump-schema` diff --git a/docs/docs/cli/zot-fix-dates.md b/docs/docs/cli/zot-fix-dates.md index 3890c74..7f7c441 100644 --- a/docs/docs/cli/zot-fix-dates.md +++ b/docs/docs/cli/zot-fix-dates.md @@ -1,6 +1,6 @@ --- title: stack zot fix-dates -sidebar_position: 99 +sidebar_position: 100 --- # `stack zot fix-dates` diff --git a/docs/docs/cli/zot-fix-fields.md b/docs/docs/cli/zot-fix-fields.md index fbbef71..ce3750a 100644 --- a/docs/docs/cli/zot-fix-fields.md +++ b/docs/docs/cli/zot-fix-fields.md @@ -1,6 +1,6 @@ --- title: stack zot fix-fields -sidebar_position: 101 +sidebar_position: 102 --- # `stack zot fix-fields` diff --git a/docs/docs/cli/zot-fix-keys.md b/docs/docs/cli/zot-fix-keys.md index 518b8fb..1af82a4 100644 --- a/docs/docs/cli/zot-fix-keys.md +++ b/docs/docs/cli/zot-fix-keys.md @@ -1,6 +1,6 @@ --- title: stack zot fix-keys -sidebar_position: 100 +sidebar_position: 101 --- # `stack zot fix-keys` diff --git a/docs/docs/cli/zot-verify-parity.md b/docs/docs/cli/zot-verify-parity.md index 92eba08..cdddbdf 100644 --- a/docs/docs/cli/zot-verify-parity.md +++ b/docs/docs/cli/zot-verify-parity.md @@ -1,6 +1,6 @@ --- title: stack zot verify-parity -sidebar_position: 102 +sidebar_position: 103 --- # `stack zot verify-parity` diff --git a/docs/docs/cli/zot.md b/docs/docs/cli/zot.md index 2f945ca..354c651 100644 --- a/docs/docs/cli/zot.md +++ b/docs/docs/cli/zot.md @@ -1,6 +1,6 @@ --- title: stack zot -sidebar_position: 97 +sidebar_position: 98 --- # `stack zot` diff --git a/docs/superpowers/specs/2026-10-25-bcda-v3-design.md b/docs/superpowers/specs/2026-10-25-bcda-v3-design.md index eae8156..5702e1e 100644 --- a/docs/superpowers/specs/2026-10-25-bcda-v3-design.md +++ b/docs/superpowers/specs/2026-10-25-bcda-v3-design.md @@ -96,3 +96,9 @@ House style: `src/bcda/ui/theme.toml` is a Streamlit theme file carrying the das ### Incremental pulls (2026-10-25) `_since` pulls are reconciled rather than loaded over the baseline — watermarks, temporality checks, tiebreakers, ledger and invariants are in `2026-10-25-bcda-since-reconciliation-design.md` (`bcda.sync`, `stack bcda sync`, the manager's **Sync** page and the Exports page's `_since` modes). + +### Provider identity: "no TIN available" (2026-10-25) + +Institutional claims never carry a TIN. In the real Large-entity v3 pull every claim type has 0 billing TINs and every institutional claim has 0 CCNs (the dictionary's `NamingSystem/CCN` identifier on the billing Organization is not emitted; half of the institutional claims have no billing provider at all — `provider` is a data-absent extension). CCLF is the same: CCLF1 has an OSCAR/CCN and NPIs, no TIN; the only TIN on claims is CCLF5's `CLM_RNDRG_PRVDR_TAX_NUM`, the TIN a carrier/DME line was billed under. CMS publishes no provider TIN or EIN anywhere on data.cms.gov — PECOS enrollments, All Owners, Change of Ownership, Provider of Services and cost reports stop at CCN · NPI · PAC id · enrollment id · name. + +What exists now: the connector gives professional and DME lines their carrier TIN as `billing_tin`; `stack load cms` vendors the institutional universe (hospital, SNF, HHA, hospice, FQHC, RHC enrollments; Provider of Services; the three cost reports — 65,161 CCN↔NPI rows on 2026-10-25) into `cms.*` and derives `cms.institutional_provider`, `cms.ccn_crosswalk`, `cms.provider_tin_observed` (NPI→TIN as seen on our own claims) and `cms.ccn_tin` (CCN→NPI→TIN, with `cms.aco_participant_tin` as the authoritative hook when a participant list is loaded). On the sandbox the crosswalk resolves nothing — its 8 facility NPIs are synthetic — which is the expected result; in production the coverage line from `stack load cms` is the number to watch. diff --git a/src/aco/load/cms.py b/src/aco/load/cms.py new file mode 100644 index 0000000..54ab02a --- /dev/null +++ b/src/aco/load/cms.py @@ -0,0 +1,381 @@ +"""Load data.cms.gov datasets into ``cms.*`` and build the CCN ↔ NPI ↔ TIN crosswalk. + +``stack load cms`` (default: the institutional universe in +:data:`cms.datacms.INSTITUTIONAL`) downloads each family's newest CSV +to ``data/cms/raw/``, loads it as ``cms.
`` with the generated +models' column names plus ``_snapshot_date`` / ``_source_url`` / +``_loaded_at``, records the ingest in ``cms.ingest_log``, then derives: + +``cms.institutional_provider`` + One row per PECOS enrollment across hospitals, SNFs, HHAs, hospices, + FQHCs and RHCs: CCN, NPI, PAC id, enrollment id, type, legal and DBA + names, address, ``source`` table. +``cms.ccn_crosswalk`` + One row per (CCN, NPI) — PECOS enrollments ∪ Provider of Services + CCNs that PECOS lacks (terminated facilities, with ``npi`` NULL) — + with the POS facility name, category and termination, and the cost + report's type of control when loaded. +``cms.provider_tin_observed`` + NPI → TIN as actually seen on our claims (``bcda.explanation_of_benefit``: + the carrier/DME rendering TIN, and billing TINs where a payload has + them), with role, claim counts and first/last ``lastUpdated``. The only + TIN source CMS data offers. +``cms.ccn_tin`` + CCN → NPI → TIN: the crosswalk joined to observed TINs and to + ``cms.aco_participant_tin`` (a table you may load from your entity's + participant list: ``ccn``, ``npi``, ``tin``, ``source``), with + ``tin_source`` saying which supplied it. NULL where nothing public or + observed knows the TIN — which is the honest state for most facilities. + +Why: see :mod:`cms.datacms`. Institutional claims never carry a TIN +(BCDA v3 or CCLF), and the v3 sandbox payload does not carry the CCN +either, so facility identity on claims is the NPI; this crosswalk is how +an NPI becomes a CCN and, where anything knows it, a TIN. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any, Callable + +log = logging.getLogger(__name__) + +PROVENANCE = ("_snapshot_date", "_source_url", "_loaded_at") + + +def load_cms( + tables: list[str] | None = None, + database: str | None = None, + *, + raw_dir: str | Path | None = None, + refresh: bool = False, + derive: bool = True, + datasets: list[dict] | None = None, + fetch: Callable[..., Path] | None = None, +) -> dict[str, int]: + """Download, load and derive. Returns ``{table: rows}``. + + *datasets* (a catalog list) and *fetch* (``(Dataset, dest_dir, refresh) -> Path``) + are injection points for tests; by default the live catalog and + :func:`cms.datacms.download` are used. + """ + import duckdb + + from cms import datacms + from cms.ingest_log import log_ingest, new_run_id + from conf import ROOT, path + + db_path = database or str(path("db.aco")) + raw = Path(raw_dir) if raw_dir else ROOT / "data" / "cms" / "raw" + names = list(tables or datacms.INSTITUTIONAL) + unknown = [t for t in names if t not in datacms.INSTITUTIONAL] + if unknown: + raise KeyError(f"not in cms.datacms.INSTITUTIONAL: {unknown}") + if datasets is None: + datasets = datacms.catalog(raw / "catalog.json", refresh=refresh) + fetch = fetch or datacms.download + + con = duckdb.connect(str(db_path), read_only=False) + con.execute('CREATE SCHEMA IF NOT EXISTS "cms"') + run_id = new_run_id() + stats: dict[str, int] = {} + try: + for t in names: + ds = datacms.resolve(t, datasets) + src = fetch(ds, raw, refresh=refresh) + n = _load_csv(con, t, src, ds) + stats[f"cms.{t}"] = n + log_ingest( + con, + module="cms", + table_name=f"cms.{t}", + rows=n, + source_file=str(src), + run_id=run_id, + ) + log.info("loaded cms.%s: %d rows from %s", t, n, src.name) + if derive: + stats.update(derive_crosswalk(con, run_id=run_id)) + finally: + con.close() + return stats + + +def _load_csv(con: Any, table: str, src: Path, ds: Any) -> int: + """``cms.
`` from *src* with sanitised column names and provenance.""" + from cms.datacms import now_iso, sanitize_column_name + + lit = str(src).replace("'", "''") + reader = ( + f"read_csv('{lit}', header=true, all_varchar=true, ignore_errors=true, " + "sample_size=-1, strict_mode=false)" + ) + cols = [r[0] for r in con.execute(f"describe select * from {reader}").fetchall()] + seen: dict[str, int] = {} + select: list[str] = [] + for c in cols: + name = sanitize_column_name(c) or "col" + k = seen.get(name, 0) + seen[name] = k + 1 + if k: + name = f"{name}_{k + 1}" + select.append(f'"{c.replace(chr(34), chr(34) * 2)}" as "{name}"') + prov = ( + f"'{ds.snapshot_date}' as _snapshot_date, " + f"'{(ds.csv_url or ds.api_url).replace(chr(39), chr(39) * 2)}' as _source_url, " + f"'{now_iso()}' as _loaded_at" + ) + con.execute( + f'CREATE OR REPLACE TABLE "cms"."{table}" AS SELECT {", ".join(select)}, {prov} FROM {reader}' + ) + return con.execute(f'SELECT count(*) FROM "cms"."{table}"').fetchone()[0] + + +def _has(con: Any, schema: str, table: str) -> bool: + return bool( + con.execute( + "select count(*) from information_schema.tables where table_schema = ? and table_name = ?", + [schema, table], + ).fetchone()[0] + ) + + +def _cols(con: Any, schema: str, table: str) -> set[str]: + return { + r[0] + for r in con.execute( + "select column_name from information_schema.columns where table_schema = ? and table_name = ?", + [schema, table], + ).fetchall() + } + + +_ENROLLMENT_COLS = ( + "enrollment_id", + "enrollment_state", + "provider_type_code", + "provider_type_text", + "npi", + "multiple_npi_flag", + "ccn", + "associate_id", + "organization_name", + "doing_business_as_name", + "incorporation_date", + "incorporation_state", + "organization_type_structure", + "organization_other_type_text", + "proprietary_nonprofit", + "address_line_1", + "address_line_2", + "city", + "state", + "zip_code", + "_snapshot_date", +) + + +def derive_crosswalk(con: Any, *, run_id: str = "") -> dict[str, int]: + """Build the four derived tables from whatever ``cms.*`` / ``bcda.*`` tables exist.""" + from cms.datacms import ENROLLMENT_TABLES + from cms.ingest_log import log_ingest + + stats: dict[str, int] = {} + + # ── institutional_provider: union of the PECOS enrollment files ── + parts = [] + for t in ENROLLMENT_TABLES: + if not _has(con, "cms", t): + continue + have = _cols(con, "cms", t) + sel = ", ".join( + (f'"{c}"' if c in have else f"NULL::VARCHAR AS {c}") + for c in _ENROLLMENT_COLS + ) + parts.append(f"SELECT {sel}, '{t}' AS source FROM cms.{t}") + if parts: + con.execute( + "CREATE OR REPLACE TABLE cms.institutional_provider AS " + + " UNION ALL ".join(parts) + ) + else: + con.execute( + "CREATE OR REPLACE TABLE cms.institutional_provider (" + + ", ".join(f"{c} VARCHAR" for c in _ENROLLMENT_COLS) + + ", source VARCHAR)" + ) + stats["cms.institutional_provider"] = _count(con, "cms.institutional_provider") + + # ── ccn_crosswalk: (ccn, npi) from PECOS ∪ POS-only CCNs ── + pos = "provider_services_file_quality_improvement_evaluation_system" + has_pos = _has(con, "cms", pos) + has_cost = _has(con, "cms", "hospital_provider_cost_report") + pos_join = ( + "LEFT JOIN (SELECT prvdr_num, any_value(fac_name) AS pos_fac_name, " + "any_value(prvdr_ctgry_cd) AS pos_category_cd, " + "any_value(prvdr_ctgry_sbtyp_cd) AS pos_subtype_cd, " + "any_value(pgm_trmntn_cd) AS pos_termination_cd, " + "any_value(trmntn_exprtn_dt) AS pos_termination_dt, " + "any_value(crtfctn_dt) AS pos_certification_dt, any_value(state_cd) AS pos_state, " + f"any_value(zip_cd) AS pos_zip FROM cms.{pos} GROUP BY 1) p ON p.prvdr_num = x.ccn" + if has_pos + else "LEFT JOIN (SELECT NULL::VARCHAR AS prvdr_num, NULL::VARCHAR AS pos_fac_name, " + "NULL::VARCHAR AS pos_category_cd, NULL::VARCHAR AS pos_subtype_cd, " + "NULL::VARCHAR AS pos_termination_cd, NULL::VARCHAR AS pos_termination_dt, " + "NULL::VARCHAR AS pos_certification_dt, NULL::VARCHAR AS pos_state, NULL::VARCHAR AS pos_zip " + "WHERE false) p ON p.prvdr_num = x.ccn" + ) + cost_join = ( + "LEFT JOIN (SELECT provider_ccn, any_value(type_of_control) AS cost_type_of_control, " + "any_value(ccn_facility_type) AS cost_ccn_facility_type, any_value(hospital_name) AS cost_hospital_name " + "FROM cms.hospital_provider_cost_report GROUP BY 1) k ON k.provider_ccn = x.ccn" + if has_cost + else "LEFT JOIN (SELECT NULL::VARCHAR AS provider_ccn, NULL::VARCHAR AS cost_type_of_control, " + "NULL::VARCHAR AS cost_ccn_facility_type, NULL::VARCHAR AS cost_hospital_name WHERE false) k " + "ON k.provider_ccn = x.ccn" + ) + pos_only = ( + f"UNION ALL SELECT prvdr_num AS ccn, NULL::VARCHAR AS npi, 'provider_of_services' AS sources, " + f"any_value(fac_name) AS organization_name, NULL::VARCHAR AS provider_type_text, " + f"any_value(state_cd) AS state FROM cms.{pos} " + "WHERE prvdr_num IS NOT NULL AND prvdr_num NOT IN (SELECT ccn FROM cms.institutional_provider WHERE ccn IS NOT NULL) " + "GROUP BY 1" + if has_pos + else "" + ) + con.execute( + "CREATE OR REPLACE TABLE cms.ccn_crosswalk AS " + "WITH x AS (" + "SELECT ccn, npi, string_agg(DISTINCT source, ',' ORDER BY source) AS sources, " + "any_value(organization_name) AS organization_name, any_value(provider_type_text) AS provider_type_text, " + "any_value(state) AS state FROM cms.institutional_provider WHERE ccn IS NOT NULL AND ccn <> '' GROUP BY 1, 2 " + f"{pos_only}) " + "SELECT x.*, p.pos_fac_name, p.pos_category_cd, p.pos_subtype_cd, p.pos_termination_cd, " + "p.pos_termination_dt, p.pos_certification_dt, p.pos_state, p.pos_zip, " + "k.cost_type_of_control, k.cost_ccn_facility_type, k.cost_hospital_name " + f"FROM x {pos_join} {cost_join}" + ) + stats["cms.ccn_crosswalk"] = _count(con, "cms.ccn_crosswalk") + + # ── provider_tin_observed: NPI → TIN from our own claims ── + if _has(con, "bcda", "explanation_of_benefit"): + have = _cols(con, "bcda", "explanation_of_benefit") + srcs = [] + if {"rndrg_prvdr_npi_num", "clm_rndrg_prvdr_tax_num"} <= have: + srcs.append( + "SELECT rndrg_prvdr_npi_num AS npi, clm_rndrg_prvdr_tax_num AS tin, 'rendering' AS role, " + "eob_type, last_updated FROM bcda.explanation_of_benefit " + "WHERE rndrg_prvdr_npi_num IS NOT NULL AND clm_rndrg_prvdr_tax_num IS NOT NULL" + ) + if {"carr_clm_blg_npi_num", "clm_blg_prvdr_tax_num"} <= have: + srcs.append( + "SELECT carr_clm_blg_npi_num, clm_blg_prvdr_tax_num, 'billing', eob_type, last_updated " + "FROM bcda.explanation_of_benefit " + "WHERE carr_clm_blg_npi_num IS NOT NULL AND clm_blg_prvdr_tax_num IS NOT NULL" + ) + if {"fac_prvdr_npi_num", "clm_blg_prvdr_tax_num"} <= have: + srcs.append( + "SELECT fac_prvdr_npi_num, clm_blg_prvdr_tax_num, 'facility', eob_type, last_updated " + "FROM bcda.explanation_of_benefit " + "WHERE fac_prvdr_npi_num IS NOT NULL AND clm_blg_prvdr_tax_num IS NOT NULL" + ) + body = " UNION ALL ".join(srcs) or ( + "SELECT NULL::VARCHAR AS npi, NULL::VARCHAR AS tin, NULL::VARCHAR AS role, " + "NULL::VARCHAR AS eob_type, NULL::VARCHAR AS last_updated WHERE false" + ) + con.execute( + "CREATE OR REPLACE TABLE cms.provider_tin_observed AS " + f"SELECT npi, tin, role, count(*) AS claims, count(DISTINCT eob_type) AS claim_types, " + f"min(last_updated) AS first_seen, max(last_updated) AS last_seen FROM ({body}) GROUP BY 1, 2, 3" + ) + else: + con.execute( + "CREATE OR REPLACE TABLE cms.provider_tin_observed (npi VARCHAR, tin VARCHAR, role VARCHAR, " + "claims BIGINT, claim_types BIGINT, first_seen VARCHAR, last_seen VARCHAR)" + ) + stats["cms.provider_tin_observed"] = _count(con, "cms.provider_tin_observed") + + # ── ccn_tin: crosswalk → observed / participant-list TIN ── + participant = ( + "LEFT JOIN (SELECT ccn, npi, any_value(tin) AS tin FROM cms.aco_participant_tin " + "GROUP BY 1, 2) a ON (a.ccn = c.ccn OR (a.ccn IS NULL AND a.npi = c.npi))" + if _has(con, "cms", "aco_participant_tin") + else "LEFT JOIN (SELECT NULL::VARCHAR AS ccn, NULL::VARCHAR AS npi, NULL::VARCHAR AS tin WHERE false) a " + "ON a.ccn = c.ccn" + ) + con.execute( + "CREATE OR REPLACE TABLE cms.ccn_tin AS " + "WITH o AS (SELECT npi, arg_max(tin, claims) AS tin, sum(claims) AS claims, count(DISTINCT tin) AS tins " + "FROM cms.provider_tin_observed GROUP BY 1) " + "SELECT c.ccn, c.npi, c.organization_name, c.provider_type_text, c.sources, " + "coalesce(a.tin, o.tin) AS tin, " + "CASE WHEN a.tin IS NOT NULL THEN 'participant_list' WHEN o.tin IS NOT NULL THEN 'observed_claims' END AS tin_source, " + "o.claims AS observed_claims, o.tins AS observed_distinct_tins, " + "c.pos_termination_cd, c.pos_termination_dt " + f"FROM cms.ccn_crosswalk c LEFT JOIN o ON o.npi = c.npi {participant}" + ) + stats["cms.ccn_tin"] = _count(con, "cms.ccn_tin") + for t in ( + "institutional_provider", + "ccn_crosswalk", + "provider_tin_observed", + "ccn_tin", + ): + log_ingest( + con, + module="cms", + table_name=f"cms.{t}", + rows=stats[f"cms.{t}"], + run_id=run_id, + ) + return stats + + +def _count(con: Any, table: str) -> int: + return con.execute(f"SELECT count(*) FROM {table}").fetchone()[0] + + +def coverage(con: Any) -> dict[str, Any]: + """How far our institutional claims resolve: facility NPI → CCN → TIN. + + Reads ``bcda.explanation_of_benefit`` (institutional families) against + ``cms.ccn_tin``. Returns counts of distinct facility NPIs on claims, + how many PECOS knows (CCN), how many have any TIN, and the claims those + cover. + """ + if not ( + _has(con, "bcda", "explanation_of_benefit") and _has(con, "cms", "ccn_tin") + ): + return {"available": False} + row = con.execute( + "WITH f AS (SELECT fac_prvdr_npi_num AS npi, count(*) AS claims FROM bcda.explanation_of_benefit " + "WHERE eob_type IN ('INPATIENT','OUTPATIENT','SNF','HHA','HOSPICE') AND fac_prvdr_npi_num IS NOT NULL GROUP BY 1), " + "x AS (SELECT npi, bool_or(ccn IS NOT NULL) AS has_ccn, bool_or(tin IS NOT NULL) AS has_tin FROM cms.ccn_tin " + "WHERE npi IS NOT NULL GROUP BY 1) " + "SELECT count(*) AS facility_npis, sum(claims) AS claims, " + "count(*) FILTER (WHERE x.has_ccn) AS npis_with_ccn, sum(claims) FILTER (WHERE x.has_ccn) AS claims_with_ccn, " + "count(*) FILTER (WHERE x.has_tin) AS npis_with_tin, sum(claims) FILTER (WHERE x.has_tin) AS claims_with_tin " + "FROM f LEFT JOIN x USING (npi)" + ).fetchone() + keys = ( + "facility_npis", + "claims", + "npis_with_ccn", + "claims_with_ccn", + "npis_with_tin", + "claims_with_tin", + ) + out: dict[str, Any] = { + "available": True, + **{k: int(v or 0) for k, v in zip(keys, row, strict=True)}, + } + out["institutional_claims_without_facility_npi"] = int( + con.execute( + "SELECT count(*) FROM bcda.explanation_of_benefit WHERE eob_type IN " + "('INPATIENT','OUTPATIENT','SNF','HHA','HOSPICE') AND fac_prvdr_npi_num IS NULL" + ).fetchone()[0] + ) + out["ccn_universe"] = _count(con, "cms.ccn_crosswalk") + out["observed_tins"] = _count(con, "cms.provider_tin_observed") + return out diff --git a/src/bcda/express/cclf.py b/src/bcda/express/cclf.py index 14e2edc..4109f1f 100644 --- a/src/bcda/express/cclf.py +++ b/src/bcda/express/cclf.py @@ -48,6 +48,15 @@ sandbox at line level; the pipeline coalesces line → header. (``fac_prvdr_npi_num`` / ``carr_clm_blg_npi_num``, tax number in ``clm_blg_prvdr_tax_num``); institutional rendering is attending → operating → rendering; DME rendering is rendering → ordering. +**There is no TIN on institutional claims** — not in BCDA v3 (the real +Large-entity pull: 0 billing TINs on every claim type, 0 CCNs on every +institutional claim, half of them with no billing provider at all) and +not in CCLF1 either; a facility is its NPI and its CCN. The only TIN on +claims is the Part B carrier/DME ``CLM_RNDRG_PRVDR_TAX_NUM``, which is +the TIN the line was billed under, so professional and DME lines get it +as ``billing_tin`` as well. CMS publishes no provider TINs; the public +universe is CCN ↔ NPI ↔ organization (``stack load cms`` → +``cms.ccn_crosswalk`` / ``cms.ccn_tin``). **Dates.** Admission / discharge come from the C4BB ``admissionperiod`` (``clm_actv_care_from_dt`` / ``clm_dschrg_dt``) with the billable period @@ -452,6 +461,18 @@ def _medical_claim(header: FrameT, items: FrameT, kind: str) -> FrameT: facility = null_str rendering_tin = nw.col("clm_rndrg_prvdr_tax_num") + # TIN. Institutional claims never carry one (CCLF1 has none; BCDA v3 + # emits none) — the facility is its NPI and CCN, see cms.ccn_tin for the + # crosswalk. On carrier/DME lines CLM_RNDRG_PRVDR_TAX_NUM is the TIN the + # line was billed under (CCLF5's only TIN), so it is the billing TIN too. + billing_tin = ( + nw.col("clm_blg_prvdr_tax_num") + if inst + else nw.coalesce( + nw.col("clm_blg_prvdr_tax_num"), nw.col("clm_rndrg_prvdr_tax_num") + ) + ) + if inst: charge = nw.coalesce( nw.col("rev_cntr_tot_chrg_amt"), @@ -561,7 +582,7 @@ def _medical_claim(header: FrameT, items: FrameT, kind: str) -> FrameT: rendering.alias("rendering_npi"), rendering_tin.alias("rendering_tin"), billing.alias("billing_npi"), - nw.col("clm_blg_prvdr_tax_num").alias("billing_tin"), + billing_tin.alias("billing_tin"), facility.alias("facility_npi"), _paid_date(), nw.col("clm_line_cvrd_pd_amt").cast(nw.Float64).alias("paid_amount"), diff --git a/src/bcda/ui/service.py b/src/bcda/ui/service.py index 4f58f20..5ad7bf0 100644 --- a/src/bcda/ui/service.py +++ b/src/bcda/ui/service.py @@ -674,6 +674,14 @@ SUMMARY_QUERIES: dict[str, str] = { "select count(*) patients, count(bene_mbi_id) with_mbi, count(bene_death_dt) deceased, " "count(distinct geo_usps_state_cd) states from bcda.patient" ), + "Facility NPI → CCN / TIN coverage (needs `stack load cms`)": ( + "with f as (select fac_prvdr_npi_num npi, count(*) claims from bcda.explanation_of_benefit " + "where eob_type in ('INPATIENT','OUTPATIENT','SNF','HHA','HOSPICE') and fac_prvdr_npi_num is not null group by 1) " + "select count(*) facility_npis, sum(claims) claims, " + "count(*) filter (where x.ccn is not null) npis_with_ccn, " + "count(*) filter (where x.tin is not null) npis_with_tin " + "from f left join (select npi, any_value(ccn) ccn, any_value(tin) tin from cms.ccn_tin where npi is not null group by 1) x using (npi)" + ), } diff --git a/src/cli/load.py b/src/cli/load.py index eba461c..fc81b18 100644 --- a/src/cli/load.py +++ b/src/cli/load.py @@ -84,6 +84,54 @@ def bcda( typer.echo("Done.") +@app.command() +def cms( + table: list[str] = typer.Option( + None, "--table", "-t", help="cms.
(default: the institutional universe)." + ), + database: str = typer.Option(None, help="DuckDB database path (default: config)."), + refresh: bool = typer.Option( + False, "--refresh", help="Re-download even if the CSV is on disk." + ), + derive: bool = typer.Option( + True, + "--derive/--no-derive", + help="Rebuild institutional_provider / ccn_crosswalk / provider_tin_observed / ccn_tin.", + ), +) -> None: + """Vendor the institutional provider universe from data.cms.gov: CCN ↔ NPI ↔ TIN. + + PECOS enrollments (hospital, SNF, HHA, hospice, FQHC, RHC), the + Provider of Services file and the cost reports, loaded as ``cms.*`` + with their models' column names, then the crosswalk tables. CMS + publishes no TIN; ``cms.ccn_tin`` carries one only where our own + claims or ``cms.aco_participant_tin`` know it. + """ + import duckdb + + from aco.load.cms import coverage, load_cms + from conf import path + + typer.echo("Loading data.cms.gov institutional datasets ...") + stats = load_cms(table or None, database=database, refresh=refresh, derive=derive) + for t, n in stats.items(): + typer.echo(f" {t}: {n:,} rows") + con = duckdb.connect(database or str(path("db.aco")), read_only=True) + try: + cov = coverage(con) + finally: + con.close() + if cov.get("available"): + typer.echo( + f"coverage: {cov['facility_npis']:,} facility NPIs on institutional claims — " + f"{cov['npis_with_ccn']:,} resolve to a CCN ({cov['claims_with_ccn']:,} of {cov['claims']:,} claims), " + f"{cov['npis_with_tin']:,} to a TIN ({cov['claims_with_tin']:,} claims); " + f"{cov['institutional_claims_without_facility_npi']:,} institutional claims carry no facility NPI at all; " + f"CCN universe {cov['ccn_universe']:,}; observed NPI→TIN pairs {cov['observed_tins']:,}" + ) + typer.echo("Done.") + + @app.command() def seed( seed_dir: Path = typer.Option(None, help="Directory containing seed files."), diff --git a/src/cms/datacms.py b/src/cms/datacms.py new file mode 100644 index 0000000..c11eae9 --- /dev/null +++ b/src/cms/datacms.py @@ -0,0 +1,287 @@ +"""data.cms.gov: catalog, dataset resolution, download — and the institutional universe. + +The ``cms.table`` models were generated from the data.cms.gov catalog +(``dev/scripts/scrape_cms_data.py``); this module is the runtime side: +find the newest version of a dataset family, fetch its CSV (or page its +API when a family has no CSV distribution), and name columns the way the +generated models do. + +Why this exists (2026-10-25, "no TIN available"): BCDA v3 institutional +claims identify the facility by **NPI only** — the real Large-entity pull +has 0 billing TINs on every claim type and 0 CCNs on every institutional +claim (the dictionary's ``NamingSystem/CCN`` identifier is simply not +emitted; half the institutional claims carry no billing provider at +all). CCLF has the same shape: CCLF1 institutional headers carry an +OSCAR/CCN and NPIs, never a TIN; the only TIN on claims is the Part B +carrier/DME ``CLM_RNDRG_PRVDR_TAX_NUM``. CMS publishes **no TIN or EIN +for any provider**: the PECOS enrollment files, All Owners, Change of +Ownership, Provider of Services and cost-report datasets all stop at +CCN · NPI · PAC (associate) id · enrollment id · legal name. So the +public universe we can vendor is CCN ↔ NPI ↔ organization; TINs come +from the claims themselves (professional lines) or from a model +entity's own participant list (``cms.aco_participant_tin``, if you load +one). :mod:`aco.load.cms` builds that crosswalk. +""" + +from __future__ import annotations + +import csv +import json +import logging +import re +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import httpx + +log = logging.getLogger(__name__) + +CATALOG_URL = "https://data.cms.gov/data.json" +API_BASE = "https://data.cms.gov/data-api/v1/dataset" +UA = {"User-Agent": "stack-cms/1.0 (+https://git.fhirworx.io/homelab/stack)"} +PAGE_SIZE = 5000 + +#: cms.
→ catalog family title. The institutional provider universe: +#: every Medicare-certified facility type that bills Part A, plus the two +#: datasets that know CCNs PECOS does not (terminated facilities in the +#: Provider of Services file; cost-report filers). +INSTITUTIONAL: dict[str, str] = { + "hospital_enrollments": "Hospital Enrollments", + "skilled_nursing_facility_enrollments": "Skilled Nursing Facility Enrollments", + "home_health_agency_enrollments": "Home Health Agency Enrollments", + "hospice_enrollments": "Hospice Enrollments", + "federally_qualified_health_center_enrollments": "Federally Qualified Health Center Enrollments", + "rural_health_clinic_enrollments": "Rural Health Clinic Enrollments", + "provider_services_file_quality_improvement_evaluation_system": ( + "Provider of Services File - Quality Improvement and Evaluation System" + ), + "hospital_provider_cost_report": "Hospital Provider Cost Report", + "skilled_nursing_facility_cost_report": "Skilled Nursing Facility Cost Report", + "home_health_agency_cost_report": "Home Health Agency Cost Report", +} +ENROLLMENT_TABLES = tuple(t for t in INSTITUTIONAL if t.endswith("_enrollments")) + +_VERSION_SUFFIX = re.compile(r"\s*:\s*\d{4}-\d{2}.*$") +_BUILTINS = { + "date", + "type", + "id", + "list", + "set", + "dict", + "str", + "int", + "float", + "bool", + "bytes", + "hash", + "format", +} + + +def sanitize_column_name(name: str) -> str: + """The generated models' column naming (``dev/scripts/scrape_cms_data.py``). + + >>> sanitize_column_name("ENROLLMENT ID") + 'enrollment_id' + >>> sanitize_column_name("SUBGROUP - ACUTE CARE") + 'subgroup_acute_care' + >>> sanitize_column_name("PROPRIETARY_NONPROFIT") + 'proprietary_nonprofit' + """ + name = name.strip() + name = re.sub(r"[-\s]+", "_", name) + name = re.sub(r"[^a-zA-Z0-9_]", "", name) + name = name.lower() + if name and name[0].isdigit(): + name = "_" + name + if name in _BUILTINS: + name = name + "_" + return name + + +def family_title(title: str) -> str: + """``"Hospital Enrollments : 2026-10-01"`` → ``"Hospital Enrollments"``.""" + return _VERSION_SUFFIX.sub("", title or "").strip() + + +def snapshot_date_from_name(name: str) -> str: + """Best-effort snapshot date from a CMS file name (``2026.09.29``, ``Q3_2026``, ``2021_Final``).""" + m = re.search(r"(\d{4})[._-](\d{2})[._-](\d{2})", name) + if m: + return f"{m.group(1)}-{m.group(2)}-{m.group(3)}" + m = re.search(r"Q([1-4])[._-](\d{4})", name) + if m: + return f"{m.group(2)}-Q{m.group(1)}" + m = re.search(r"(? str: + if self.csv_url: + return self.csv_url.rsplit("/", 1)[-1] + return f"{self.table}.api.csv" + + @property + def snapshot_date(self) -> str: + """The file's own date (``…_2026.09.29.csv``) over the catalog version's.""" + return snapshot_date_from_name(self.filename) or snapshot_date_from_name( + self.version_title + ) + + +# ── catalog ─────────────────────────────────────────────────── + + +def _get(url: str, **kw: Any) -> httpx.Response: + for attempt in range(4): + try: + r = httpx.get(url, headers=UA, timeout=120, follow_redirects=True, **kw) + if r.status_code in (429, 500, 502, 503, 504) and attempt < 3: + time.sleep(2**attempt) + continue + r.raise_for_status() + return r + except httpx.TransportError: + if attempt == 3: + raise + time.sleep(2**attempt) + raise RuntimeError("unreachable") # pragma: no cover + + +def catalog( + cache: Path | None = None, *, max_age_hours: float = 24, refresh: bool = False +) -> list[dict]: + """The data.cms.gov DCAT catalog's ``dataset`` list, cached on disk.""" + if cache is not None and cache.exists() and not refresh: + age = time.time() - cache.stat().st_mtime + if age < max_age_hours * 3600: + return json.loads(cache.read_text())["dataset"] + data = _get(CATALOG_URL).json() + if cache is not None: + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text(json.dumps(data)) + return data["dataset"] + + +def _family_uuid(entry: dict) -> str: + ident = str(entry.get("identifier", "")) + m = re.search(r"/dataset/([0-9a-f-]{36})", ident) + return m.group(1) if m else "" + + +def resolve(table: str, datasets: list[dict], *, title: str | None = None) -> Dataset: + """Newest version of the family behind ``cms.
``. + + *title* overrides the :data:`INSTITUTIONAL` mapping. The CSV is the + newest version's ``text/csv`` distribution; the API URL is the family + endpoint (``/dataset//data``), which always serves the + newest version. + """ + want = title or INSTITUTIONAL.get(table) + if not want: + raise KeyError(f"no data.cms.gov title known for cms.{table}") + versions = [d for d in datasets if family_title(d.get("title", "")) == want] + if not versions: + raise LookupError(f"{want!r} not in the data.cms.gov catalog") + versions.sort( + key=lambda d: (d.get("modified", ""), d.get("title", "")), reverse=True + ) + newest = versions[0] + csv_url = "" + api_url = "" + fam = _family_uuid(newest) + for dist in newest.get("distribution", []) or []: + mt = str(dist.get("mediaType") or dist.get("format") or "").lower() + url = str(dist.get("downloadURL") or dist.get("accessURL") or "") + if not csv_url and "csv" in mt and url: + csv_url = url + if url.startswith(API_BASE) and url.endswith("/data"): + if fam and f"/{fam}/" in url: + api_url = url + elif not api_url: + api_url = url + if not api_url and fam: + api_url = f"{API_BASE}/{fam}/data" + return Dataset( + table=table, + title=want, + version_title=str(newest.get("title", want)), + modified=str(newest.get("modified", ""))[:10], + family_uuid=fam, + csv_url=csv_url, + api_url=api_url, + ) + + +# ── download ────────────────────────────────────────────────── + + +def download(ds: Dataset, dest_dir: Path, *, refresh: bool = False) -> Path: + """Fetch the dataset's CSV into *dest_dir* (skipped when already there). + + Families without a CSV distribution are paged through the data API + (``size``/``offset``) and written as CSV, so the loader sees one shape. + """ + dest_dir.mkdir(parents=True, exist_ok=True) + dest = dest_dir / ds.filename + if dest.exists() and dest.stat().st_size > 0 and not refresh: + return dest + tmp = dest.with_suffix(dest.suffix + ".part") + t0 = time.monotonic() + if ds.csv_url: + with httpx.stream( + "GET", ds.csv_url, headers=UA, timeout=600, follow_redirects=True + ) as r: + r.raise_for_status() + with tmp.open("wb") as fh: + for chunk in r.iter_bytes(1 << 20): + fh.write(chunk) + else: + _page_api_to_csv(ds.api_url, tmp) + tmp.replace(dest) + log.info( + "downloaded %s (%d bytes) in %.1fs", + dest.name, + dest.stat().st_size, + time.monotonic() - t0, + ) + return dest + + +def _page_api_to_csv(api_url: str, dest: Path) -> None: + offset = 0 + header: list[str] | None = None + with dest.open("w", newline="") as fh: + writer = None + while True: + rows = _get(api_url, params={"size": PAGE_SIZE, "offset": offset}).json() + if not rows: + break + if header is None: + header = list(rows[0].keys()) + writer = csv.DictWriter(fh, fieldnames=header, extrasaction="ignore") + writer.writeheader() + assert writer is not None + writer.writerows(rows) + offset += len(rows) + if len(rows) < PAGE_SIZE: + break + + +def now_iso() -> str: + return datetime.now(tz=timezone.utc).isoformat(timespec="seconds") diff --git a/tests/aco/test_load_cms.py b/tests/aco/test_load_cms.py new file mode 100644 index 0000000..4ae8eeb --- /dev/null +++ b/tests/aco/test_load_cms.py @@ -0,0 +1,398 @@ +"""aco.load.cms — load data.cms.gov CSVs and derive the CCN ↔ NPI ↔ TIN crosswalk.""" + +from __future__ import annotations + +import csv +from pathlib import Path + +import duckdb +import pytest + +from aco.load.cms import coverage, derive_crosswalk, load_cms +from cms import datacms +from tests.cms.test_datacms import _catalog + +HOSP = [ + ( + "ENROLLMENT ID", + "ENROLLMENT STATE", + "PROVIDER TYPE CODE", + "PROVIDER TYPE TEXT", + "NPI", + "MULTIPLE NPI FLAG", + "CCN", + "ASSOCIATE ID", + "ORGANIZATION NAME", + "DOING BUSINESS AS NAME", + "INCORPORATION DATE", + "INCORPORATION STATE", + "ORGANIZATION TYPE STRUCTURE", + "ORGANIZATION OTHER TYPE TEXT", + "PROPRIETARY NONPROFIT", + "ADDRESS LINE 1", + "ADDRESS LINE 2", + "CITY", + "STATE", + "ZIP CODE", + "SUBGROUP - GENERAL", + "CAH OR HOSPITAL CCN", + ), + ( + "O1", + "TN", + "00-09", + "PART A PROVIDER - HOSPITAL", + "1467408781", + "N", + "440058", + "5193632180", + "SOUTHERN TENNESSEE MEDICAL CENTER LLC", + "HIGHPOINT", + "11/09/1998", + "DE", + "LLC", + "", + "P", + "185 HOSPITAL RD", + "", + "WINCHESTER", + "TN", + "373982404", + "Y", + "440058", + ), + ( + "O2", + "TX", + "00-09", + "PART A PROVIDER - HOSPITAL", + "1000000002", + "Y", + "450001", + "5000000002", + "TEXAS GENERAL", + "", + "", + "TX", + "CORP", + "", + "N", + "1 MAIN", + "", + "AUSTIN", + "TX", + "78701", + "Y", + "450001", + ), +] +SNF = [ + ( + "ENROLLMENT ID", + "ENROLLMENT STATE", + "PROVIDER TYPE CODE", + "PROVIDER TYPE TEXT", + "NPI", + "MULTIPLE NPI FLAG", + "CCN", + "ASSOCIATE ID", + "ORGANIZATION NAME", + "DOING BUSINESS AS NAME", + "INCORPORATION DATE", + "INCORPORATION STATE", + "ORGANIZATION TYPE STRUCTURE", + "ORGANIZATION OTHER TYPE TEXT", + "PROPRIETARY_NONPROFIT", + "NURSING HOME PROVIDER NAME", + "AFFILIATION ENTITY NAME", + "AFFILIATION ENTITY ID", + "ADDRESS LINE 1", + "ADDRESS LINE 2", + "CITY", + "STATE", + "ZIP CODE", + ), + ( + "S1", + "TN", + "00-10", + "PART A PROVIDER - SNF", + "1000000003", + "N", + "445001", + "5000000003", + "WINCHESTER SNF", + "", + "", + "TN", + "LLC", + "", + "P", + "WINCHESTER SNF", + "", + "", + "2 ELM", + "", + "WINCHESTER", + "TN", + "37398", + ), +] +POS = [ + ( + "PRVDR_CTGRY_SBTYP_CD", + "PRVDR_CTGRY_CD", + "FAC_NAME", + "PRVDR_NUM", + "STATE_CD", + "PGM_TRMNTN_CD", + "TRMNTN_EXPRTN_DT", + "CRTFCTN_DT", + "ZIP_CD", + ), + ( + "01", + "01", + "SOUTHERN TENNESSEE MEDICAL CENTER", + "440058", + "TN", + "00", + "", + "19900101", + "37398", + ), + ( + "01", + "01", + "CLOSED HOSPITAL", + "440999", + "TN", + "01", + "20200101", + "19800101", + "37000", + ), +] +COST = [ + ( + "rpt_rec_num", + "Provider CCN", + "Hospital Name", + "CCN Facility Type", + "Type of Control", + ), + ("1", "440058", "SOUTHERN TENNESSEE MEDICAL CENTER", "STACH", "Proprietary"), +] + + +def _write(dir_: Path, name: str, rows: list[tuple]) -> Path: + p = dir_ / name + with p.open("w", newline="") as fh: + csv.writer(fh).writerows(rows) + return p + + +@pytest.fixture +def fixtures(tmp_path: Path) -> dict[str, Path]: + d = tmp_path / "raw" + d.mkdir() + files = { + "hospital_enrollments": _write(d, "Hospital_Enrollments_2026.09.29.csv", HOSP), + "skilled_nursing_facility_enrollments": _write( + d, "SNF_Enrollments_2026.09.29.csv", SNF + ), + "provider_services_file_quality_improvement_evaluation_system": _write( + d, "Hospital_and_other.DATA.Q3_2026.csv", POS + ), + "hospital_provider_cost_report": _write(d, "CostReport_2021_Final.csv", COST), + } + return files + + +def _catalog_all() -> list[dict]: + cat = _catalog() + for table, title in datacms.INSTITUTIONAL.items(): + if title not in {datacms.family_title(d["title"]) for d in cat}: + cat.append( + { + "title": f"{title} : 2026-10-01", + "modified": "2026-10-19", + "identifier": "https://data.cms.gov/data-api/v1/dataset/00000000-0000-0000-0000-000000000000/data-viewer", + "distribution": [ + { + "mediaType": "text/csv", + "downloadURL": f"https://x/{table}.csv", + } + ], + } + ) + return cat + + +def test_load_and_derive(tmp_path: Path, fixtures: dict[str, Path]) -> None: + db = tmp_path / "aco.duckdb" + con = duckdb.connect(str(db)) + con.execute("create schema bcda") + con.execute( + "create table bcda.explanation_of_benefit as select * from (values " + "('INPATIENT', '1467408781', NULL, NULL, NULL, '2026-05-01'), " + "('INPATIENT', '1000000002', NULL, NULL, NULL, '2026-05-01'), " + "('INPATIENT', NULL, NULL, NULL, NULL, '2026-05-01'), " + "('CARRIER', NULL, '1467408781', '621234567', NULL, '2026-06-01'), " + "('CARRIER', NULL, '1467408781', '621234567', NULL, '2026-07-01'), " + "('CARRIER', NULL, '1999999999', '000000001', NULL, '2026-07-01'), " + "('OUTPATIENT', '1467408781', NULL, NULL, '777777777', '2026-07-01')" + ") t(eob_type, fac_prvdr_npi_num, rndrg_prvdr_npi_num, clm_rndrg_prvdr_tax_num, clm_blg_prvdr_tax_num, last_updated)" + ) + con.close() + + def fetch(ds, dest_dir, refresh=False): # noqa: ANN001, ANN202 + return fixtures[ds.table] + + stats = load_cms( + list(fixtures), + database=str(db), + raw_dir=tmp_path / "raw", + datasets=_catalog_all(), + fetch=fetch, + ) + assert ( + stats["cms.hospital_enrollments"] == 2 + and stats["cms.skilled_nursing_facility_enrollments"] == 1 + ) + assert stats["cms.institutional_provider"] == 3 + assert ( + stats["cms.ccn_crosswalk"] == 4 + ) # 3 PECOS rows + the POS-only closed hospital + assert ( + stats["cms.provider_tin_observed"] == 3 + ) # two rendering pairs + one facility pair + assert stats["cms.ccn_tin"] == 4 + + con = duckdb.connect(str(db), read_only=True) + cols = {r[0] for r in con.execute("describe cms.hospital_enrollments").fetchall()} + assert { + "enrollment_id", + "ccn", + "npi", + "subgroup_general", + "cah_or_hospital_ccn", + "_snapshot_date", + "_source_url", + "_loaded_at", + } <= cols + assert con.execute( + "select distinct _snapshot_date from cms.hospital_enrollments" + ).fetchall() == [("2026-09-29",)] + assert con.execute( + "select distinct proprietary_nonprofit from cms.institutional_provider order by 1" + ).fetchall() == [("N",), ("P",)] + xw = { + r[0]: r + for r in con.execute( + "select ccn, npi, sources, pos_fac_name, pos_termination_cd, cost_type_of_control from cms.ccn_crosswalk" + ).fetchall() + } + assert xw["440058"][1:] == ( + "1467408781", + "hospital_enrollments", + "SOUTHERN TENNESSEE MEDICAL CENTER", + "00", + "Proprietary", + ) + assert ( + xw["440999"][1] is None + and xw["440999"][2] == "provider_of_services" + and xw["440999"][4] == "01" + ) + assert xw["445001"][2] == "skilled_nursing_facility_enrollments" + tin = { + r[0]: r[1:] + for r in con.execute( + "select ccn, tin, tin_source, observed_claims from cms.ccn_tin" + ).fetchall() + } + # the hospital's NPI rendered two carrier lines under one TIN and billed one outpatient claim under another: + # arg_max by claims picks the two-claim TIN; both are visible in provider_tin_observed + assert tin["440058"] == ("621234567", "observed_claims", 3) + assert tin["450001"] == (None, None, None) and tin["440999"] == (None, None, None) + assert ( + con.execute( + "select count(*) from cms.ingest_log where module = 'cms'" + ).fetchone()[0] + == len(fixtures) + 4 + ) + cov = coverage(con) + assert cov["available"] and cov["facility_npis"] == 2 and cov["claims"] == 3 + assert ( + cov["npis_with_ccn"] == 2 + and cov["npis_with_tin"] == 1 + and cov["claims_with_tin"] == 2 + ) + assert ( + cov["institutional_claims_without_facility_npi"] == 1 + and cov["ccn_universe"] == 4 + ) + con.close() + + # a participant list wins over observed TINs + con = duckdb.connect(str(db)) + con.execute( + "create table cms.aco_participant_tin as select '450001' ccn, NULL::varchar npi, '741111111' tin, 'participant list PY2026' as \"source\"" + ) + derive_crosswalk(con) + assert con.execute( + "select tin, tin_source from cms.ccn_tin where ccn = '450001'" + ).fetchone() == ("741111111", "participant_list") + con.close() + + +def test_derive_without_anything(tmp_path: Path) -> None: + con = duckdb.connect(str(tmp_path / "x.duckdb")) + con.execute("create schema cms") + stats = derive_crosswalk(con) + assert stats == { + "cms.institutional_provider": 0, + "cms.ccn_crosswalk": 0, + "cms.provider_tin_observed": 0, + "cms.ccn_tin": 0, + } + assert coverage(con) == {"available": False} + con.close() + + +def test_unknown_table_and_cli(tmp_path: Path, monkeypatch) -> None: + with pytest.raises(KeyError): + load_cms(["nope"], database=str(tmp_path / "x.duckdb"), datasets=[]) + from typer.testing import CliRunner + + from cli import app + + monkeypatch.setattr( + "aco.load.cms.load_cms", + lambda *a, **k: {"cms.hospital_enrollments": 2, "cms.ccn_tin": 2}, + ) + monkeypatch.setattr( + "aco.load.cms.coverage", + lambda con: { + "available": True, + "facility_npis": 2, + "npis_with_ccn": 2, + "claims_with_ccn": 3, + "claims": 3, + "npis_with_tin": 1, + "claims_with_tin": 2, + "institutional_claims_without_facility_npi": 1, + "ccn_universe": 4, + "observed_tins": 3, + }, + ) + db = tmp_path / "aco.duckdb" + duckdb.connect(str(db)).close() + result = CliRunner().invoke( + app, ["load", "cms", "--database", str(db), "-t", "hospital_enrollments"] + ) + assert result.exit_code == 0, result.output + assert ( + "cms.ccn_tin: 2 rows" in result.output and "2 resolve to a CCN" in result.output + ) diff --git a/tests/bcda/test_cclf_v3.py b/tests/bcda/test_cclf_v3.py index 0c736f0..2bcdc40 100644 --- a/tests/bcda/test_cclf_v3.py +++ b/tests/bcda/test_cclf_v3.py @@ -624,6 +624,8 @@ class TestMedicalClaimV3: "TIN-RND", "BLG", ) + # the carrier TIN is the TIN the line was billed under (CCLF5's only TIN) + assert r["billing_tin"] == "TIN-RND" assert ( r["facility_npi"] is None and r["bill_type_code"] is None diff --git a/tests/bcda/test_ui_service.py b/tests/bcda/test_ui_service.py index 99cf2ec..67e3f7a 100644 --- a/tests/bcda/test_ui_service.py +++ b/tests/bcda/test_ui_service.py @@ -427,7 +427,7 @@ class TestLocalSide: "create table bcda.patient as select 'A' bene_mbi_id, null::date bene_death_dt, 'TX' geo_usps_state_cd" ) con.execute( - "create table bcda.explanation_of_benefit as select 'DDPS' system_type, 'PDE' eob_type, 'FinalAction' final_action, 1.5 clm_pmt_amt" + "create table bcda.explanation_of_benefit as select 'DDPS' system_type, 'PDE' eob_type, 'FinalAction' final_action, 1.5 clm_pmt_amt, NULL::varchar fac_prvdr_npi_num" ) con.execute( "create table bcda.coverage as select 'Part A' class_plan, 'active' status, 'A' subscriber_id" @@ -435,6 +435,10 @@ class TestLocalSide: con.execute( "create table input_layer.medical_claim as select 'institutional' claim_type, 'c' claim_id, 1.0 paid_amount, 2.0 charge_amount" ) + con.execute("create schema cms") + con.execute( + "create table cms.ccn_tin as select NULL::varchar npi, NULL::varchar ccn, NULL::varchar tin where false" + ) con.close() ro = duckdb.connect(str(db), read_only=True) counts = svc.table_counts(ro) diff --git a/tests/cms/test_datacms.py b/tests/cms/test_datacms.py new file mode 100644 index 0000000..8eea922 --- /dev/null +++ b/tests/cms/test_datacms.py @@ -0,0 +1,242 @@ +"""cms.datacms — catalog resolution, column naming, download paths.""" + +from __future__ import annotations + +from pathlib import Path + +import httpx +import pytest + +from cms import datacms + +HOSPITAL_HEADER = [ + "ENROLLMENT ID", + "ENROLLMENT STATE", + "PROVIDER TYPE CODE", + "PROVIDER TYPE TEXT", + "NPI", + "MULTIPLE NPI FLAG", + "CCN", + "ASSOCIATE ID", + "ORGANIZATION NAME", + "DOING BUSINESS AS NAME", + "INCORPORATION DATE", + "INCORPORATION STATE", + "ORGANIZATION TYPE STRUCTURE", + "ORGANIZATION OTHER TYPE TEXT", + "PROPRIETARY NONPROFIT", + "ADDRESS LINE 1", + "ADDRESS LINE 2", + "CITY", + "STATE", + "ZIP CODE", + "PRACTICE LOCATION TYPE", + "LOCATION OTHER TYPE TEXT", + "SUBGROUP - GENERAL", + "SUBGROUP - ACUTE CARE", + "SUBGROUP - ALCOHOL DRUG", + "SUBGROUP - CHILDREN", +] + + +def test_column_names_match_generated_models() -> None: + from cms.table.hospital_enrollments import HospitalEnrollments + from cms.table.hospital_provider_cost_report import HospitalProviderCostReport + from cms.table.skilled_nursing_facility_enrollments import ( + SkilledNursingFacilityEnrollments, + ) + + fields = set(HospitalEnrollments.model_fields) + # the generated model predates a few SUBGROUP columns CMS added; the + # loader takes its columns from the CSV, so only the stable core must map + for h in HOSPITAL_HEADER[:22]: + assert datacms.sanitize_column_name(h) in fields, h + assert datacms.sanitize_column_name("PROPRIETARY_NONPROFIT") in set( + SkilledNursingFacilityEnrollments.model_fields + ) + cost = set(HospitalProviderCostReport.model_fields) + for h in ( + "rpt_rec_num", + "Provider CCN", + "Hospital Name", + "CCN Facility Type", + "Type of Control", + ): + assert datacms.sanitize_column_name(h) in cost, h + assert datacms.sanitize_column_name("3-Day Rule") == "_3_day_rule" + assert datacms.sanitize_column_name("Type") == "type_" + + +def test_family_title_and_snapshot_date() -> None: + assert ( + datacms.family_title("Hospital Enrollments : 2026-10-01") + == "Hospital Enrollments" + ) + assert datacms.family_title("Hospital Enrollments") == "Hospital Enrollments" + assert ( + datacms.snapshot_date_from_name("Hospital_Enrollments_2026.09.29.csv") + == "2026-09-29" + ) + assert ( + datacms.snapshot_date_from_name("Hospital_and_other.DATA.Q3_2026.csv") + == "2026-Q3" + ) + assert datacms.snapshot_date_from_name("CostReport_2021_Final.csv") == "2021" + assert datacms.snapshot_date_from_name("nothing") == "" + + +def _catalog() -> list[dict]: + fam = "f6f6505c-e8b0-4d57-b258-e2b94133aaf2" + return [ + { + "title": "Hospital Enrollments : 2026-09-01", + "modified": "2026-09-03", + "identifier": f"https://data.cms.gov/data-api/v1/dataset/{fam}/data-viewer", + "distribution": [ + { + "mediaType": "text/csv", + "downloadURL": "https://x/old/Hospital_Enrollments_2026.08.29.csv", + }, + {"accessURL": f"https://data.cms.gov/data-api/v1/dataset/{fam}/data"}, + ], + }, + { + "title": "Hospital Enrollments : 2026-10-01", + "modified": "2026-10-19", + "identifier": f"https://data.cms.gov/data-api/v1/dataset/{fam}/data-viewer", + "distribution": [ + {"accessURL": f"https://data.cms.gov/data-api/v1/dataset/{fam}/data"}, + { + "mediaType": "text/csv", + "downloadURL": "https://x/new/Hospital_Enrollments_2026.09.29.csv", + }, + { + "accessURL": "https://data.cms.gov/data-api/v1/dataset/71b9466f-aaaa-bbbb-cccc-000000000000/data" + }, + ], + }, + { + "title": "Hospital Provider Cost Report : 2021-12-05", + "modified": "2026-10-05", + "identifier": "https://data.cms.gov/data-api/v1/dataset/98b7383e-ad67-44d5-a7d8-0a80a92a72c1/data-viewer", + "distribution": [ + { + "accessURL": "https://data.cms.gov/data-api/v1/dataset/98b7383e-ad67-44d5-a7d8-0a80a92a72c1/data" + } + ], + }, + ] + + +def test_resolve_picks_newest_version_and_family_api() -> None: + ds = datacms.resolve("hospital_enrollments", _catalog()) + assert ds.version_title.endswith("2026-10-01") and ds.modified == "2026-10-19" + assert ds.csv_url.endswith("Hospital_Enrollments_2026.09.29.csv") + assert ds.api_url.endswith("f6f6505c-e8b0-4d57-b258-e2b94133aaf2/data") + assert ( + ds.filename == "Hospital_Enrollments_2026.09.29.csv" + and ds.snapshot_date == "2026-09-29" + ) + cost = datacms.resolve("hospital_provider_cost_report", _catalog()) + assert ( + cost.csv_url == "" and cost.filename == "hospital_provider_cost_report.api.csv" + ) + with pytest.raises(LookupError): + datacms.resolve("hospice_enrollments", _catalog()) + with pytest.raises(KeyError): + datacms.resolve("nope", _catalog()) + assert ( + datacms.resolve("nope", _catalog(), title="Hospital Enrollments").table + == "nope" + ) + + +def test_catalog_cache(tmp_path: Path, monkeypatch) -> None: + calls = [] + + class R: + def json(self): # noqa: ANN202 + calls.append(1) + return {"dataset": _catalog()} + + monkeypatch.setattr(datacms, "_get", lambda url, **kw: R()) + cache = tmp_path / "catalog.json" + assert len(datacms.catalog(cache)) == 3 and cache.exists() and calls == [1] + assert len(datacms.catalog(cache)) == 3 and calls == [1] # served from disk + assert len(datacms.catalog(cache, refresh=True)) == 3 and calls == [1, 1] + assert len(datacms.catalog(None)) == 3 + + +def test_download_csv_and_api_paging(tmp_path: Path, monkeypatch) -> None: + ds = datacms.resolve("hospital_enrollments", _catalog()) + + class Stream: + def __init__(self, *a, **k): # noqa: ANN002, ANN003 + pass + + def __enter__(self): # noqa: ANN204 + return self + + def __exit__(self, *a): # noqa: ANN002, ANN204 + return False + + def raise_for_status(self) -> None: + pass + + def iter_bytes(self, n): # noqa: ANN001, ANN202 + yield b'"CCN","NPI"\n' + yield b"440058,1467408781\n" + + monkeypatch.setattr(datacms.httpx, "stream", lambda *a, **k: Stream()) + p = datacms.download(ds, tmp_path) + assert p.name == ds.filename and p.read_text().count("\n") == 2 + # cached: the stream is not opened again + monkeypatch.setattr( + datacms.httpx, + "stream", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("re-download")), + ) + assert datacms.download(ds, tmp_path) == p + + cost = datacms.resolve("hospital_provider_cost_report", _catalog()) + pages = [ + [{"Provider CCN": "010001", "Hospital Name": "A"}] * datacms.PAGE_SIZE, + [{"Provider CCN": "010002", "Hospital Name": "B"}], + ] + + class J: + def __init__(self, rows): # noqa: ANN001 + self.rows = rows + + def json(self): # noqa: ANN202 + return self.rows + + seen = [] + + def fake_get(url, **kw): # noqa: ANN001, ANN003, ANN202 + seen.append(kw["params"]["offset"]) + return J(pages.pop(0) if pages else []) + + monkeypatch.setattr(datacms, "_get", fake_get) + p = datacms.download(cost, tmp_path) + assert seen == [0, datacms.PAGE_SIZE] + lines = p.read_text().splitlines() + assert ( + lines[0] == "Provider CCN,Hospital Name" and len(lines) == datacms.PAGE_SIZE + 2 + ) + + +def test_get_retries(monkeypatch) -> None: + attempts = [] + + def fake(url, **kw): # noqa: ANN001, ANN003, ANN202 + attempts.append(url) + if len(attempts) == 1: + raise httpx.ConnectError("boom") + if len(attempts) == 2: + return httpx.Response(503, request=httpx.Request("GET", url)) + return httpx.Response(200, json={"ok": 1}, request=httpx.Request("GET", url)) + + monkeypatch.setattr(datacms.httpx, "get", fake) + monkeypatch.setattr(datacms.time, "sleep", lambda s: None) + assert datacms._get("https://x").json() == {"ok": 1} and len(attempts) == 3