diff --git a/dev/vendor/MANIFEST.toml b/dev/vendor/MANIFEST.toml index beeb1f0..27a7701 100644 --- a/dev/vendor/MANIFEST.toml +++ b/dev/vendor/MANIFEST.toml @@ -25,6 +25,35 @@ files = [ "shared_files/gzip_feature_test/test_gzip_encoded.ndjson", "shared_files/gzip_feature_test/test_no_encoding.ndjson", "conf/configs/sandbox.yml", + "conf/configs/prod.yml", + "bcda/fhir/param.go", + "bcda/client/bluebutton.go", + "bcda/service/service.go", + "bcda/constants/constants.go", + "bcda/constants/types.go", + "bcda/models/models.go", +] + +[[source]] +name = "bcda-static-site" +kind = "github" +repo = "CMSgov/bcda-static-site" +ref = "main" +commit = "cd14babf8eca7247c35a6ba63ae5c221764ba236" +license = "LICENSE" +files = [ + "LICENSE", + "src/_pages/api-documentation/filter-claims-data.md", + "src/_pages/api-documentation/access-claims-data.md", + "src/_pages/api-documentation/how-to-migrate-v3.md", + "src/_pages/bcda-data/partially-adjudicated-claims-data.md", + "src/_pages/bcda-data/comparison-bcda-cclf-files.md", + "src/_pages/support/faqs.md", + "src/_announcements/2025-py-data-limit-ended.md", + "src/_announcements/bcda-data-limited-in-jan-2026-for-yearly-updates.md", + "src/_announcements/bcda-launches-v3.md", + "src/_announcements/bcda-v3-expanded-access-general-availability.md", + "src/_announcements/bcda-performance-improvement-to-speed-up-job-requests.md", ] [[source]] @@ -34,6 +63,8 @@ base = "https://bcda.cms.gov/assets/downloads/" files = [ "BCDA_v3_Data_Dictionary.xlsx", "BCDA_v2_Data_Dictionary.xlsx", + "BCDA_Partially_Adjudicated_Data_Dictionary.xlsx", + "unmapped-fields-between-cclf-and-bcda.xlsx", "v3_Patient.ndjson", "v3_Coverage.ndjson", "v3_ExplanationOfBenefit.ndjson", @@ -44,10 +75,17 @@ files = [ "bcda-app/bcda/api/requests.go" = "3ae1b33dec6e0d9c2c93352f3da7a713ffeeda545e5be4d00643b43e65569456" "bcda-app/bcda/api/v3/api.go" = "18a1dbc627d01a4d9f53113a6e45671494445da3fb8f9c877437f4ed8d62d871" "bcda-app/bcda/api/v3/api_test.go" = "97c0ca1d0af2f5fe2fa68102b8c6d6d9709f2775ed58cd291886e62e66cd869c" +"bcda-app/bcda/client/bluebutton.go" = "e70841e241d3b1a8a6b9dff6d029fdfc28d87f007ca86a4e7e0e202d3b729d62" +"bcda-app/bcda/constants/constants.go" = "bbfbcaf66ed1e9d4cf8160a2d3b18c9cc38a47444868181e36643e3d8f74fdef" +"bcda-app/bcda/constants/types.go" = "dafa4f745ef57c71289d65e05b642dd9e165db98a53ec2f3e6865ce2c9bf2b8c" +"bcda-app/bcda/fhir/param.go" = "693c26317f01741bfc5bd62b7955438858459e607b37e3caa8a56bbff7088ada" +"bcda-app/bcda/models/models.go" = "496a4c0212de5a7f86930c7b00b8d880f89b6708b29da2e99d015d46ef3b25df" "bcda-app/bcda/responseutils/v3/writer.go" = "d5bb9e8dfcad75bc3cf5cadf0f811e1b16d4e09cd0755970517ab3ab0867f56c" "bcda-app/bcda/responseutils/v3/writer_test.go" = "9056a6f0c4e6dc747375e232a982064d3e616b9832baa90f94c7bbe241bd58bc" +"bcda-app/bcda/service/service.go" = "f98c770026ec19cea0543937dc402f8c59a4be5415fac9632925ef8e78465158" "bcda-app/bcda/web/middleware/validation.go" = "dc4dfdf730c24961746f0b8f576b5a1cf912d7cec5e237401126738e6a82ed6a" "bcda-app/bcda/web/middleware/validation_test.go" = "c16f268ae3186b6881aa06ef1374d93117a969e87eafd13edcde0ac766a6b5e1" +"bcda-app/conf/configs/prod.yml" = "e644db4b609ce3f72a2b281db6a1bd24fe9648e8013a1de570710500173f0bc5" "bcda-app/conf/configs/sandbox.yml" = "a954b23194796658ddcfc8282ef3de488356a6076ed84266cfce6959df68e7f5" "bcda-app/shared_files/gzip_feature_test/corrupt_gz_file.ndjson" = "41b355a94e96ef4ed92c6fde19ca67d7341a5f95bd67e6b1e8a65f1b83be2b36" "bcda-app/shared_files/gzip_feature_test/test_gzip_encoded.ndjson" = "b5a7caf480fec615a1487f7bde4974f373395d98f245598c146844b1003f9a8f" @@ -55,8 +93,22 @@ files = [ "bcda-app/swagger.json" = "3565bdea5a5f0817b3abb95432645ef15e32179dbc18fc2a4d3de434d293e03f" "bcda-app/test/postman_test/BCDA_V3_Tests.postman_collection.json" = "fbb0c523f078f215bee178a9010fd331287c3b52ea6d712ea328e392ed77c237" "bcda-app/test/postman_test/sandbox.postman_environment.json" = "6d1866a8ae06185390ff80180bff0c38d93f591d71066d51b6a1f3a80cf6377c" +"bcda-site/BCDA_Partially_Adjudicated_Data_Dictionary.xlsx" = "9451a3988d7d1c8c3375c5a64c370bd19f35abc0febe3c0c8d36927d4bb1f233" "bcda-site/BCDA_v2_Data_Dictionary.xlsx" = "9c91eae07869f573c5578da9606336b00f1768180a947e2c4275864877f88810" "bcda-site/BCDA_v3_Data_Dictionary.xlsx" = "335547374a66ce88b4c88cf0657ec09f5e731af6b7ede998e0b162f6594ec3bc" +"bcda-site/unmapped-fields-between-cclf-and-bcda.xlsx" = "9818933895ce58bed27f9208758f7edd0594a6f6bc48770726b8f89c6af01313" "bcda-site/v3_Coverage.ndjson" = "cbe8820255f7a15d896b9098a33422d1e012cacc059316ac4894272132ff2986" "bcda-site/v3_ExplanationOfBenefit.ndjson" = "fafd76d16c335de44d1b28c388ac3de873809d3b2c429612ec6b8741db0fa82f" "bcda-site/v3_Patient.ndjson" = "7eb427266676415c6beee0fa705d7ad31d2335971f711bdfe0c63712392ba351" +"bcda-static-site/LICENSE" = "6d489af6292662d9e36d34ce49423784984a5f6e41d7b58f49b01264df59fa03" +"bcda-static-site/src/_announcements/2025-py-data-limit-ended.md" = "ff8014d6f2bbe232d176c62ea7999ad941fb584444242524b2e3615b2b8203d6" +"bcda-static-site/src/_announcements/bcda-data-limited-in-jan-2026-for-yearly-updates.md" = "bcf226627c84fe91c99f47c3987b426806ed458da474b7ce38e1a3029209410e" +"bcda-static-site/src/_announcements/bcda-launches-v3.md" = "efadb9f0a157844ff5b818273eb53fb5c719f12db35bc0f80384fffc3ab4f99c" +"bcda-static-site/src/_announcements/bcda-performance-improvement-to-speed-up-job-requests.md" = "9729ffa12daac14a6ae398fb841a0b0df414046cd5fb625ecf1eb862069313ba" +"bcda-static-site/src/_announcements/bcda-v3-expanded-access-general-availability.md" = "e0404747067c0c2723ec49b357eba68f5b3d8f934bf4c9221b1b3810cbbf44ff" +"bcda-static-site/src/_pages/api-documentation/access-claims-data.md" = "c48ef7cbc7a9fd54161758245d8de0ce3b81f030453da7ecb2f37ccc4eea6e57" +"bcda-static-site/src/_pages/api-documentation/filter-claims-data.md" = "e098ac1af2283796a1b64eb9b30c396a98f281e7a13de0f74a926f13c7412a69" +"bcda-static-site/src/_pages/api-documentation/how-to-migrate-v3.md" = "b07d928b182e41cfef65eb50a9d04d846cbca2120b8d4eab472b4d2df2689d76" +"bcda-static-site/src/_pages/bcda-data/comparison-bcda-cclf-files.md" = "c87000f64a7a16e1180d654881ce561ad7c63885f9228c8d78be10e703792a28" +"bcda-static-site/src/_pages/bcda-data/partially-adjudicated-claims-data.md" = "cc65fa79d657b6c41d5c989b830bf7e64adb62270cd7fbc0728123265d5423d8" +"bcda-static-site/src/_pages/support/faqs.md" = "f8655fb91ba744485a70adcf08491e04ee059ace7e7054769059b75cfbfed875" diff --git a/dev/vendor/bcda-app/bcda/client/bluebutton.go b/dev/vendor/bcda-app/bcda/client/bluebutton.go new file mode 100644 index 0000000..70d3c04 --- /dev/null +++ b/dev/vendor/bcda-app/bcda/client/bluebutton.go @@ -0,0 +1,603 @@ +package client + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "github.com/ccoveille/go-safecast" + backoff "github.com/cenkalti/backoff/v4" + + fhirClient "github.com/CMSgov/bcda-app/bcda/client/fhir" + "github.com/CMSgov/bcda-app/bcda/constants" + + "github.com/CMSgov/bcda-app/bcda/fhir" + "github.com/CMSgov/bcda-app/bcda/utils" + "github.com/CMSgov/bcda-app/bcdaworker/queueing/worker_types" + "github.com/CMSgov/bcda-app/conf" + + "github.com/pkg/errors" + + "github.com/sirupsen/logrus" + + "github.com/pborman/uuid" +) + +var logger logrus.FieldLogger + +const ( + clientIDHeader = "BULK-CLIENTID" + jobIDHeader = "BULK-JOBID" + TransactionIDHeader = "TRANSACTIONID" +) + +// BlueButtonConfig holds the configuration settings needed to create a BlueButtonClient +// TODO (BCDA-3755): Move the other env vars used in NewBlueButtonClient to this struct +type BlueButtonConfig struct { + BBServer string + BBBasePath string +} + +// NewConfig generates a new BlueButtonConfig using various environment variables. +func NewConfig(basePath string) BlueButtonConfig { + var server string + if basePath == constants.BFDV3Path { + server = conf.GetEnv("V3_BB_SERVER_LOCATION") + } else { + server = conf.GetEnv("BB_SERVER_LOCATION") + } + return BlueButtonConfig{ + BBServer: server, + BBBasePath: basePath, + } +} + +type ClaimsWindow struct { + Earliest time.Time + Latest time.Time +} + +type APIClient interface { + GetExplanationOfBenefit(jobData worker_types.JobEnqueueArgs, patientID string, claimsWindow ClaimsWindow) (*fhir.Bundle, error) + GetPatient(jobData worker_types.JobEnqueueArgs, patientID string) (*fhir.Bundle, error) + GetCoverage(jobData worker_types.JobEnqueueArgs, beneficiaryID string) (*fhir.Bundle, error) + GetPatientByMbi(jobData worker_types.JobEnqueueArgs, mbi string) (string, error) + GetClaim(jobData worker_types.JobEnqueueArgs, mbi string, claimsWindow ClaimsWindow) (*fhir.Bundle, error) + GetClaimResponse(jobData worker_types.JobEnqueueArgs, mbi string, claimsWindow ClaimsWindow) (*fhir.Bundle, error) +} + +type BlueButtonClient struct { + client fhirClient.Client + + maxTries uint64 + retryInterval time.Duration + + bbServer string + BBBasePath string +} + +// Ensure BlueButtonClient satisfies the interface +var _ APIClient = &BlueButtonClient{} + +func NewBlueButtonClient(config BlueButtonConfig) (*BlueButtonClient, error) { + certFile := conf.GetEnv("BB_CLIENT_CERT_FILE") + keyFile := conf.GetEnv("BB_CLIENT_KEY_FILE") + pageSize := utils.GetEnvInt("BB_CLIENT_PAGE_SIZE", 0) + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, errors.Wrap(err, "could not load Blue Button keypair") + } + + tlsConfig := &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12} + + if strings.ToLower(conf.GetEnv("BB_CHECK_CERT")) != "false" { + caFilePaths := strings.Split(conf.GetEnv("BB_CLIENT_CA_FILE"), ",") + caCertPool := x509.NewCertPool() + + for _, caFile := range caFilePaths { + caCert, err := os.ReadFile(filepath.Clean(caFile)) + if err != nil { + return nil, errors.Wrap(err, "could not read CA file") + } + + if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { + return nil, errors.New("could not append CA certificate(s)") + } + } + + tlsConfig.RootCAs = caCertPool + } else { + tlsConfig.InsecureSkipVerify = true // #nosec G402 this is only used in localdev/ci testing + logger.Warn("Blue Button certificate check disabled") + } + + transport := &http.Transport{ + TLSClientConfig: tlsConfig, + // Ensure that we have compression enabled. This allows the transport to request for gzip content + // and handle the decompression transparently. + // See: https://golang.org/src/net/http/transport.go?s=3396:10950#L182 for more information + DisableCompression: false, + } + var timeout int + if timeout, err = strconv.Atoi(conf.GetEnv("BB_TIMEOUT_MS")); err != nil { + logger.Warn(errors.Wrap(err, "Could not get Blue Button timeout from environment variable; using default value of 10000.")) + timeout = 10000 + } + + hl := &httpLogger{transport, logger} + httpClient := &http.Client{Transport: hl, Timeout: time.Duration(timeout) * time.Millisecond} + client := fhirClient.NewClient(httpClient, pageSize) + maxTries, err := safecast.ToUint64(utils.GetEnvInt("BB_REQUEST_MAX_TRIES", 3)) + if err != nil { + logger.Warn(errors.Wrap(err, "Could not convert Blue Button max retries from environment variable")) + } + retryInterval := time.Duration(utils.GetEnvInt("BB_REQUEST_RETRY_INTERVAL_MS", 1000)) * time.Millisecond + return &BlueButtonClient{client, maxTries, retryInterval, config.BBServer, config.BBBasePath}, nil +} + +// SetLogger sets the logger to be used in the client. +// Since both the API and worker use the bluebutton client, we need +// to be able to set the appropriate logger based on who is using the client. +func SetLogger(log logrus.FieldLogger) { + logger = log +} + +func (bbc *BlueButtonClient) GetPatient(jobData worker_types.JobEnqueueArgs, patientID string) (*fhir.Bundle, error) { + header := make(http.Header) + header.Add("IncludeAddressFields", "true") + params := GetDefaultParams() + params.Set("_id", patientID) + updateParamWithLastUpdated(¶ms, jobData.Since, jobData.TransactionTime) + + u, err := bbc.getURL("Patient", params) + if err != nil { + return nil, err + } + + return bbc.makeBundleDataRequest("GET", u, jobData, header, nil) +} + +func (bbc *BlueButtonClient) GetPatientByMbi(jobData worker_types.JobEnqueueArgs, mbi string) (string, error) { + headers := createURLEncodedHeader() + params := GetDefaultParams() + params.Set("identifier", fmt.Sprintf("http://hl7.org/fhir/sid/us-mbi|%s", mbi)) + + u, err := bbc.getURL("Patient/_search", url.Values{}) + if err != nil { + return "", err + } + + return bbc.getRawData("POST", jobData, u, headers, strings.NewReader(params.Encode())) +} + +func (bbc *BlueButtonClient) GetCoverage(jobData worker_types.JobEnqueueArgs, beneficiaryID string) (*fhir.Bundle, error) { + params := GetDefaultParams() + params.Set("beneficiary", beneficiaryID) + updateParamWithLastUpdated(¶ms, jobData.Since, jobData.TransactionTime) + + u, err := bbc.getURL("Coverage", params) + if err != nil { + return nil, err + } + + return bbc.makeBundleDataRequest("GET", u, jobData, nil, nil) +} + +func (bbc *BlueButtonClient) GetClaim(jobData worker_types.JobEnqueueArgs, mbi string, claimsWindow ClaimsWindow) (*fhir.Bundle, error) { + headers := createURLEncodedHeader() + params := GetDefaultParams() + updateParamsWithClaimsDefaults(¶ms, mbi) + updateParamWithServiceDate(¶ms, claimsWindow) + updateParamWithLastUpdated(¶ms, jobData.Since, jobData.TransactionTime) + + u, err := bbc.getURL("Claim/_search", url.Values{}) + if err != nil { + return nil, err + } + + return bbc.makeBundleDataRequest("POST", u, jobData, headers, strings.NewReader(params.Encode())) +} + +func (bbc *BlueButtonClient) GetClaimResponse(jobData worker_types.JobEnqueueArgs, mbi string, claimsWindow ClaimsWindow) (*fhir.Bundle, error) { + headers := createURLEncodedHeader() + params := GetDefaultParams() + updateParamsWithClaimsDefaults(¶ms, mbi) + updateParamWithServiceDate(¶ms, claimsWindow) + updateParamWithLastUpdated(¶ms, jobData.Since, jobData.TransactionTime) + + u, err := bbc.getURL("ClaimResponse/_search", url.Values{}) + if err != nil { + return nil, err + } + + return bbc.makeBundleDataRequest("POST", u, jobData, headers, strings.NewReader(params.Encode())) +} + +func (bbc *BlueButtonClient) GetExplanationOfBenefit(jobData worker_types.JobEnqueueArgs, patientID string, claimsWindow ClaimsWindow) (*fhir.Bundle, error) { + header := make(http.Header) + header.Add("IncludeTaxNumbers", "true") + params := GetDefaultParams() + params.Set("patient", patientID) + + if bbc.BBBasePath != constants.BFDV3Path { + params.Set("excludeSAMHSA", "true") + } else { + params.Set("_security:not", "42CFRPart2") + } + + updateParamWithServiceDate(¶ms, claimsWindow) + updateParamWithLastUpdated(¶ms, jobData.Since, jobData.TransactionTime) + updateParamWithTypeFilter(¶ms, jobData.TypeFilter) + setRestrictiveServiceDateWindow(¶ms) + + u, err := bbc.getURL("ExplanationOfBenefit", params) + if err != nil { + return nil, err + } + + return bbc.makeBundleDataRequest("GET", u, jobData, header, nil) +} + +func (bbc *BlueButtonClient) GetMetadata() (string, error) { + u, err := bbc.getURL("metadata", GetDefaultParams()) + if err != nil { + return "", err + } + jobData := worker_types.JobEnqueueArgs{} + + return bbc.getRawData("GET", jobData, u, nil, nil) +} + +func (bbc *BlueButtonClient) makeBundleDataRequest(method string, u *url.URL, jobData worker_types.JobEnqueueArgs, headers http.Header, body io.Reader) (*fhir.Bundle, error) { + var b *fhir.Bundle + for ok := true; ok; { + result, nextURL, err := bbc.tryBundleRequest(method, u, jobData, headers, body) + if err != nil { + return nil, err + } + + if b == nil { + b = result + } else { + b.Entries = append(b.Entries, result.Entries...) + } + + u = nextURL + ok = nextURL != nil + } + + return b, nil +} + +func (bbc *BlueButtonClient) tryBundleRequest(method string, u *url.URL, jobData worker_types.JobEnqueueArgs, headers http.Header, body io.Reader) (*fhir.Bundle, *url.URL, error) { + var ( + result *fhir.Bundle + nextURL *url.URL + err error + ) + + eb := backoff.NewExponentialBackOff() + eb.InitialInterval = bbc.retryInterval + b := backoff.WithMaxRetries(eb, bbc.maxTries) + + err = backoff.RetryNotify(func() error { + req, err := http.NewRequest(method, u.String(), body) + if err != nil { + logger.Error(err) + return err + } + + for key, values := range headers { + for _, value := range values { + req.Header.Add(key, value) + } + } + addDefaultRequestHeaders(req, uuid.NewRandom(), jobData) + + result, nextURL, err = bbc.client.DoBundleRequest(req) + if err != nil { + logger.Error(err) + } + return err + }, + b, + func(err error, d time.Duration) { + logger.Infof("Blue Button request failed %s. Retry in %s", err.Error(), d.String()) + }, + ) + + if err != nil { + return nil, nil, fmt.Errorf("blue button request failed %d time(s) %s", bbc.maxTries, err.Error()) + } + + return result, nextURL, nil +} + +func (bbc *BlueButtonClient) getRawData(method string, jobData worker_types.JobEnqueueArgs, u *url.URL, headers http.Header, body io.Reader) (string, error) { + eb := backoff.NewExponentialBackOff() + eb.InitialInterval = bbc.retryInterval + b := backoff.WithMaxRetries(eb, bbc.maxTries) + + var result string + + err := backoff.RetryNotify(func() error { + req, err := http.NewRequest(method, u.String(), body) + if err != nil { + logger.Error(err) + return err + } + + for key, values := range headers { + for _, value := range values { + req.Header.Add(key, value) + } + } + addDefaultRequestHeaders(req, uuid.NewRandom(), jobData) + + result, err = bbc.client.DoRaw(req) + if err != nil { + logger.Error(err) + } + return err + }, + b, + func(err error, d time.Duration) { + logger.Infof("Blue Button request failed %s. Retry in %s", err, d.String()) + }, + ) + + if err != nil { + return "", fmt.Errorf("blue button request failed %d time(s) %s", bbc.maxTries, err.Error()) + } + + return result, nil +} + +func (bbc *BlueButtonClient) getURL(path string, params url.Values) (*url.URL, error) { + u, err := url.Parse(fmt.Sprintf("%s%s/%s/", bbc.bbServer, bbc.BBBasePath, path)) + if err != nil { + return nil, err + } + u.RawQuery = params.Encode() + + return u, nil +} + +func addDefaultRequestHeaders(req *http.Request, reqID uuid.UUID, jobData worker_types.JobEnqueueArgs) { + // Info for BB backend: https://jira.cms.gov/browse/BLUEBUTTON-483 + req.Header.Add("keep-alive", "") + req.Header.Add(constants.BBHeaderTS, time.Now().String()) + req.Header.Add(constants.BBHeaderOriginQID, reqID.String()) + req.Header.Add(constants.BBHeaderOriginQC, "1") + req.Header.Add(constants.BBHeaderOriginURL, req.URL.String()) + req.Header.Add("IncludeIdentifiers", "mbi") + req.Header.Add(jobIDHeader, strconv.Itoa(jobData.ID)) + req.Header.Add(clientIDHeader, jobData.CMSID) + req.Header.Add(TransactionIDHeader, jobData.TransactionID) + + // We SHOULD NOT be specifying "Accept-Encoding: gzip" on the request header. + // If we specify this header at the client level, then we must be responsible for decompressing the response. + // This header should be automatically set by the underlying http.Transport which will handle the decompression transparently + // Details: https://golang.org/src/net/http/transport.go#L2432 + // req.Header.Add("Accept-Encoding", "gzip") + + // Do not set BB-specific headers with blank values + // Leaving them here, commented out, in case we want to set them to real values in future + //req.Header.Add("BlueButton-BeneficiaryId", "") + //req.Header.Add("BlueButton-OriginatingIpAddress", "") + //req.Header.Add("BlueButton-BackendCall", "") +} + +func GetDefaultParams() (params url.Values) { + params = url.Values{} + params.Set("_format", "application/fhir+json") + return params +} + +// updateParamWithServiceDate adds service-date query parameters for the ClaimsWindow, +// requesting claims on or after Earliest ("ge") and on or before Latest ("le"). +func updateParamWithServiceDate(params *url.Values, claimsWindow ClaimsWindow) { + // ServiceDate only uses yyyy-mm-dd + const isoDate = "2006-01-02" + + if !claimsWindow.Earliest.IsZero() { + params.Add("service-date", fmt.Sprintf("ge%s", claimsWindow.Earliest.Format(isoDate))) + } + + if !claimsWindow.Latest.IsZero() { + params.Add("service-date", fmt.Sprintf("le%s", claimsWindow.Latest.Format(isoDate))) + } +} + +type serviceDateVal struct { + prefix string + date time.Time +} + +// setRestrictiveServiceDateWindow sets the most restrictive window of time from which to pull data from BFD via service-date params. +// BFD only allows for one earliest boundary and one latest boundary. +// We need to do some comparisons to make sure we apply the most restrictive option for each. +// eg "gt2025" is a more restrictive earliest boundary than "gt2024" as that will only get us data starting from 2025 and not all the way back to 2024. +// known edge cases that are currently not accounted for (low risk, low priority, time constraints): +// - we dont allow for hours, mins, secs, tz, etc +func setRestrictiveServiceDateWindow(params *url.Values) { + serviceDates := (*params)["service-date"] + if len(serviceDates) <= 0 { + return + } + + var earliestDates, latestDates []serviceDateVal + for _, date := range serviceDates { + if strings.HasPrefix(date, "ge") || strings.HasPrefix(date, "gt") { + parsed, err := fhir.ParseDateString(date[2:]) + if err == nil { + earliestDates = append(earliestDates, serviceDateVal{prefix: date[:2], date: parsed}) + } + } else if strings.HasPrefix(date, "le") || strings.HasPrefix(date, "lt") { + parsed, err := fhir.ParseDateString(date[2:]) + if err == nil { + latestDates = append(latestDates, serviceDateVal{prefix: date[:2], date: parsed}) + } + } else if strings.HasPrefix(date, "eq") { + parsed, err := fhir.ParseDateString(date[2:]) + if err == nil { + earliestDates = append(earliestDates, serviceDateVal{prefix: "ge", date: parsed}) + latestDates = append(latestDates, serviceDateVal{prefix: "lt", date: parseLatestDateFromEqualPrefix(date[2:])}) + } + } else if strings.HasPrefix(date, "20") { + parsed, err := fhir.ParseDateString(date) + if err == nil { + earliestDates = append(earliestDates, serviceDateVal{prefix: "ge", date: parsed}) + latestDates = append(latestDates, serviceDateVal{prefix: "lt", date: parseLatestDateFromEqualPrefix(date)}) + } + } + } + + // remove any zero dates to cleanse list of invalids + earliestDates = slices.DeleteFunc(earliestDates, func(s serviceDateVal) bool { return s.date.IsZero() }) + latestDates = slices.DeleteFunc(latestDates, func(s serviceDateVal) bool { return s.date.IsZero() }) + + // sort earliest in descending order, and latest in ascending order so we can pick the most restrictive for each + slices.SortFunc(earliestDates, func(a, b serviceDateVal) int { + if c := b.date.Compare(a.date); c != 0 { + return c + } + + if a.prefix == "gt" { + return -1 + } + if b.prefix == "gt" { + return 1 + } + + return 0 + }) + slices.SortFunc(latestDates, func(a, b serviceDateVal) int { + if c := a.date.Compare(b.date); c != 0 { + return c + } + + if a.prefix == "lt" { + return -1 + } + if b.prefix == "lt" { + return 1 + } + + return 0 + }) + + // remove all existing service-date instances so we can start from scratch + params.Del("service-date") + if len(earliestDates) > 0 { + params.Add("service-date", earliestDates[0].prefix+earliestDates[0].date.Format("2006-01-02")) + } + if len(latestDates) > 0 { + params.Add("service-date", latestDates[0].prefix+latestDates[0].date.Format("2006-01-02")) + } +} + +func parseLatestDateFromEqualPrefix(date string) time.Time { + // date interval of a day; latest date is the beginning of the next day + parsedDate, err := time.Parse("2006-01-02", date) + if err == nil { + return parsedDate.AddDate(0, 0, 1) + } + + // date interval of a month; latest date is the beginning of first day of the next month + parsedDate, err = time.Parse("2006-01", date) + if err == nil { + return parsedDate.AddDate(0, 1, 0) + } + + // date interval of a year; latest date is the beginning of the first day of the next year + parsedDate, err = time.Parse("2006", date) + if err == nil { + return parsedDate.AddDate(1, 0, 0) + } + + // if a time is specified, treat it like a day interval; latest date is the beginning of the next day + // ToDo: update this and upstream/downstream logic to properly handle times + parsedDate, err = fhir.ParseDateString(date) + if err == nil { + return parsedDate.AddDate(0, 0, 1) + } + + return time.Time{} +} + +func updateParamWithLastUpdated(params *url.Values, since string, transactionTime time.Time) { + // upper bound will always be set + params.Set("_lastUpdated", "le"+transactionTime.Format(time.RFC3339Nano)) + + // only set the lower bound parameter if it exists and begins with "gt" (to align with what is expected in _lastUpdated) + if len(since) > 0 && strings.HasPrefix(since, "gt") { + params.Add("_lastUpdated", since) + } +} + +func updateParamWithTypeFilter(params *url.Values, typeFilter fhir.TypeFilterSubquery) { + for _, subqueryParam := range typeFilter.QueryParameters { + params.Add(subqueryParam.Name, subqueryParam.Value) + } +} + +func updateParamsWithClaimsDefaults(params *url.Values, mbi string) { + params.Set("excludeSAMHSA", "true") + params.Set("includeTaxNumbers", "true") + params.Set("isHashed", "false") + params.Set("mbi", mbi) +} + +func createURLEncodedHeader() http.Header { + headers := make(http.Header) + headers.Add("Content-Type", "application/x-www-form-urlencoded") + + return headers +} + +type httpLogger struct { + t *http.Transport + l logrus.FieldLogger +} + +func (h *httpLogger) RoundTrip(req *http.Request) (*http.Response, error) { + go h.logRequest(req.Clone(context.Background())) + resp, err := h.t.RoundTrip(req) + if resp != nil { + h.logResponse(req, resp) + } + return resp, err +} + +func (h *httpLogger) logRequest(req *http.Request) { + h.l.WithFields(logrus.Fields{ + "bb_query_id": req.Header.Get(constants.BBHeaderOriginQID), + "bb_query_ts": req.Header.Get(constants.BBHeaderTS), + "bb_uri": req.URL.String(), + "job_id": req.Header.Get(jobIDHeader), + "cms_id": req.Header.Get(clientIDHeader), + }).Infoln("request") +} + +func (h *httpLogger) logResponse(req *http.Request, resp *http.Response) { + h.l.WithFields(logrus.Fields{ + "resp_status": resp.StatusCode, + "bb_query_id": req.Header.Get(constants.BBHeaderOriginQID), + "bb_query_ts": req.Header.Get(constants.BBHeaderTS), + "bb_uri": req.URL.String(), + "job_id": req.Header.Get(jobIDHeader), + "cms_id": req.Header.Get(clientIDHeader), + "transaction_id": req.Header.Get(TransactionIDHeader), + }).Infoln("response") +} diff --git a/dev/vendor/bcda-app/bcda/constants/constants.go b/dev/vendor/bcda-app/bcda/constants/constants.go new file mode 100644 index 0000000..fc4df09 --- /dev/null +++ b/dev/vendor/bcda-app/bcda/constants/constants.go @@ -0,0 +1,68 @@ +package constants + +const DevACOUUID = "0c527d2e-2e8a-4808-b11d-0fa06baf8254" +const SmallACOUUID = "3461C774-B48F-11E8-96F8-529269fb1459" +const MediumACOUUID = "C74C008D-42F8-4ED9-BF88-CEE659C7F692" +const LargeACOUUID = "8D80925A-027E-43DD-8AED-9A501CC4CD91" + +const ImportInprog = "In-Progress" +const ImportComplete = "Completed" +const ImportFail = "Failed" + +// This is set during compilation. See dockerfiles for usage +var Version = "latest" + +const Adjudicated = "adjudicated" +const PartiallyAdjudicated = "partially-adjudicated" + +// Cli.go constants +const CliCMSIDArg = "cms-id" +const CliCMSIDDesc = "CMS ID of ACO" +const CliArchDesc = "How long files should wait in archive before deletion" +const CliRemoveArchDesc = "Remove job directory and files from archive and update job status to Expired" +const CliAuthToolsCategory = "Authentication tools" +const CliDataImpCategory = "Data import" + +const ContentType = "Content-Type" +const JsonContentType = "application/json" +const FHIRJsonContentType = "application/fhir+json" + +const BBHeaderTS = "BlueButton-OriginalQueryTimestamp" +const BBHeaderOriginURL = "BlueButton-OriginalUrl" +const BBHeaderOriginQID = "BlueButton-OriginalQueryId" +const BBHeaderOriginQ = "BlueButton-OriginalQuery" +const BBHeaderOriginQC = "BlueButton-OriginalQueryCounter" + +const CCLFFileRetID = "%s RETURNING id" +const JobKeyCreateErr = "failed to create job key: %w" + +const JOBIDPath = "/jobs/{jobID}" + +const IssuerSSAS = "ssas" + +const EmptyString = "" + +const FiveSeconds = "5" +const FiveHundredSeconds = "500" + +const CCLF8FileNum = int(8) + +const BFDV1Path = "/v1/fhir" +const BFDV2Path = "/v2/fhir" +const BFDV3Path = "/v3/fhir" +const V3Version = "v3" + +const GetExistingBenes = "GetExistingBenes" +const GetNewAndExistingBenes = "GetNewAndExistingBenes" + +const DefaultRegion = "us-east-1" + +const PublisherName = "Centers for Medicare & Medicaid Services" +const SoftwareName = "Beneficiary Claims Data API" +const SoftwareDescription = "The Beneficiary Claims Data API (BCDA) enables healthcare organizations participating in the Medicare Shared Savings Program or a CMS Innovation Model to retrieve Medicare Part A, Part B, and Part D claims data for their prospectively assigned or assignable beneficiaries." + +const SmartOAuthURIsExtensionURL = "http://fhir-registry.smarthealthit.org/StructureDefinition/oauth-uris" +const RestfulSecurityServiceSystem = "http://terminology.hl7.org/CodeSystem/restful-security-service" +const BFDSystemTypeURL = "https://bluebutton.cms.gov/fhir/CodeSystem/System-Type" +const BFDFinalActionURL = "https://bluebutton.cms.gov/fhir/CodeSystem/Final-Action" +const WarningsAndInfoFileName = "warnings-and-info.ndjson" diff --git a/dev/vendor/bcda-app/bcda/constants/types.go b/dev/vendor/bcda-app/bcda/constants/types.go new file mode 100644 index 0000000..20e7166 --- /dev/null +++ b/dev/vendor/bcda-app/bcda/constants/types.go @@ -0,0 +1,9 @@ +package constants + +type DataRequestType uint8 + +const ( + DefaultRequest DataRequestType = iota + RetrieveNewBeneHistData // Allows caller to retrieve all of the data for newly attributed beneficiaries + Runout // Allows caller to retrieve claims data for beneficiaries no longer attributed to the ACO +) diff --git a/dev/vendor/bcda-app/bcda/fhir/param.go b/dev/vendor/bcda-app/bcda/fhir/param.go new file mode 100644 index 0000000..12fcf51 --- /dev/null +++ b/dev/vendor/bcda-app/bcda/fhir/param.go @@ -0,0 +1,304 @@ +package fhir + +import ( + "fmt" + "slices" + "strings" + "time" + + "github.com/CMSgov/bcda-app/bcda/constants" +) + +type TypeFilterSubquery struct { + ResourceType string + QueryParameters []TypeFilterSubqueryParam +} + +type TypeFilterSubqueryParam struct { + Name string + Value string +} + +type TypeFilterParamName string + +const ( + TypeFilterParamOutcome TypeFilterParamName = "outcome" + TypeFilterParamServiceDate TypeFilterParamName = "service-date" + TypeFilterParamTag TypeFilterParamName = "_tag" +) + +type ParameterParsingError struct { + Details string +} + +func (e ParameterParsingError) Error() string { + return fmt.Sprintf("malformed parameter: %s", e.Details) +} + +type ParameterValidationError struct { + Details string +} + +func (e ParameterValidationError) Error() string { + return fmt.Sprintf("invalid parameter: %s", e.Details) +} + +func ParseTypeFilterSubquery(s string) (TypeFilterSubquery, error) { + // Expected format is: ? + resourceType, params, found := strings.Cut(s, "?") + if !found { + return TypeFilterSubquery{}, ParameterParsingError{Details: fmt.Sprintf("_typeFilter parameter missing question mark %s", s)} + } else if len(resourceType) == 0 { + return TypeFilterSubquery{}, ParameterParsingError{Details: fmt.Sprintf("_typeFilter parameter missing resource type %s", s)} + } else if len(params) == 0 { + return TypeFilterSubquery{}, ParameterParsingError{Details: fmt.Sprintf("_typeFilter parameter missing value %s", s)} + } + + var subqueryParams []TypeFilterSubqueryParam + paramAry := strings.SplitSeq(params, "&") + for paramPair := range paramAry { + name, value, found := strings.Cut(paramPair, "=") + if !found { + return TypeFilterSubquery{}, ParameterParsingError{Details: fmt.Sprintf("_typeFilter value missing equals sign: %s", paramPair)} + } + subqueryParams = append(subqueryParams, TypeFilterSubqueryParam{Name: name, Value: value}) + } + return TypeFilterSubquery{ResourceType: resourceType, QueryParameters: subqueryParams}, nil +} + +func ValidateTypeFilterSubquery(subquery TypeFilterSubquery) error { + if subquery.ResourceType != "ExplanationOfBenefit" { + return ParameterValidationError{Details: fmt.Sprintf("invalid _typeFilter Resource Type (Only EOBs valid): %s", subquery.ResourceType)} + } + + var serviceDateParams []DateParam + for _, param := range subquery.QueryParameters { + if strings.HasPrefix(param.Name, string(TypeFilterParamTag)) { + tag, err := ParseTokenParam(param) + if err != nil { + return err + } + err = ValidateTag(tag) + if err != nil { + return err + } + } else if strings.HasPrefix(param.Name, string(TypeFilterParamServiceDate)) { + date, err := ParseDateParam(param) + if err != nil { + return err + } + serviceDateParams = append(serviceDateParams, date) + } else if strings.HasPrefix(param.Name, string(TypeFilterParamOutcome)) { + outcome, err := ParseTokenParam(param) + if err != nil { + return err + } + err = ValidateOutcome(outcome) + if err != nil { + return err + } + } else { + return ParameterValidationError{Details: fmt.Sprintf("invalid _typeFilter subquery parameter: %s", param.Name)} + } + } + err := ValidateServiceDates(serviceDateParams) + if err != nil { + return err + } + return nil +} + +func GetTagParams(subquery TypeFilterSubquery) ([]TokenParam, error) { + tagParams := []TokenParam{} + for _, param := range subquery.QueryParameters { + if strings.HasPrefix(param.Name, string(TypeFilterParamTag)) { + tag, err := ParseTokenParam(param) + if err != nil { + return nil, err + } + tagParams = append(tagParams, tag) + } + } + return tagParams, nil +} + +// https://hl7.org/fhir/R4/search.html#string +type StringParam struct { + Name string + Modifier string + Values []string +} + +func ParseStringParam(subqueryParam TypeFilterSubqueryParam) (StringParam, error) { + s := StringParam{} + s.Name, s.Modifier, _ = strings.Cut(subqueryParam.Name, ":") + if len(s.Name) == 0 { + return s, ParameterParsingError{Details: "string parameter missing name"} + } + if len(subqueryParam.Value) == 0 { + return s, ParameterParsingError{Details: fmt.Sprintf("string parameter missing value: %s", s.Name)} + } + s.Values = strings.Split(subqueryParam.Value, ",") + return s, nil +} + +// https://hl7.org/fhir/R4/search.html#date +type DateParam struct { + Name string + Prefix string + Datetimes []string +} + +func ParseDateParam(subqueryParam TypeFilterSubqueryParam) (DateParam, error) { + d := DateParam{} + if len(subqueryParam.Name) == 0 { + return d, ParameterParsingError{Details: "date parameter missing key"} + } + d.Name = subqueryParam.Name + + if len(subqueryParam.Value) == 0 { + return d, ParameterParsingError{Details: fmt.Sprintf("date parameter missing value: %s", d.Name)} + } + + // BFD only supports eq, ge, gt, lt, le as of 2026-08-17. See: https://cmsgov.slack.com/archives/CMT1YS2KY/p1786716165942379 + var prefixes = []string{"eq", "lt", "gt", "le", "ge"} // "ne", "sa", "eb", "ap"} + afterPrefix := "" + if len(subqueryParam.Value) >= 2 && slices.Contains(prefixes, subqueryParam.Value[:2]) { + d.Prefix = subqueryParam.Value[:2] + afterPrefix = subqueryParam.Value[2:] + } else { + afterPrefix = subqueryParam.Value + } + + if len(afterPrefix) == 0 { + return d, ParameterParsingError{Details: fmt.Sprintf("value must include a valid FHIR date for date parameter %s", d.Name)} + } + + d.Datetimes = strings.Split(afterPrefix, ",") + + for _, datetime := range d.Datetimes { + _, err := ParseDateString(datetime) + if err != nil { + return d, ParameterParsingError{Details: fmt.Sprintf("invalid date parameter value: %s", datetime)} + } + } + return d, nil +} + +func ParseDateString(datetime string) (time.Time, error) { + var datetimeFormats = []string{ + "2006", + "2006-01", + "2006-01-02", + "2006-01-02T15:04:05", + "2006-01-02T15:04:05+07:00", + "2006-01-02T15:04:05-07:00", + "2006-01-02T15:04:05Z", + } + for _, format := range datetimeFormats { + if parsed, err := time.Parse(format, datetime); err == nil { + return parsed, nil + } + } + return time.Time{}, ParameterParsingError{Details: fmt.Sprintf("invalid date parameter value: %s", datetime)} +} + +// ValidateServiceDates ensures that service-date parameters have valid keys, non-delimited values, +// and at most one earliest ("gt"/"ge") and one latest ("lt"/"le") condition. +// Exact date conditions ("eq" or "") establish both an earliest and a latest boundary. +func ValidateServiceDates(serviceDateParams []DateParam) error { + earliestBoundCount, latestBoundCount := 0, 0 + for _, sd := range serviceDateParams { + if sd.Name != string(TypeFilterParamServiceDate) { + return ParameterValidationError{Details: fmt.Sprintf("invalid key for service-date parameter: %s", sd.Name)} + } + if len(sd.Datetimes) > 1 { + return ParameterValidationError{Details: "invalid service-date parameter value: comma-separated values are not supported"} + } + switch sd.Prefix { + case "lt", "le": + latestBoundCount = latestBoundCount + 1 + case "gt", "ge": + earliestBoundCount = earliestBoundCount + 1 + case "eq", "": + latestBoundCount = latestBoundCount + 1 + earliestBoundCount = earliestBoundCount + 1 + } + } + if earliestBoundCount > 1 || latestBoundCount > 1 { + return ParameterValidationError{Details: "invalid service-date parameter value: conflicting prefix conditions"} + } + return nil +} + +// https://hl7.org/fhir/R4/search.html#token +type TokenParam struct { + Name string + Modifier string + Values []TokenValue +} + +type TokenValue struct { + System string + Code string +} + +func ParseTokenParam(subqueryParam TypeFilterSubqueryParam) (TokenParam, error) { + t := TokenParam{} + if len(subqueryParam.Name) == 0 { + return t, ParameterParsingError{Details: "token parameter missing key"} + } + t.Name, t.Modifier, _ = strings.Cut(subqueryParam.Name, ":") + if len(subqueryParam.Value) == 0 { + return t, ParameterParsingError{Details: fmt.Sprintf("token parameter missing value %s", t.Name)} + } + values := strings.SplitSeq(subqueryParam.Value, ",") + for value := range values { + before, after, found := strings.Cut(value, "|") + if found { + t.Values = append(t.Values, TokenValue{System: before, Code: after}) + } else { + t.Values = append(t.Values, TokenValue{Code: before}) + } + } + return t, nil +} + +var validTagTokens = map[string][]string{ + constants.BFDSystemTypeURL: {"SharedSystem", "NationalClaimsHistory", "DDPS"}, + constants.BFDFinalActionURL: {"FinalAction", "NotFinalAction"}, +} + +func ValidateTag(t TokenParam) error { + if t.Name != string(TypeFilterParamTag) { + return ParameterValidationError{Details: fmt.Sprintf("invalid key for _tag parameter: %s", t.Name)} + } + if len(t.Modifier) > 0 { + return ParameterValidationError{Details: fmt.Sprintf("invalid _tag parameter; modifier %s not supported", t.Modifier)} + } + for _, value := range t.Values { + if len(value.System) == 0 || len(value.Code) == 0 { + return ParameterValidationError{Details: "invalid _tag parameter value. Searching by _tag requires a token (system|code) to be specified"} + } + validTagCodes, ok := validTagTokens[value.System] + if !ok || !slices.Contains(validTagCodes, value.Code) { + return ParameterValidationError{Details: fmt.Sprintf("invalid _tag parameter, unsupported code %s for system %s", value.Code, value.System)} + } + } + return nil +} + +func ValidateOutcome(t TokenParam) error { + if t.Name != string(TypeFilterParamOutcome) { + return ParameterValidationError{Details: fmt.Sprintf("invalid key for outcome parameter: %s", t.Name)} + } + if len(t.Modifier) > 0 { + return ParameterValidationError{Details: fmt.Sprintf("invalid outcome parameter; modifier %s not supported", t.Modifier)} + } + for _, value := range t.Values { + if value.Code != "complete" && value.Code != "partial" { + return ParameterValidationError{Details: fmt.Sprintf("invalid outcome code: %s. Supported outcome codes are 'complete' and 'partial'", value.Code)} + } + } + return nil +} diff --git a/dev/vendor/bcda-app/bcda/models/models.go b/dev/vendor/bcda-app/bcda/models/models.go new file mode 100644 index 0000000..1724b31 --- /dev/null +++ b/dev/vendor/bcda-app/bcda/models/models.go @@ -0,0 +1,174 @@ +package models + +import ( + "fmt" + "strings" + "time" + + "github.com/pborman/uuid" +) + +const ( + JobStatusArchived JobStatus = "Archived" + JobStatusCancelled JobStatus = "Cancelled" + JobStatusCancelledExpired JobStatus = "CancelledExpired" // JobStatusCancelledExpired represents a job that has been cancelled whose data has been cleaned up + JobStatusCompleted JobStatus = "Completed" + JobStatusExpired JobStatus = "Expired" + JobStatusFailed JobStatus = "Failed" + JobStatusFailedExpired JobStatus = "FailedExpired" // JobStatusFailedExpired represents a job that failed whose data has been cleaned up + JobStatusInProgress JobStatus = "In Progress" + JobStatusPending JobStatus = "Pending" +) + +var AllJobStatuses = []JobStatus{ + JobStatusArchived, + JobStatusCancelled, + JobStatusCancelledExpired, + JobStatusCompleted, + JobStatusExpired, + JobStatusFailed, + JobStatusFailedExpired, + JobStatusInProgress, + JobStatusPending, +} + +type JobStatus string +type Job struct { + ID uint + ACOID uuid.UUID `json:"aco_id"` + RequestURL string `json:"request_url"` // request_url + Status JobStatus `json:"status"` // status + TransactionTime time.Time // most recent data load transaction time from BFD + JobCount int + CreatedAt time.Time + UpdatedAt time.Time + BenesAttributedToACO int // Total beneficiaries attributed to ACO at time of job request +} + +func (j *Job) StatusMessage(numCompletedJobKeys int) string { + if j.Status == JobStatusInProgress && j.JobCount > 0 { + pct := float64(numCompletedJobKeys) / float64(j.JobCount) * 100 + return fmt.Sprintf("%s (%d%%)", j.Status, int(pct)) + } + + return string(j.Status) +} + +// BlankFileName contains the naming convention for empty ndjson file +const BlankFileName string = "blank.ndjson" + +type JobKey struct { + ID uint + JobID uint `json:"job_id"` + // Although que_job records are temporary, we store the ID to ensure + // that workers are never duplicating job keys. + QueJobID *int64 + FileName string + ResourceType string + BenesWithData int // count of beneficiaries with entry data + BenesRetrievedPercent int // percent of beneficiaries successfully retrieved from BFD +} + +func (j *JobKey) IsError() bool { + return strings.Contains(j.FileName, "-error.ndjson") +} + +// ACO represents an Accountable Care Organization. +type ACO struct { + ID uint + UUID uuid.UUID `json:"uuid"` + CMSID *string `json:"cms_id"` + Name string `json:"name"` + ClientID string `json:"client_id"` + GroupID string `json:"group_id"` + SystemID string `json:"system_id"` + TerminationDetails *Termination `json:"termination"` +} + +// Denylisted returns bool based on TerminationDetails. +// If CutoffDate is specified, the ACO is denylisted when time.Now() >= CutoffDate. +// If CutoffDate is not specified, it falls back to checking DenylistType. +func (aco *ACO) Denylisted() bool { + if aco.TerminationDetails != nil { + if !aco.TerminationDetails.CutoffDate.IsZero() { + return !time.Now().Before(aco.TerminationDetails.CutoffDate) + } + return aco.TerminationDetails.DenylistType == Involuntary || aco.TerminationDetails.DenylistType == Voluntary + } + return false +} + +type CCLFFileType int16 + +const ( + FileTypeDefault CCLFFileType = iota + FileTypeRunout +) + +// String returns the letter associated with the CCLFFileType. +func (t CCLFFileType) String() string { + return [...]string{"Y", "R"}[t] +} + +type CCLFFile struct { + ID uint + CCLFNum int + Name string + ACOCMSID string + Timestamp time.Time + PerformanceYear int + ImportStatus string + Type CCLFFileType + // CreatedAt is automatically set by the database. This is only + // set/pulled when querying data in the DB. + CreatedAt time.Time +} + +// "The MBI has 11 characters, like the Health Insurance Claim Number (HICN), which can have up to 11." +// https://www.cms.gov/Medicare/New-Medicare-Card/Understanding-the-MBI-with-Format.pdf +type CCLFBeneficiary struct { + ID uint + FileID uint + MBI string + BlueButtonID string +} + +// A BenePrefsFile is a basic file representation that can be stored in a database. +type BenePrefsFile struct { + ID uint + Name string + Timestamp time.Time + ImportStatus string +} + +// BenePrefsFilenameMetadata is metadata information parsed from the filename. +type BenePrefsFilenameMetadata struct { + Name string + Timestamp time.Time + FilePath string + Imported bool + DeliveryDate time.Time + FileID uint +} + +func (m BenePrefsFilenameMetadata) String() string { + if m.FilePath != "" { + return m.FilePath + } + return m.Name +} + +// A BenePrefsRecord represents a single record parsed from an bene-prefs file. +type BenePrefsRecord struct { + ID uint + FileID uint + MBI string + SourceCode string + EffectiveDt time.Time + PrefIndicator string + SAMHSASourceCode string + SAMHSAEffectiveDt time.Time + SAMHSAPrefIndicator string + ACOCMSID string + BeneficiaryLinkKey int +} diff --git a/dev/vendor/bcda-app/bcda/service/service.go b/dev/vendor/bcda-app/bcda/service/service.go new file mode 100644 index 0000000..6e6f43a --- /dev/null +++ b/dev/vendor/bcda-app/bcda/service/service.go @@ -0,0 +1,710 @@ +package service + +import ( + "context" + goerrors "errors" + "fmt" + "regexp" + "slices" + "time" + + "github.com/ccoveille/go-safecast" + "github.com/pborman/uuid" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/CMSgov/bcda-app/bcda/constants" + bcdaerrors "github.com/CMSgov/bcda-app/bcda/errors" + "github.com/CMSgov/bcda-app/bcda/models" + "github.com/CMSgov/bcda-app/bcda/utils" + "github.com/CMSgov/bcda-app/bcdaworker/queueing/worker_types" + "github.com/CMSgov/bcda-app/conf" + "github.com/CMSgov/bcda-app/log" + "github.com/CMSgov/bcda-app/middleware" +) + +// Ensure service satisfies the interface +var _ Service = &service{} + +// Service contains all of the methods needed to interact with the data represented in the models package +type Service interface { + GetCutoffTime(ctx context.Context, reqType constants.DataRequestType, since time.Time, timeConstraints TimeConstraints, fileType models.CCLFFileType) (time.Time, string) + FindOldCCLFFile(ctx context.Context, cmsID string, since time.Time, cclfTimestamp time.Time) (uint, error) + GetQueJobs(ctx context.Context, args worker_types.PrepareJobArgs) (queJobs []*worker_types.JobEnqueueArgs, benesAttributed int, err error) + GetJobAndKeys(ctx context.Context, jobID uint) (*models.Job, []*models.JobKey, error) + GetJobKey(ctx context.Context, jobID uint, filename string) (*models.JobKey, error) + GetJobs(ctx context.Context, acoID uuid.UUID, statuses ...models.JobStatus) ([]*models.Job, error) + CancelJob(ctx context.Context, jobID uint) (uint, error) + GetJobPriority(acoID string, resourceType string, sinceParam bool) int16 + GetLatestCCLFFile(ctx context.Context, cmsID string, earlierTime time.Time, laterTime time.Time, fileType models.CCLFFileType) (*models.CCLFFile, error) + GetACOConfigForID(cmsID string) (*ACOConfig, bool) + GetTimeConstraints(ctx context.Context, cmsID string) (TimeConstraints, error) + GetAttributionStatusDates(ctx context.Context, cmsID string, timeConstraints TimeConstraints, fileType models.CCLFFileType) (lastUpdated time.Time, expirationDate time.Time, err error) + IsV3NoPartialClaimsModel(model string) bool +} + +type service struct { + repository models.Repository + + logger logrus.FieldLogger + + stdCutoffDuration time.Duration + sp suppressionParameters + rp runoutParameters + bbBasePath string + + // These are always searched in order and first matching config is used for any given ACO. + acoConfigs []ACOConfig + v3NoPartialClaimsModels []string +} + +func NewService(r models.Repository, cfg *Config, basePath string) Service { + return &service{ + repository: r, + logger: log.API, + stdCutoffDuration: cfg.CutoffDuration, + sp: suppressionParameters{ + includeSuppressedBeneficiaries: false, + lookbackDays: cfg.SuppressionLookbackDays, + }, + rp: runoutParameters{ + // Runouts apply to claims data for the previous year. + claimThruDate: cfg.RunoutConfig.claimThru, + CutoffDuration: cfg.RunoutConfig.CutoffDuration, + }, + bbBasePath: basePath, + acoConfigs: cfg.ACOConfigs, + v3NoPartialClaimsModels: cfg.V3NoPartialClaimsModels, + } +} + +type TimeConstraints struct { + AttributionDate time.Time + OptOutDate time.Time + ClaimsDate time.Time +} + +type suppressionParameters struct { + includeSuppressedBeneficiaries bool + lookbackDays int +} + +type runoutParameters struct { + // All claims data occur at or before this date + claimThruDate time.Time + // Amount of time the callers can retrieve runout data (relative to when runout data was ingested) + CutoffDuration time.Duration +} + +func (s *service) GetCutoffTime(ctx context.Context, reqType constants.DataRequestType, since time.Time, timeConstraints TimeConstraints, fileType models.CCLFFileType) (cutoffTime time.Time, complexDataRequestType string) { + hasAttributionDate := !timeConstraints.AttributionDate.IsZero() + // for default requests, runouts, or any requests where the Since parameter is + // after a terminated ACO's attribution date, we should only retrieve exisiting benes + if reqType == constants.DefaultRequest || + reqType == constants.Runout || + (hasAttributionDate && since.After(timeConstraints.AttributionDate)) { + complexDataRequestType = constants.GetExistingBenes + // only set a cutoffTime if there is no attributionDate set + if timeConstraints.AttributionDate.IsZero() { + if fileType == models.FileTypeDefault && s.stdCutoffDuration > 0 { + cutoffTime = time.Now().Add(-1 * s.stdCutoffDuration) + } else if fileType == models.FileTypeRunout && s.rp.CutoffDuration > 0 { + cutoffTime = time.Now().Add(-1 * s.rp.CutoffDuration) + } + } + } else if reqType == constants.RetrieveNewBeneHistData { + complexDataRequestType = constants.GetNewAndExistingBenes + // only set cutoffTime if there is no attributionDate set + if s.stdCutoffDuration > 0 && timeConstraints.AttributionDate.IsZero() { + cutoffTime = time.Now().Add(-1 * s.stdCutoffDuration) + } + } + + return cutoffTime, complexDataRequestType +} + +// GetAttributionStatusDates provides the last updated date and the final date for which the most recent CCLF file for a CMS ID +// can be used to provide attribution information for data requests of a given file type. +// +// The expiration date can be limited by the date of the most recent CCLF file or the termination of the model entity in question. +func (s *service) GetAttributionStatusDates(ctx context.Context, cmsID string, timeConstraints TimeConstraints, fileType models.CCLFFileType) (lastUpdated time.Time, expirationDate time.Time, err error) { + latestFile, err := s.GetLatestCCLFFile(ctx, cmsID, expirationDate, expirationDate, fileType) + if err != nil { + s.logger.Errorf("could not fetch latest CCLF file for attribution status: %s", err) + return time.Time{}, time.Time{}, err + } + + lastUpdated = latestFile.Timestamp + switch fileType { + case models.FileTypeDefault: + expirationDate = latestFile.Timestamp.Add(s.stdCutoffDuration) + case models.FileTypeRunout: + expirationDate = latestFile.Timestamp.Add(s.rp.CutoffDuration) + default: + s.logger.Errorf("unsupported file type for fetching attribution expiration date") + return lastUpdated, expirationDate, fmt.Errorf("unsupported file type for fetching attribution expiration date") + } + + if !timeConstraints.AttributionDate.IsZero() && timeConstraints.AttributionDate.Before(expirationDate) { + return lastUpdated, timeConstraints.AttributionDate, nil + } + return lastUpdated, expirationDate, nil +} + +// FindOldCCLFFile finds an older CCLF file depending on passed in timestamps +func (s *service) FindOldCCLFFile(ctx context.Context, cmsID string, since time.Time, cclfTimestamp time.Time) (cclfFileOldID uint, err error) { + // Retrieve an older CCLF file for beneficiary comparison. + // This should be older than cclfFileNew AND prior to the "since" parameter, if provided. + // + // e.g. + // - If it’s October 2023 + // - and they request all beneficiary data “since January 1st, 2023" + // - any beneficiaries added in 2023 are considered "new." + // + oldFileTime := since + + // If the _since parameter is more recent than the latest CCLF file timestamp, e + // set the old file's time to be older than the newest file's timestamp. + if !since.IsZero() && cclfTimestamp.Sub(since) < 0 { + oldFileTime = cclfTimestamp.Add(-1 * time.Second) + } + + cclfFileOld, err := s.GetLatestCCLFFile( + ctx, + cmsID, + time.Time{}, + oldFileTime, + models.FileTypeDefault, + ) + if err != nil { + return 0, err + } + + return cclfFileOld.ID, nil +} + +func (s *service) GetQueJobs(ctx context.Context, args worker_types.PrepareJobArgs) (queJobs []*worker_types.JobEnqueueArgs, benesAttributed int, err error) { + var ( + beneficiaries, newBeneficiaries []*models.CCLFBeneficiary + jobs []*worker_types.JobEnqueueArgs + ) + + // for default requests, runouts, or any requests where the Since parameter is + // after a terminated ACO's attribution date, we should only retrieve exisiting benes + switch args.ComplexDataRequestType { + case constants.GetExistingBenes: + beneficiaries, err = s.getBeneficiaries(ctx, args) + if err != nil { + return nil, 0, err + } + case constants.GetNewAndExistingBenes: + newBeneficiaries, beneficiaries, err = s.getNewAndExistingBeneficiaries(ctx, args) + if err != nil { + return nil, 0, err + } + // add new beneficiaries to the job queue; use a default time value to ensure + // that we retrieve the full history for these beneficiaries + jobs, err = s.createQueueJobs(ctx, args, time.Time{}, newBeneficiaries) + if err != nil { + return nil, 0, err + } + queJobs = append(queJobs, jobs...) + default: + return nil, 0, fmt.Errorf("unsupported RequestType %d", args.RequestType) + } + + // add existiing beneficiaries to the job queue + jobs, err = s.createQueueJobs(ctx, args, args.Since, beneficiaries) + if err != nil { + return nil, 0, err + } + + benesAttributed = len(newBeneficiaries) + len(beneficiaries) + args.Job.BenesAttributedToACO = benesAttributed + err = s.repository.UpdateJob(ctx, args.Job) + if err != nil { + return nil, 0, fmt.Errorf("failed to update job with benesAttributed: %w", err) + } + + queJobs = append(queJobs, jobs...) + + return queJobs, benesAttributed, nil +} + +func (s *service) GetJobAndKeys(ctx context.Context, jobID uint) (*models.Job, []*models.JobKey, error) { + j, err := s.repository.GetJobByID(ctx, jobID) + if err != nil { + return nil, nil, err + } + + keys, err := s.repository.GetJobKeys(ctx, jobID) + if err != nil { + return nil, nil, err + } + + return j, keys, nil +} + +func (s *service) GetJobKey(ctx context.Context, jobID uint, filename string) (*models.JobKey, error) { + return s.repository.GetJobKey(ctx, jobID, filename) +} + +func (s *service) GetJobs(ctx context.Context, acoID uuid.UUID, statuses ...models.JobStatus) ([]*models.Job, error) { + jobs, err := s.repository.GetJobs(ctx, acoID, statuses...) + if err != nil { + return nil, err + } + + if jobs == nil { + return nil, JobsNotFoundError{acoID, statuses} + } + return jobs, nil +} + +type JobsNotFoundError struct { + ACOID uuid.UUID + StatusTypes []models.JobStatus +} + +func (e JobsNotFoundError) Error() string { + return fmt.Sprintf("no Jobs found for acoID %s with job statuses %s", + e.ACOID, e.StatusTypes) +} + +func (s *service) CancelJob(ctx context.Context, jobID uint) (uint, error) { + // Assumes the job exists and retrieves the job by ID + job, err := s.repository.GetJobByID(ctx, jobID) + if err != nil { + return 0, err + } + + if job.Status == models.JobStatusPending || job.Status == models.JobStatusInProgress || job.Status == models.JobStatusCompleted { + job.Status = models.JobStatusCancelled + err = s.repository.UpdateJob(ctx, *job) + if err != nil { + return 0, ErrJobNotCancelled + } + return jobID, nil + } + + // Return 0, ErrJobNotCancellable when attempting to cancel a non-cancellable job. + return 0, ErrJobNotCancellable +} + +// createQueueJobs expands a single "prepare job" request into concrete queue jobs by: +// - choosing the effective resource + data types for the ACO/config (including BFD v3 rules) +// - splitting beneficiaries into batches sized for the target resource type +// - emitting one queue job per (resourceType × dataType × beneficiaryBatch) +func (s *service) createQueueJobs(ctx context.Context, args worker_types.PrepareJobArgs, since time.Time, beneficiaries []*models.CCLFBeneficiary) (jobs []*worker_types.JobEnqueueArgs, err error) { + sinceArg := formatSinceArg(since) + resourceTypes, effectiveDataTypes, err := s.getEffectiveQueueJobConfig(args) + if err != nil { + return nil, err + } + + for _, resourceType := range resourceTypes { + maxBeneficiaries, err := getMaxBeneCount(resourceType) + if err != nil { + return nil, err + } + + for _, beneficiaryIDs := range chunkBeneficiaryIDs(beneficiaries, maxBeneficiaries) { + resourceJobs, err := s.createJobsForResourceChunk(ctx, args, sinceArg, resourceType, beneficiaryIDs, effectiveDataTypes) + if err != nil { + return nil, err + } + jobs = append(jobs, resourceJobs...) + } + } + + return jobs, nil +} + +// formatSinceArg converts a time into the "_lastUpdated" query parameter format expected by Blue Button. +// A zero time means "no since filter", which is represented as the empty string. +func formatSinceArg(since time.Time) string { + if since.IsZero() { + return "" + } + + // Persist in format ready for usage with _lastUpdated -- i.e., prepended with 'gt'. + return "gt" + since.Format(time.RFC3339Nano) +} + +// getEffectiveQueueJobConfig determines which resource types should be queued and which claim +// data types should be requested, based on the matching ACO config and BFD API version. +func (s *service) getEffectiveQueueJobConfig(args worker_types.PrepareJobArgs) ([]string, []string, error) { + acoCfg, ok := s.GetACOConfigForID(args.CMSID) + if !ok { + return nil, nil, &bcdaerrors.InvalidACOConfigError{CMSID: args.CMSID} + } + + resourceTypes := append([]string(nil), args.ResourceTypes...) + effectiveDataTypes := append([]string(nil), acoCfg.Data...) + if args.BFDPath != constants.BFDV3Path { + return resourceTypes, effectiveDataTypes, nil + } + + excluded := slices.Contains(s.v3NoPartialClaimsModels, acoCfg.Model) + if !excluded && !slices.Contains(acoCfg.Data, constants.PartiallyAdjudicated) { + effectiveDataTypes = append(effectiveDataTypes, constants.PartiallyAdjudicated) + } + + return resourceTypes, effectiveDataTypes, nil +} + +// chunkBeneficiaryIDs splits beneficiaries into fixed-size ID batches for queueing. +func chunkBeneficiaryIDs(beneficiaries []*models.CCLFBeneficiary, maxBeneficiaries int) [][]string { + if len(beneficiaries) == 0 { + return nil + } + + chunks := make([][]string, 0, (len(beneficiaries)+maxBeneficiaries-1)/maxBeneficiaries) + currentChunk := make([]string, 0, maxBeneficiaries) + + for _, beneficiary := range beneficiaries { + currentChunk = append(currentChunk, fmt.Sprint(beneficiary.ID)) + if len(currentChunk) < maxBeneficiaries { + continue + } + + chunks = append(chunks, currentChunk) + currentChunk = make([]string, 0, maxBeneficiaries) + } + + if len(currentChunk) > 0 { + chunks = append(chunks, currentChunk) + } + + return chunks +} + +// createJobsForResourceChunk produces enqueue arguments for a single resource type and beneficiary batch, +// creating a distinct job per compatible claim data type. +func (s *service) createJobsForResourceChunk(ctx context.Context, args worker_types.PrepareJobArgs, sinceArg string, resourceType string, beneficiaryIDs []string, effectiveDataTypes []string) ([]*worker_types.JobEnqueueArgs, error) { + resource, ok := GetClaimType(resourceType) + if !ok { + // This should never be possible, would have returned earlier. + return nil, errors.New("Invalid resource type: " + resourceType) + } + + jobs := make([]*worker_types.JobEnqueueArgs, 0, len(effectiveDataTypes)) + for _, dataType := range effectiveDataTypes { + if !resource.SupportsClaimType(dataType) { + continue + } + + enqueueArgs, err := s.buildQueueJobArgs(ctx, args, sinceArg, beneficiaryIDs, resourceType, dataType) + if err != nil { + return nil, err + } + jobs = append(jobs, enqueueArgs) + } + + return jobs, nil +} + +func (s *service) buildQueueJobArgs(ctx context.Context, args worker_types.PrepareJobArgs, sinceArg string, beneficiaryIDs []string, resourceType string, dataType string) (*worker_types.JobEnqueueArgs, error) { + jobID, err := safecast.ToInt(args.Job.ID) + if err != nil { + log.API.Errorln(err) + } + + enqueueArgs := worker_types.JobEnqueueArgs{ + ID: jobID, + ACOID: args.ACOID.String(), + CMSID: args.CMSID, + BeneficiaryIDs: beneficiaryIDs, + ResourceType: resourceType, + Since: sinceArg, + TypeFilter: args.TypeFilter, + TransactionID: ctx.Value(middleware.CtxTransactionKey).(string), + TransactionTime: getQueueJobTransactionTime(args, dataType), + BBBasePath: args.BFDPath, + DataType: dataType, + } + + if !s.setClaimsDate(&enqueueArgs, args) { + return nil, &bcdaerrors.InvalidACOConfigError{CMSID: args.CMSID} + } + + return &enqueueArgs, nil +} + +func getQueueJobTransactionTime(args worker_types.PrepareJobArgs, dataType string) time.Time { + // Partially-adjudicated data can lag adjudicated claims ingestion, so use job creation time. + if dataType == constants.PartiallyAdjudicated { + return args.CreationTime + } + + return args.Job.TransactionTime +} + +// Returns the beneficiaries associated with the latest CCLF file for the given request conditions, +// split between existing beneficiaries and newly-attributed beneficiaries. +func (s *service) getNewAndExistingBeneficiaries(ctx context.Context, args worker_types.PrepareJobArgs) (newBeneficiaries, beneficiaries []*models.CCLFBeneficiary, err error) { + cclfFileNew, err := s.repository.GetCCLFFileByID(ctx, args.CCLFFileNewID) + if err != nil { + return nil, nil, err + } + if cclfFileNew == nil { + return nil, nil, fmt.Errorf("no CCLF8 file found for cmsID %s", args.CMSID) + } + + if !args.Since.IsZero() && cclfFileNew.CreatedAt.Sub(args.Since) < 0 { + // Retrieve all of the benes associated with this CCLF file. + benes, err := s.getBenesByFileID(ctx, cclfFileNew.ID, args) + if err != nil { + return nil, nil, err + } + + if len(benes) == 0 { + return nil, nil, fmt.Errorf("found 0 new beneficiaries from CCLF8 file for cmsID %s cclfFiledID %d", args.CMSID, cclfFileNew.ID) + } + + return newBeneficiaries, benes, nil + } + + cclfFileOld, err := s.repository.GetCCLFFileByID(ctx, args.CCLFFileOldID) + if err != nil { + return nil, nil, err + } + + if cclfFileOld == nil { + s.logger.Infof("Unable to find CCLF8 File for cmsID %s prior to date: %s; all beneficiaries will be considered NEW", args.CMSID, args.Since) + + newBeneficiaries, err = s.getBenesByFileID(ctx, cclfFileNew.ID, args) + if err != nil { + return nil, nil, err + } + if len(newBeneficiaries) == 0 { + return nil, nil, fmt.Errorf("found 0 new beneficiaries from CCLF8 file for cmsID %s cclfFiledID %d", args.CMSID, cclfFileNew.ID) + } + return newBeneficiaries, nil, nil + } + + oldMBIs, err := s.repository.GetCCLFBeneficiaryMBIs(ctx, cclfFileOld.ID) + if err != nil { + return nil, nil, fmt.Errorf("failed to retrieve MBIs for cmsID %s cclfFileID %d %s", args.CMSID, cclfFileOld.ID, err.Error()) + } + + // Retrieve all of the benes associated with this CCLF file. + benes, err := s.getBenesByFileID(ctx, cclfFileNew.ID, args) + if err != nil { + return nil, nil, err + } + if len(benes) == 0 { + return nil, nil, fmt.Errorf("found 0 new or existing beneficiaries from CCLF8 file for cmsID %s cclfFiledID %d", args.CMSID, cclfFileNew.ID) + } + + // Split the results beteween new and old benes based on the existence of the bene in the old map + oldMBIMap := make(map[string]struct{}, len(oldMBIs)) + for _, oldMBI := range oldMBIs { + oldMBIMap[oldMBI] = struct{}{} + } + for _, bene := range benes { + if _, ok := oldMBIMap[bene.MBI]; ok { + beneficiaries = append(beneficiaries, bene) + } else { + newBeneficiaries = append(newBeneficiaries, bene) + } + } + + return newBeneficiaries, beneficiaries, nil +} + +func (s *service) getBeneficiaries(ctx context.Context, args worker_types.PrepareJobArgs) ([]*models.CCLFBeneficiary, error) { + cclfFile, err := s.repository.GetCCLFFileByID(ctx, args.CCLFFileNewID) + if err != nil { + return nil, err + } + if cclfFile == nil { + return nil, fmt.Errorf("no CCLF8 file found for cmsID %s", args.CMSID) + } + + benes, err := s.getBenesByFileID(ctx, cclfFile.ID, args) + if err != nil { + return nil, err + } + if len(benes) == 0 { + return nil, fmt.Errorf("found 0 beneficiaries from CCLF8 file for cmsID %s cclfFiledID %d", args.CMSID, cclfFile.ID) + } + + return benes, nil +} + +func (s *service) getBenesByFileID(ctx context.Context, cclfFileID uint, args worker_types.PrepareJobArgs) ([]*models.CCLFBeneficiary, error) { + var ( + ignoredMBIs []string + err error + ) + if !s.sp.includeSuppressedBeneficiaries { + upperBound := args.OptOutDate + if args.OptOutDate.IsZero() { + upperBound = time.Now() + } + + if cfg, ok := s.GetACOConfigForID(args.CMSID); ok { + if !cfg.IgnoreSuppressions { + ignoredMBIs, err = s.repository.GetSuppressedMBIs(ctx, s.sp.lookbackDays, upperBound) + if err != nil { + return nil, fmt.Errorf("failed to retreive suppressedMBIs %s", err.Error()) + } + } + } else { + return nil, &bcdaerrors.InvalidACOConfigError{CMSID: args.CMSID} + } + + } + + benes, err := s.repository.GetCCLFBeneficiaries(ctx, cclfFileID, ignoredMBIs) + if err != nil { + return nil, fmt.Errorf("failed to get beneficiaries %s", err.Error()) + } + + return benes, nil +} + +// setClaimsDate computes the claims window to apply on the args +func (s *service) setClaimsDate(args *worker_types.JobEnqueueArgs, prepareArgs worker_types.PrepareJobArgs) bool { + // If the caller made a request for runout data + // it takes precedence over any other claims date + // that may be applied + if prepareArgs.RequestType == constants.Runout { + args.ClaimsWindow.Latest = s.rp.claimThruDate + } else if !prepareArgs.ClaimsDate.IsZero() { + args.ClaimsWindow.Latest = prepareArgs.ClaimsDate + } + + // Applies the earliest boundary (lower bound) from the first matching ACOConfig + cfg, ok := s.GetACOConfigForID(prepareArgs.CMSID) + if ok { + args.ClaimsWindow.Earliest = cfg.LookbackTime(prepareArgs.CMSID) + } + + return ok +} + +// Gets the priority for the job where the lower the number the higher the priority in the queue. +// Priority is based on the request parameters that the job is executing on. +// Note: River queue library requires a priority between 1 and 4 (inclusive) +func (s *service) GetJobPriority(acoID string, resourceType string, sinceParam bool) int16 { + if isPriorityACO(acoID) { + return int16(1) // priority level for jobs for synthetic ACOs that are used for smoke testing + } else { + return int16(4) // default priority level for jobs + } +} + +// GetACOConfigForID gets first matching currently loaded ACOConfig for the specified cmsID +func (s *service) GetACOConfigForID(cmsID string) (*ACOConfig, bool) { + for _, cfg := range s.acoConfigs { + if cfg.patternExp.MatchString(cmsID) { + return &cfg, true + } + } + + return nil, false +} + +// IsV3NoPartialClaimsModel checks if the provided model is listed in V3NoPartialClaimsModels +func (s *service) IsV3NoPartialClaimsModel(model string) bool { + return slices.Contains(s.v3NoPartialClaimsModels, model) +} + +// Checks to see if an ACO is priority ACO based on a regex pattern provided by an +// environment variable. +func isPriorityACO(acoID string) bool { + if priorityACOPattern := conf.GetEnv("PRIORITY_ACO_REG_EX"); priorityACOPattern != "" { + priorityACORegex := regexp.MustCompile(priorityACOPattern) + if priorityACORegex.MatchString(acoID) { + return true + } + } + return false +} + +func getMaxBeneCount(requestType string) (int, error) { + const ( + BCDA_FHIR_MAX_RECORDS_EOB_DEFAULT = 50 + BCDA_FHIR_MAX_RECORDS_PATIENT_DEFAULT = 5000 + BCDA_FHIR_MAX_RECORDS_COVERAGE_DEFAULT = 4000 + BCDA_FHIR_MAX_RECORDS_CLAIM_DEFAULT = 4000 + BCDA_FHIR_MAX_RECORDS_CLAIMRESPONSE_DEFAULT = 4000 + ) + var envVar string + var defaultVal int + + switch requestType { + case "ExplanationOfBenefit": + envVar = "BCDA_FHIR_MAX_RECORDS_EOB" + defaultVal = BCDA_FHIR_MAX_RECORDS_EOB_DEFAULT + case "Patient": + envVar = "BCDA_FHIR_MAX_RECORDS_PATIENT" + defaultVal = BCDA_FHIR_MAX_RECORDS_PATIENT_DEFAULT + case "Coverage": + envVar = "BCDA_FHIR_MAX_RECORDS_COVERAGE" + defaultVal = BCDA_FHIR_MAX_RECORDS_COVERAGE_DEFAULT + case "Claim": + envVar = "BCDA_FHIR_MAX_RECORDS_CLAIM" + defaultVal = BCDA_FHIR_MAX_RECORDS_CLAIM_DEFAULT + case "ClaimResponse": + envVar = "BCDA_FHIR_MAX_RECORDS_CLAIM_RESPONSE" + defaultVal = BCDA_FHIR_MAX_RECORDS_CLAIMRESPONSE_DEFAULT + default: + err := errors.New("invalid request type") + return -1, err + } + maxBeneficiaries := utils.GetEnvInt(envVar, defaultVal) + + return maxBeneficiaries, nil +} + +func (s *service) GetLatestCCLFFile(ctx context.Context, cmsID string, earlierTime time.Time, laterTime time.Time, fileType models.CCLFFileType) (*models.CCLFFile, error) { + cclfFile, err := s.repository.GetLatestCCLFFile(ctx, cmsID, constants.CCLF8FileNum, constants.ImportComplete, earlierTime, laterTime, fileType) + if err != nil { + return nil, err + } + + if cclfFile == nil { + return nil, CCLFNotFoundError{8, cmsID, fileType, earlierTime} + } + + return cclfFile, nil +} + +// GetTimeConstraints searches for any time bounds that we should apply on the associated ACO +func (s *service) GetTimeConstraints(ctx context.Context, cmsID string) (TimeConstraints, error) { + var constraint TimeConstraints + aco, err := s.repository.GetACOByCMSID(ctx, cmsID) + if err != nil { + return constraint, fmt.Errorf("failed to retrieve aco: %w", err) + } + + // If aco is not terminated, then we should not apply any time constraints + if aco.TerminationDetails == nil { + return constraint, nil + } + + constraint.AttributionDate = aco.TerminationDetails.AttributionDate() + constraint.ClaimsDate = aco.TerminationDetails.ClaimsDate() + constraint.OptOutDate = aco.TerminationDetails.OptOutDate() + return constraint, nil +} + +type CCLFNotFoundError struct { + FileNumber int + CMSID string + FileType models.CCLFFileType + CutoffTime time.Time +} + +func (e CCLFNotFoundError) Error() string { + return fmt.Sprintf("no CCLF%d file found for cmsID %s fileType %d cutoffTime %s", + e.FileNumber, e.CMSID, e.FileType, e.CutoffTime.String()) +} + +var ( + ErrJobNotCancelled = goerrors.New("job was not cancelled due to internal server error") + ErrJobNotCancellable = goerrors.New("job was not cancelled because it is not Pending or In Progress") +) diff --git a/dev/vendor/bcda-app/conf/configs/prod.yml b/dev/vendor/bcda-app/conf/configs/prod.yml new file mode 100644 index 0000000..3e89521 --- /dev/null +++ b/dev/vendor/bcda-app/conf/configs/prod.yml @@ -0,0 +1,111 @@ +aco_config: + # Kidney Care Choices (KCC) Model - Comprehensive Kidney Care Contracting + - model: 'CKCC' # Active, managed by 4i + name_pattern: '^C\d{4}$' + performance_year_transition: '01/01' + lookback_period: 2 + data: + - 'adjudicated' + # Kidney Care Choices (KCC) Model - Kidney Care First + - model: 'KCF' # Ended, managed by 4i + name_pattern: '^K\d{4}$' + performance_year_transition: '01/01' + lookback_period: 2 + data: + - 'adjudicated' + # ACO REACH Model, formerly Global and Professional Direct Contracting Model (GPDC) Model + - model: 'DC' # Active, managed by 4i + name_pattern: '^D\d{4}$' + performance_year_transition: '04/01' + lookback_period: 3 + data: + - 'partially-adjudicated' + - 'adjudicated' + # Medicare Shared Savings Program + - model: 'SSP' # Active, managed by ACO-MS + name_pattern: '^A\d{4}$' + data: + - 'adjudicated' + # Comprehensive ESRD Care (CEC) Model + - model: 'CEC' # Ended + name_pattern: '^E\d{4}$' + data: + - 'adjudicated' + # Next Generation ACO Model + - model: 'NGACO' # Ended + name_pattern: '^V\d{3}$' + data: + - 'adjudicated' + # A config for BCDA's test entities + - model: 'TEST' # Active, managed by BCDA Admin + name_pattern: '^TEST\d{3}$' + data: + - 'adjudicated' + - 'partially-adjudicated' + # Maryland Total Cost of Care (MD TCoC) Model + - model: 'TCOCMD' # Ended, managed by BCDA Admin + name_pattern: '^CT\d{4,6}$' + data: + - 'adjudicated' + - 'partially-adjudicated' + attribution_file: + file_type: 'csv' + name_pattern: '(P|T)\.(PCPB)\.(M)([0-9][0-9])(\d{2})\.(D\d{6}\.T\d{6})\d' + model_identifier: 'PCPB' + file_performance_year: 4 + file_date: 6 + metadata_matches: 7 + ignore_suppressions: true + # CMS Data Aggregation Contract + - model: 'CDAC' # Ended, managed by IHP + name_pattern: '^DA\d{4}$' + data: + - 'adjudicated' + - 'partially-adjudicated' + attribution_file: + file_type: 'csv' + name_pattern: '(P|T)\.(BCD)\.(DA\d{4}).(MBIY)(\d{2})\.(D\d{6}\.T\d{6})\d' + model_identifier: 'BCD' + file_performance_year: 5 + file_date: 6 + metadata_matches: 7 + # GUIDE (Guiding an Improved Dementia Experience) Model + - model: 'GUIDE' # Active, managed by IHP + name_pattern: '^GUIDE-\d{4}$' + data: + - 'adjudicated' + attribution_file: + file_type: 'csv' + name_pattern: '(P|T)\.(GUIDE)\.(GUIDE-)(\d{4})\.(Y)(\d{2})\.(D\d{6}\.T\d{6})\d' + model_identifier: 'GUIDE' + file_performance_year: 6 + file_date: 7 + metadata_matches: 8 + # ACCESS (Advancing Chronic Care with Effective, Scalable Solutions) Model + - model: 'ACCESS' # Active, managed by IHP + name_pattern: '^ACCES\d{5}$' # only one 's' due to character limits + lookback_period: 3 + data: + - 'adjudicated' + attribution_file: + file_type: 'csv' + name_pattern: '(P|T)\.(ACCESS)\.(ACCES\d{5})\.(Y)(\d{2})\.(D\d{6}\.T\d{6})\d' + model_identifier: 'ACCESS' + file_performance_year: 5 + file_date: 6 + metadata_matches: 7 + # Increasing Organ Transplant Access (IOTA) Model + - model: 'IOTA' # Active, managed via 4i + name_pattern: '^IOTA\d{3}$' + data: + - 'adjudicated' + - 'partially-adjudicated' +rate_limit_config: + all: true + acos: [] +v1_v2_deny_regexes: + - '^IOTA\d{3}' + - '^ACCES\d{5}$' + - '^GUIDE-\d{4}$' +v3_no_partial_claims_models: + - 'GUIDE' diff --git a/dev/vendor/bcda-site/BCDA_Partially_Adjudicated_Data_Dictionary.xlsx b/dev/vendor/bcda-site/BCDA_Partially_Adjudicated_Data_Dictionary.xlsx new file mode 100644 index 0000000..40ab711 Binary files /dev/null and b/dev/vendor/bcda-site/BCDA_Partially_Adjudicated_Data_Dictionary.xlsx differ diff --git a/dev/vendor/bcda-site/unmapped-fields-between-cclf-and-bcda.xlsx b/dev/vendor/bcda-site/unmapped-fields-between-cclf-and-bcda.xlsx new file mode 100644 index 0000000..5443234 Binary files /dev/null and b/dev/vendor/bcda-site/unmapped-fields-between-cclf-and-bcda.xlsx differ diff --git a/dev/vendor/bcda-static-site/LICENSE b/dev/vendor/bcda-static-site/LICENSE new file mode 100644 index 0000000..1625c17 --- /dev/null +++ b/dev/vendor/bcda-static-site/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. \ No newline at end of file diff --git a/dev/vendor/bcda-static-site/src/_announcements/2025-py-data-limit-ended.md b/dev/vendor/bcda-static-site/src/_announcements/2025-py-data-limit-ended.md new file mode 100644 index 0000000..fe856af --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_announcements/2025-py-data-limit-ended.md @@ -0,0 +1,22 @@ +--- +layout: announcement +page_title: "End of BCDA data limitations" +description: "Performance year (PY) 2026 data is now accessible." +show-side-nav: false +in-page-nav: true +lead_paragraph: "Performance year (PY) 2026 data now available." +published_date: 2026-02-12 +--- + +All model entities can now access PY 2026 data using the `/Group/all` and `/Patient` endpoints. + +- Use the `all` identifier for `/Group` and `/Patient` endpoint requests. +- Use the `all` identifier **for your first PY 2026 data request.** There will be no service date restrictions for this data. + +## Requesting PY 2025 runout data + +Use the `runout` identifier separately with the `/Group` endpoint. This will return data for enrollees who were attributed to your organization in PY 2025, but not PY 2026\. + +- Use the `_since` parameter in conjunction with the `runout` identifier to limit data to updates occurring since your last runout request. +- The `runout` identifier will be updated each month up to and including July 2026. +- After July 2026, new data for enrollees who were attributed to your organization in PY 2025, but not PY 2026, will not be included. \ No newline at end of file diff --git a/dev/vendor/bcda-static-site/src/_announcements/bcda-data-limited-in-jan-2026-for-yearly-updates.md b/dev/vendor/bcda-static-site/src/_announcements/bcda-data-limited-in-jan-2026-for-yearly-updates.md new file mode 100644 index 0000000..c200830 --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_announcements/bcda-data-limited-in-jan-2026-for-yearly-updates.md @@ -0,0 +1,57 @@ +--- +layout: announcement +page_title: "BCDA data limited in Jan 2026 for yearly updates" +description: "Claims with 2026 service dates will be unavailable throughout January 2026." +show-side-nav: false +in-page-nav: true +lead_paragraph: "Claims with 2026 service dates will be unavailable throughout January 2026." +published_date: 2025-12-03 +--- + +Certain BCDA data is unavailable every January while CMS updates attributed beneficiaries for the new performance year (PY). Normal functionality will be restored to individual entities as soon as their data is ready. This typically takes 1 month.  + +PY 2026 starts Jan 1, 2026 and ends Dec 31, 2026.  + +## What data will be unavailable in Jan 2026? + +Requests to `/Group/all` and `/Patient` endpoints will be disabled while we wait for updated attribution info for your model entity. + +## How should I request PY 2025 runout data? + +To request runout data for enrollees attributed to your organization in PY 2025: + +* Use the `runout` identifier with the `/Group` endpoint.  +* Claims with a service date in 2026 will be excluded. Data returned will have a service date no later than December 31, 2025.  +* Use the [\_since parameter]({{ '/api-documentation/filter-claims-data.html#the-_since-parameter' | relative_url }}) with the `runout` identifier to return data updated since your last runout request. [Review our documentation on the runout identifier]({{ '/api-documentation/filter-claims-data.html#the-runout-identifier' | relative_url }}).  + +## How will I know when data is available again? + +We’ll notify you in the [BCDA Google Group](https://groups.google.com/g/bc-api) when the limitation ends for each supported model. At that point, you can use the `all` identifier to request data, including 2026 claims, with the `/Group` and `/Patient` endpoints as usual. + +## Examples of requests/outcomes to expect in Jan 2026 + +- A caller makes a request to the `/Patient` or `/Group` endpoint using the `all` identifier. + - GET `/api/v2/Patient/$export` + - GET `/api/v2/Group/all/$export` + +- The caller receives a `404 Not Found` response and an `OperationOutcome` error. + - An example `OperationOutcome.fhir.json` is attached. + - Callers should use the `/api/v2/Group/runout/$export` endpoint. + +- A caller makes a request to the `/Group` endpoint using the `runout` identifier. + - GET `/api/v2/Group/runout/$export` + +- The caller receives a `202 Accepted` response. + - Content-Location header sent with the job location. + - `Content-Location: /api/v2/jobs/{job_id}` + +- A caller polls the `/Jobs` endpoint until the job completes. + - GET `/api/v2/jobs/{job_id}` + +- The caller can now download the job data. + - GET `/data/2/output.json` + - All claims data will be capped at December 31, 2025. + +## Need help? + +Keep an eye on the [Google Group conversation](https://groups.google.com/g/bc-api/c/0v78i8P1sKo) to ask questions and read updates about BCDA data availability. If you can’t find what you need, send an email to [bcapi@cms.hhs.gov](mailto:bcapi@cms.hhs.gov).  \ No newline at end of file diff --git a/dev/vendor/bcda-static-site/src/_announcements/bcda-launches-v3.md b/dev/vendor/bcda-static-site/src/_announcements/bcda-launches-v3.md new file mode 100644 index 0000000..f2a127f --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_announcements/bcda-launches-v3.md @@ -0,0 +1,61 @@ +--- +layout: announcement +page_title: "BCDA v3: Enhanced timeliness and performance for value-based care" +description: "Core improvements to data sources and alignment mean timelier, more person-centered care. " +show-side-nav: false +in-page-nav: true +lead_paragraph: "Core improvements to data sources and alignment mean timelier, more person-centered care." +published_date: 2026-07-01 +--- + +Centers for Medicare & Medicaid Services (CMS) is excited to announce version 3 (v3) of its [FHIR](http://hl7.org/fhir/) Medicare claims data sharing platform. v3 drives improvements to CMS Beneficiary Claims Data API (BCDA), [Blue Button API](https://bluebutton.cms.gov/), and [Claims Data to Part D Sponsors API (AB2D)](https://ab2d.cms.gov/). [BCDA v3]({{ '/about/introducing-v3.html' | relative_url }}) helps model entities coordinate care more effectively, track performance on CMS quality measures, and improve patient outcomes. + +## BCDA v3 improvements + +With BCDA v3, model entities get patient claims data earlier, allowing them to identify issues and intervene sooner. v3 strengthens alignment with Fast Healthcare Interoperability Resources (FHIR) standards and other CMS data sources, reducing time and money spent on integrations and reconciliation. Driving these improvements are: + +- A new, single source of Medicare claims data +- Better alignment with sources such as Claim and Claim Line Feed (CCLF) files + +These fundamental changes improve data frequency, tracking, filtering, mapping, and FHIR conformance. + +### Timelier, more frequent updates + +v3 offers timelier updates to patient and coverage data, meaning: + +- Shorter time between full processing (adjudication) and data availability +- Increased frequency of updates to patient and coverage information + +### Easier claims tracking + +BCDA v3 makes it easier to track claims through the adjudication cycle. It uses one FHIR resource instead of 3 to represent fully and partially processed versions of a claim. v3 also simplifies the way BCDA denotes claims status. + +### Enhanced filtering capabilities + +The new [_typeFilter query parameter]({{ '/api-documentation/filter-claims-data.html#the-_typefilter-parameter' | relative_url }}) provides additional options for complex data workflows. + +### Improved conformance with select FHIR Implementation Guides + +v3 standardizes extensions while providing better FHIR conformance. CodeSystems are represented using `StructureDefinition` and `CodeSystem` resources. + +### Simplified, reliable data mapping + +BCDA v3 data aligns more closely with CCLF files, and generates its Data Dictionary automatically. + +## The IDR: a single source of claims data + +v3 consolidates upstream data to a single data source, the Integrated Data Repository (IDR). This change: + +- Reduces mismatches between BCDA resources and CCLF files +- Resolves missing data for newly attributed enrollees +- Fixes issues for enrollees assigned more than one `BENE_ID` +- Simplifies linking between partially and fully adjudicated claims +- Uses consistent claim identifiers across all adjudication phases + +## What this means for healthcare organizations + +Through a unified data structure and consolidated data sources, BCDA v3 can reduce development time, improve data quality, and ensure providers have up-to-date information about every Medicare patient. + +## What this means for current users + +The v1 and v2 endpoints will be disabled on July 30, 2027. Refer to the [v3 Migration Guide]({{ '/api-documentation/how-to-migrate-v3.html' | relative_url }}) for steps and support on upgrading to v3. \ No newline at end of file diff --git a/dev/vendor/bcda-static-site/src/_announcements/bcda-performance-improvement-to-speed-up-job-requests.md b/dev/vendor/bcda-static-site/src/_announcements/bcda-performance-improvement-to-speed-up-job-requests.md new file mode 100644 index 0000000..4adec18 --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_announcements/bcda-performance-improvement-to-speed-up-job-requests.md @@ -0,0 +1,23 @@ +--- +layout: announcement +page_title: "BCDA performance improvement to speed up job requests" +show-side-nav: false +in-page-nav: true +published_date: 2024-12-01 +description: "Reducing resources file and increasing rate limits speeds up the time to complete long-running jobs with BCDA." +lead_paragraph: "New changes decrease the time it takes to complete long-running jobs. " +custom_excerpt: "New changes decrease the time it takes to complete long-running jobs. " +--- + +We've decreased the time it takes to complete long-running jobs by reducing the number of EOB resources per ndjson file. We've also increased the rate limit on API requests by 10X to accommodate the additional files. + +What's changing? +- EOB resources per ndjson file are reduced from 200 to 50. +- We've increased the rate limit from 300 to 3,000 per 5 minutes from a single IP address. + +Why is it changing? +- The change in EOB file volume decreases the time it takes to complete long-running jobs like requests for historical data. +- The API rate limit increase lowers likelihood of having your requests throttled. + +What can you do to prepare? +- There's no action required. It can be helpful to revisit our guidance on [handling 429 errors]({{ "/api-documentation/access-claims-data.html#response-example-too-many-requests" | relative_url }}) caused by exceeding the API rate limit. \ No newline at end of file diff --git a/dev/vendor/bcda-static-site/src/_announcements/bcda-v3-expanded-access-general-availability.md b/dev/vendor/bcda-static-site/src/_announcements/bcda-v3-expanded-access-general-availability.md new file mode 100644 index 0000000..e0fa307 --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_announcements/bcda-v3-expanded-access-general-availability.md @@ -0,0 +1,25 @@ +--- +layout: announcement +page_title: "BCDA v3: Expanded access, general availability" +description: "Over 600 new organizations eligible for BCDA in time for v3 general availability" +show-side-nav: false +in-page-nav: true +lead_paragraph: "Over 600 new organizations eligible for BCDA in time for v3 general availability" +published_date: 2026-10-07 +--- + +Summer 2026 increased the reach of FHIR-enabled Medicare claims data thanks to new [Innovation Models](https://www.cms.gov/priorities/innovation/models) and the general availability release of BCDA v3. + +## 3 new payment models can access BCDA + +As of July 2026, participants in the [ACCESS](https://www.cms.gov/priorities/innovation/innovation-models/access), [GUIDE](https://www.cms.gov/priorities/innovation/innovation-models/guide), and [IOTA](https://www.cms.gov/priorities/innovation/innovation-models/iota) models have access to Medicare claims data. Timely claims data will help model participants treat and manage complex conditions. These include chronic conditions, dementia, and needs arising around kidney transplants. + +## General availability means faster access to v3 + +With the phased rollout complete (from early adopter program, to limited release, to general availability), eligible organizations can now initiate their own access to BCDA v3. As announced in the [BCDA Google Group](https://groups.google.com/g/bc-api), you no longer need to submit a request to [onboard]({{ '/production-access.html' | relative_url }}) or [migrate]({{ '/api-documentation/how-to-migrate-v3.html' | relative_url }}) in order to get started. + +## Why move to v3? + +With v3, platform-level improvements lead to earlier access to claims data. Earlier access supports timely follow-ups and interventions. A closer alignment between v3 and other CMS data sources supports a more detailed, longitudinal picture of health for people with Medicare. + +Join the [BCDA Google Group](https://groups.google.com/g/bc-api) for additional news and support. diff --git a/dev/vendor/bcda-static-site/src/_pages/api-documentation/access-claims-data.md b/dev/vendor/bcda-static-site/src/_pages/api-documentation/access-claims-data.md new file mode 100644 index 0000000..509f609 --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_pages/api-documentation/access-claims-data.md @@ -0,0 +1,612 @@ +--- +layout: api-docs +page_title: "How to Access Claims Data" +seo_title: "" +description: "Learn how to access Medicare enrollees' Parts A, B, and D claims data in the BCDA production environment for performance tracking and risk analysis." +in-page-nav: true +feedback_id: "f179685c" +--- + +# {{ page.page_title }} + +Learn how to access claims data for both the sandbox and production environments and use Beneficiary Claims Data API (BCDA) endpoints. + +The sandbox and production environments follow similar instructions. They support the same workflow, endpoints, parameters, and resource types. + + + + + + + + + + + + + + + + + + + + + + + +
Sandbox and Production environments comparison
SandboxProduction
Available to everyone via test credentialsMust complete the steps for production credentials
Contains test claims dataContains real Medicare enrollee data
sandbox.bcda.cms.govapi.bcda.cms.gov
+ + +
+
+

BCDA recommends using v3 of the API

+

This is the latest version which follows the FHIR R4 specification.

+
+
+ +## Instructions + +### 1. Get a bearer token + +You will need a [bearer token]({{ '/api-documentation/get-a-bearer-token.html' | relative_url }}) to call the API. The process requires credentials, which are formatted as a client ID and client secret. + +
+
+

Jobs will not be interrupted when the bearer token expires. In progress and queued jobs will continue to run.

+
+
+ +### 2. Start a job + +Make a `GET` request to the /Group or /Patient endpoint to start a data export job. The examples below are sandbox curl requests to /Group. Follow along in your terminal or using a tool like Postman. + +#### Request all resource types + +By default, the `GET` request returns all available [resource types]({{ '/bcda-data.html#resource-types' | relative_url }}). + +{% capture curlSnippet %}{% raw %} +GET /api/v3/Group/all/$export +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +Use the [_type parameter]({{ '/api-documentation/filter-claims-data.html' | relative_url }}) to specify which resource types you'd like returned. + +#### Request header + +The header must contain your bearer token. You may receive a `401` response if your credentials are invalid or expired. `Bearer ` must be included in the header with a capital B and followed by a space. + +{% capture curlSnippet %}{% raw %} +Authorization: Bearer {bearer_token} +Accept: application/fhir+json +Prefer: respond-async +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="yaml" %} + +#### Example curl commands to start a job + +- Combine your `GET` request for resources with the request header. +- The dollar sign `$` before `export` in the URL indicates the endpoint is an operation, rather than a CRUD interaction. +- PowerShell users will need to replace backslash characters `\` with backticks ( \` ) to properly escape the `$export` operation. + +
+
+

Remember to use the correct URL for your environment

+

Use sandbox.bcda.cms.gov to access the sandbox or api.bcda.cms.gov to access the production environment.

+
+
+ + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Group/all/\$export" \ + -H "Accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Group/all/\$export?_type=ExplanationOfBenefit,Patient" \ + -H "Accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Group/all/\$export?_type=Patient" \ + -H "Accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +#### Response example: successful request + +A `202 Accepted` response with a Content-Location header indicates a successful request. + +#### Response header example + +You'll need the job ID in the Content-Location header to check your job status. + +{% capture curlSnippet %}{% raw %} +Content-Location: https://sandbox.bcda.cms.gov/api/v3/jobs/{job_id} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Response example: too many requests + +A `429 Too Many Requests` response can occur due to 2 reasons: + +1. Making too many HTTP requests within a period of time +2. Trying to recreate jobs already marked as "In-Progress.” For reference, you can view both existing and past jobs using the [/jobs endpoint]({{ '/api-documentation/access-claims-data.html' | relative_url }}#request-job-history). + +Wait until the period of time specified in the `Retry-After: ` header passes before making any more requests. This makes sure your client can adapt without manual intervention, even if the rate-limiting parameters change. + +### 3. Check job status + +Make a `GET` request to check the status using the job ID from step 2. You may need another bearer token if it's been over 20 minutes since it was generated. + +#### Request to check the job status + +{% capture curlSnippet %}{% raw %} +GET https://sandbox.bcda.cms.gov/api/v3/jobs/{job_id} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Request header + +{% capture curlSnippet %}{% raw %} +Authorization: Bearer {bearer_token} +Accept: application/fhir+json +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="yaml" %} + +#### curl command to check the job status + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/jobs/{job_id}" \ + -H "Accept: application/fhir+json" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +#### Response example: incomplete job + +A `202` response indicates your job is still processing. The status will change to `200 OK` when the export is complete and the data is ready for download. + +{% capture curlSnippet %}{% raw %} +202 Accepted +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Response header example: incomplete job + +The X-Progress will have a percentage indicating your estimated progress. + +{% capture curlSnippet %}{% raw %} +X-Progress: In Progress, 80% +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Response example: completed job + +You'll receive a `200 OK` response with the output URL(s) needed to download the data. In the example URLs below, `42` indicates the job ID. + +There is a separate URL for each resource type requested. The following example shows a request for all resource types. + +{% capture curlSnippet %}{% raw %} +{ + "transactionTime": "2026-12-09T20:44:01.705398Z", + "request": "https://sandbox.bcda.cms.gov/api/v3/Group/all/$export", + "requiresAccessToken": true, + "output": [ + { + "type": "ExplanationOfBenefit", + "url": "https://sandbox.bcda.cms.gov/data/42/afd22dfa-c239-4063-8882-eb2712f9f638.ndjson" + }, + { + "type": "Coverage", + "url": "https://sandbox.bcda.cms.gov/data/42/f76a0b76-48ed-4033-aad9-d3eec37e7e83.ndjson" + }, + { + "type": "Patient", + "url": "https://sandbox.bcda.cms.gov/data/42/f92dcf16-63a2-448e-a12a-3bf677f966ed.ndjson" + } + ], + "error": [], + "JobID": 42 +} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="json" %} + +### 4. Download the data + +Make a `GET` request to download your data using the URL(s) from step 3. + +If you're downloading from more than 1 URL, make multiple download requests concurrently to save time. Large files may take significantly longer to download. + +
+
+

Files expire after 24 hours

+

You can download these files using a new bearer token. After 24 hours, the files will expire and you will need to restart the job.

+
+
+ +#### Request to download the data + +{% capture curlSnippet %}{% raw %} +GET https://sandbox.bcda.cms.gov/data/{job_id}/{file_name} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Request header: compressed data files + +Request compressed data files with the optional `Accept-Encoding: gzip` header in your requests for faster download times. Afterward, decompress (unzip) the files into NDJSON format. + +{% capture curlSnippet %}{% raw %} +Authorization: Bearer {bearer_token} +Accept-Encoding: gzip +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="yaml" %} + +#### curl command to download the data + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/data/{job_id}/{file_name}" \ + -H "Accept-Encoding: gzip" \ + -H "Authorization: Bearer {bearer_token}" +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +If some of the data can't be exported due to errors, details can be found at the URLs in the error field. The errors are provided in an NDJSON file as FHIR OperationOutcome resources. + +#### Response example + +By default, you'll receive the requested data as FHIR resources in NDJSON format. Each resource type will appear as a separate, labeled file. + +
+
+

Test data from the sandbox contains only negative Patient IDs.

+
+
+ +
    +
  1. ExplanationOfBenefit.ndjson {% include sprite.html icon="file_download" class="text-middle" size="2" %}
    2. +
  2. Patient.ndjson {% include sprite.html icon="file_download" class="text-middle" size="2" %}
    3. +
  3. Coverage.ndjson {% include sprite.html icon="file_download" class="text-middle" size="2" %}
    4. +
+ +## Other BCDA endpoints + +### Cancel a job + +Cancel any active job. If the request is successful, you'll receive a `202` response. + +#### Request to cancel a job + +{% capture curlSnippet %}{% raw %} +DELETE /api/v3/jobs/{job_id} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Request header + +{% capture curlSnippet %}{% raw %} +Authorization: Bearer {bearer_token} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="yaml" %} + +#### curl command to cancel a job + +{% capture curlSnippet %}{% raw %} +curl -X DELETE "https://sandbox.bcda.cms.gov/api/v3/jobs/{job_id}" \ + -H "Accept: application/fhir+json" \ + -H "Authorization: Bearer {bearer_token}" +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +### Request job history + +Retrieve details on your organization's historical requests, including the start and end datetime, unique ID, original valueString request, and status. + +#### Request to retrieve all past jobs + +If your organization has no jobs to return, you'll receive a `404 ERROR` response. + + {% capture curlSnippet %}{% raw %} +GET /api/v3/jobs +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Request that filters jobs by end state + +Supply the _status parameter to filter for jobs with a specific end state. BCDA jobs have 9 possible end states: Completed, Archived, Expired, Failed, FailedExpired, In Progress, Pending, Cancelled, and CancelledExpired. + +These are how the job end states map to the 4 supported values you can receive in the response body: + +- `Archived`, `Expired`, `Completed` → `completed` +- `FailedExpired`, `Failed` → `failed` +- `Pending`, `In Progress` → `in-progress` +- `CancelledExpired`, `Cancelled` → `cancelled` + +The example below is a filtered request for all past archived jobs. If any are found, the response will list the status as `completed`. Even so, the filter will only return archived jobs; it will exclude expired and completed jobs. + +{% capture curlSnippet %}{% raw %} +GET /api/v3/jobs?_status=Archived +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Request header + +{% capture curlSnippet %}{% raw %} +Authorization: Bearer {bearer_token} +Accept: application/fhir+json +Prefer: respond-async +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="yaml" %} + +#### curl command to check the job status + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/jobs" \ + -H "Accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +#### Response example: completed job + +The response will contain a bundle of resources for each historical job. Each resource section in the response represents a single past job request. + +This example shows 1 historical job with a `completed` status. Since this was an unfiltered request, the job could either be archived, expired, or completed. + +{% capture curlSnippet %}{% raw %} +{ + "entry": [ + { + "resource": { + "executionPeriod": { + "end": "2026-08-14T00:07:48+00:00", + "start": "2026-08-13T00:07:48+00:00" + }, + "identifier": [ + { + "system": "https://bcda.cms.gov/api/v3/jobs", + "use": "official", + "value": "1" + } + ], + "input": [ + { + "type": { + "text": "BULK FHIR Export" + }, + "valueString": "GET https://bcda.test.gov/this-is-a-test" + } + ], + "intent": "order", + "resourceType": "Task", + "status": "completed" + } + } + ], + "resourceType": "Bundle", + "total": 1, + "type": "searchset" +} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="json" %} + +### Request attribution status + +Check your attribution status for a timestamp of when your attribution data was last updated. By comparing the timestamp to the date of your most recent job, you can determine if your organization has new claims data to download. + +#### Request to check attribution status + +{% capture curlSnippet %}{% raw %} +GET /api/v3/attribution_status +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Request header + +{% capture curlSnippet %}{% raw %} +Authorization: Bearer {bearer_token} +Accept: application/json +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="yaml" %} + +#### curl command to check attribution status + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/attribution_status" \ + -H "Accept: application/json" \ + -H "Authorization: Bearer {bearer_token}" +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +#### Response example +If BCDA has never ingested an attribution or runout file for your organization, you'll receive a `404 not found` response. +{% capture curlSnippet %}{% raw %} +{ + "ingestion_dates": [ + { + "type": "last_attribution_update", + "timestamp": "2020-12-22 22:31:40.397916+00" + }, + { + "type": "last_runout_update", + "timestamp": "2020-12-22 22:31:40.397916+00" + } + ] +} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="json" %} + +### Check API status + +Retrieve metadata to view the current status and release or FHIR version of the API. A bearer token, and therefore a response header, is not required. + +#### Request to check API status + +{% capture curlSnippet %}{% raw %} +GET /api/v3/metadata +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### curl command to check API status + +{% capture curlSnippet %}{% raw %} +curl "https://sandbox.bcda.cms.gov/api/v3/metadata" +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +#### Response example + +The response will contain a FHIR Capability Statement resource in JSON format. The example below shows the API is active, using FHIR Release 4.0.1, and that the API release version is r292. + +{% capture curlSnippet %}{% raw %} +{ + "resourceType": "CapabilityStatement", + "status": "active", + "date": "2026-07-17T16:20:37Z", + "publisher": "Centers for Medicare & Medicaid Services", + "kind": "instance", + "instantiates": [ + "https://prod-sbx.fhir.bfd.cmscloud.local/v3/fhir/metadata", + "http://hl7.org/fhir/uv/bulkdata/CapabilityStatement/bulk-data" + ], + "software": { + "name": "Beneficiary Claims Data API", + "version": "r292", + "releaseDate": "2026-07-17T16:20:37Z" + }, + "implementation": { + "description": "The Beneficiary Claims Data API (BCDA) enables healthcare organizations participating in the Medicare Shared Savings Program or a CMS Innovation Model to retrieve Medicare Part A, Part B, and Part D claims data for their prospectively assigned or assignable beneficiaries.", + "url": "https://sandbox.bcda.cms.gov" + }, + "fhirVersion": "4.0.1", + "format": [ + "application/json", + "application/fhir+json" + ], + "rest": [ + { + "mode": "server", + "security": { + "cors": true, + "service": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/restful-security-service", + "code": "OAuth", + "display": "OAuth" + } + ], + "text": "OAuth" + } + ], + "extension": [ + { + "url": "http://fhir-registry.smarthealthit.org/StructureDefinition/oauth-uris", + "extension": [ + { + "url": "token", + "valueUri": "https://sandbox.bcda.cms.gov/auth/token" + } + ] + } + ] + }, + "interaction": [ + { + "code": "batch" + }, + { + "code": "search-system" + } + ], + "resource": [ + { + "type": "Patient", + "operation": [ + { + "name": "export", + "definition": "http://hl7.org/fhir/uv/bulkdata/OperationDefinition/patient-export", + "documentation": "By default, the patient $export will return ExplanationOfBenefit resources with a meta.tag with a system of 'https://bluebutton.cms.gov/fhir/CodeSystem/System-Type' and code of either NationalClaimsHistory or DDPS. In order to return ExplanationOfBenefit resources with other system types (like SharedSystem), use the _typeFilter parameter." + } + ], + "searchParam": [ + { + "name": "_since", + "type": "date", + "documentation": "Return resources updated after the date provided for existing and newly attributed enrollees." + }, + { + "name": "_type", + "type": "string", + "documentation": "Comma-delimited list of FHIR resource types to include in the export. By default, all supported resource types are returned." + }, + { + "name": "_typeFilter", + "type": "string", + "documentation": "Use a URL-encoded FHIR subquery to further-refine patient export results." + } + ] + }, + { + "type": "Group", + "operation": [ + { + "name": "export", + "definition": "http://hl7.org/fhir/uv/bulkdata/OperationDefinition/group-export", + "documentation": "By default, the group $export will return ExplanationOfBenefit resources with a meta.tag with a system of 'https://bluebutton.cms.gov/fhir/CodeSystem/System-Type' and code of either NationalClaimsHistory or DDPS. In order to return ExplanationOfBenefit resources with other system types (like SharedSystem), use the _typeFilter parameter." + } + ], + "searchParam": [ + { + "name": "_since", + "type": "date", + "documentation": "Return resources updated after the date provided for existing enrollees and all resources for newly attributed enrollees." + }, + { + "name": "_type", + "type": "string", + "documentation": "Comma-delimited list of FHIR resource types to include in the export. By default, all supported resource types are returned." + }, + { + "name": "_typeFilter", + "type": "string", + "documentation": "Use a URL-encoded FHIR subquery to further-refine group export results." + } + ] + }, + { + "type": "ExplanationOfBenefit", + "searchParam": [ + { + "name": "_tag", + "type": "token", + "documentation": "Filter ExplanationOfBenefit by the meta.tag element. Pass full token as |. Supported codes in the 'https://bluebutton.cms.gov/fhir/CodeSystem/System-Type' system are: 'SharedSystem', 'NationalClaimsHistory', and 'DDPS'. By Default, only NationalClaimsHistory and DDPS claims will be returned." + }, + { + "name": "outcome", + "type": "token", + "documentation": "Filter ExplanationOfBenefit by the outcome element. Supported values: 'partial' and 'complete'." + } + ] + } + ] + } + ] +} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="json" %} diff --git a/dev/vendor/bcda-static-site/src/_pages/api-documentation/filter-claims-data.md b/dev/vendor/bcda-static-site/src/_pages/api-documentation/filter-claims-data.md new file mode 100644 index 0000000..de242ce --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_pages/api-documentation/filter-claims-data.md @@ -0,0 +1,240 @@ +--- +layout: api-docs +page_title: "How to Filter Claims Data" +seo_title: "" +description: "Learn how to filter Medicare claims data requests using the _since, _type, and _typeFilter parameters." +in-page-nav: true +feedback_id: "72cee024" +tabs: + - label: v3 + url: /api-documentation/filter-claims-data.html + - label: v2 + url: /v2/api-documentation/filter-claims-data.html +--- + +# {{ page.page_title }} + +{% include tabs.html items=page.tabs %} + +BCDA offers 3 query parameters to filter or specify the resources returned: +- **_since:** applies a date boundary to your request +- **_type:** limits your request to 1 or more specific resource types +- **_typeFilter:** limits your request to resources matching the provided criteria + +Use parameters in the sandbox or production environment to speed up download times and reduce file size. + +## The _since parameter + +The _since parameter lets you filter for claims data last updated after a specified date. Dates must be in the [FHIR standard instant format](https://www.hl7.org/fhir/datatypes.html#instant): `YYYY-MM-DDThh:mm:ss.sss+zz:zz`. + +- Sample date: July 1, 2026 12:00 PM EST +- Formatted sample: `2026-07-01T12:00:00.000-05:00` + +
+
+

Requesting data from before 03/26/2026

+

The FHIR resources available using BCDA v3 were first created in March 2026. Regardless of the service date, no resources from BCDA v3 will have a lastUpdated date earlier than March 2026.

+

If you specify a date prior to 03/26/2026 with the _since parameter, your request will include a large amount of additional historical data for your enrollees. Your request using _since with a date on or later than 03/26/2026 will include resources marked with appropriate lastUpdated dates.

+

If you wish to filter claims based on the date of service, use the _typeFilter parameter with service-date in your v3 request.

+
+
+ + +We recommend new model entities run an unfiltered request for all historical data before using _since subsequently for incremental exports of new data. You can use the transactionTime from your most recent job as the specified date. + +The _since parameter can be used with /Group or /Patient, but each endpoint will return data for newly attributed enrollees differently. + +### Using _since with /Patient + +Using _since with /Patient will return resources updated after the date provided for existing and newly attributed enrollees. + +Newly attributed enrollees are those who’ve been assigned to your model entity since your last attribution date. If you don’t apply _since, BCDA will return data as early as March 1, 2021. + +#### Example request using _since with /Patient + + +{% capture curlSnippet %}{% raw %} +GET /api/v3/Patient/$export?_type=Patient&_since=2026-07-01T08:00:00.000-05:00 +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Example curl command using _since with /Patient + +This command combines the `GET` request and request header. The dollar sign `$` before `export` in the URL indicates the endpoint is an action, not a resource. + + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Patient/\$export?_type=Patient&_since=2026-07-01T08:00:00.000-05:00" \ + -H "Accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +### Using _since with /Group + +Using _since with /Group will return resources updated after the date provided for existing enrollees and all resources for newly attributed enrollees. + +This lets you retrieve all new claims data with a single request. If you don't apply _since, BCDA will return data as early as March 1, 2021. + +#### Example request using _since with /Group + +The request below will return: + +- Claims data last updated since July 1, 2026 for existing enrollees +- Claims data as far back as 2014 for all new enrollees attributed to your organization in the last month + +{% capture curlSnippet %}{% raw %} +GET /api/v3/Group/all/$export?_type=Patient&_since=2026-07-01T08:00:00.000-05:00 +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Example curl command using _since with /Group + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Group/all/\$export?_type=Patient&_since=2026-07-01T08:00:00.000-05:00" \ + -H "Accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +## The _type parameter + +The _type parameter lets you specify which resource types you'd like returned. Otherwise, making a `GET` request to start a job with the /Group or /Patient endpoint will return all resource types. + +### Using _type + +You'll need to use commas when specifying multiple resource types. The examples below are curl requests to /Group using _type. + +#### Example request for 1 resource type + +{% capture curlSnippet %}{% raw %} +GET /api/v3/Group/all/$export?_type=Patient +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Example request for 2 resource types + +{% capture curlSnippet %}{% raw %} +GET /api/v3/Group/all/$export?_type=ExplanationOfBenefit,Patient +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Example curl command using _type + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Group/all/\$export?_type=ExplanationOfBenefit,Patient" \ + -H "Accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +## The _typeFilter parameter + +
+
+

Only available in v3

+

The _typeFilter parameter is only available in v3.

+
+
+ +The _typeFilter parameter lets you use resource metadata to create finer-grained filtering criteria. BCDA will only return claims data that meets the specified criteria provided as a URL Encoded FHIR REST API query. + +### Using _typeFilter + +Currently, _typeFilter supports queries using the `_tag`, `outcome`, and `service-date` search parameters for the ExplanationOfBenefit resource: + +* **_tag:** filters the export based on the `meta.tag` element. Pass a full token as `|`. + * Supported codes in the `https://bluebutton.cms.gov/fhir/CodeSystem/System-Type` system are: + * `SharedSystem` + * `NationalClaimsHistory` + * `DDPS` + * If you do not specify a System-Type _tag filter, the export will include only `NationalClaimsHistory` and `DDPS` claims by default. + +* **outcome:** filters the export based on the processing status of the claim. The two values for this param, `partial` and `complete`, represent the "Partially Adjudicated" and "Fully Adjudicated" claim statuses. + +* **service-date:** filters the export based on the claim's service date. The service date is the date that the care occurred within a billable period. This is a FHIR date param format (ex. `gt2026-01-14`) + + +#### Example request using _typeFilter + +The request below will return: +- All Patient and Coverage resources +- ExplanationOfBenefit resources of fully adjudicated claims only + +{% capture curlSnippet %}{% raw %} +GET /api/v3/Group/all/$export?_typeFilter=ExplanationOfBenefit%3Foutcome%3Dcomplete +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +#### Example curl command using _typeFilter + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Group/all/\$export?_typeFilter=ExplanationOfBenefit%3Foutcome%3Dcomplete" \ + -H "Accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +## The \`runout` identifier + +The \`runout` identifier lets you request runouts data for enrollees attributed to your model entity the previous year, but not the current year. Claims data returned will have a service date no later than December 31 of the previous year. + +### Using the `runout` identifier + +The examples below are `GET` requests made to the /Group endpoint. + +#### Example request for 1 resource type using `runout` + +{% capture curlSnippet %}{% raw %} +GET /api/v3/Group/runout/$export?_type=Patient +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Group/runout/\$export?_type=Patient" \ + -H "accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +#### Example request for 2 resource types using `runout` + +{% capture curlSnippet %}{% raw %} +GET /api/v3/Group/runout/$export?_type=ExplanationOfBenefit,Patient +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Group/runout/\$export?_type=ExplanationOfBenefit,Patient" \ + -H "accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} + +#### Request for all resources using `runout` + +{% capture curlSnippet %}{% raw %} +GET /api/v3/Group/runout/$export +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" %} + +{% capture curlSnippet %}{% raw %} +curl -X GET "https://sandbox.bcda.cms.gov/api/v3/Group/runout/\$export" \ + -H "accept: application/fhir+json" \ + -H "Prefer: respond-async" \ + -H "Authorization: Bearer {bearer_token}" \ + -i +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="shell" can_copy=true %} diff --git a/dev/vendor/bcda-static-site/src/_pages/api-documentation/how-to-migrate-v3.md b/dev/vendor/bcda-static-site/src/_pages/api-documentation/how-to-migrate-v3.md new file mode 100644 index 0000000..2a112b5 --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_pages/api-documentation/how-to-migrate-v3.md @@ -0,0 +1,345 @@ +--- +layout: api-docs +page_title: "How to Migrate to v3" +seo_title: "How to Migrate to BCDA v3 | CMS Beneficiary Claims Data API" +description: "Learn about new endpoints, FHIR resource consolidation, and updated URLs and IDs." +in-page-nav: true +feedback_id: "8a4c0b7b" +--- + +# {{ page.page_title }} + +BCDA v3 became available July 1, 2026. Existing v1 and v2 users will need to migrate to v3 by July 30, 2027. + +Along with background information, this guide: + +- Introduces the [v3 Data Dictionary]({{ '/assets/downloads/BCDA_v3_Data_Dictionary.xlsx' | relative_url }}) +- Helps you reformat [requests](#requesting-data-from-new-v3-endpoints) and [URLs](#new-extension-and-code-system-urls) in v3 +- Introduces the [_typeFilter]({{ '/api-documentation/filter-claims-data.html#the-_typefilter-parameter' | relative_url }}) parameter +- Provides additional info on mapping and matching claims and variables between versions + +## Migration overview + +As you migrate from BCDA v1/v2 to v3, you’ll need to become familiar with changes including: + +- New endpoint URL structure +- Unified ExplanationOfBenefit resource for all claims +- Updated extension and code system URLs +- Different resource IDs requiring new matching strategies + +Please note that your initial v3 data request may take extra time. Additionally, as we continue to refine the v3 dataset, some fields will occasionally be re-loaded on our end. Corrected claims will show a more recent `last_updated` date and may result in larger than expected payloads. We will provide updates when these reloads are expected. + +## BCDA v3 overview + +BCDA v3 introduces access to more timely and accurate Medicare claims data and additional benefits to BCDA users including: + +- More consistent claims data by using a single data source, the CMS Integrated Data Repository. +- Improved alignment with sources such as the Claim and Claim Line Feed (CCLF) files through more standard claim and patient identifiers. + +### Changes to BCDA in v3 + +If you are currently using BCDA v1 or BCDA v2, there are changes to the API and FHIR Resources you need to be aware of in v3: + +- There is a new v3 endpoint: `https://api.bcda.cms.gov/api/v3/` +- Partially adjudicated claims1 will be represented as `ExplanationOfBenefit` (EOB) FHIR resources in v3, instead of `Claim`/`ClaimResponse` in v2 +- There will be new Extension and Code System URLs to reference +- Resource IDs are different between versions + +

1 Medical claims that have been submitted but not fully processed and paid by Medicare.

+ +### v3 Data Dictionary + +Download the BCDA v3 Data Dictionary to learn about: + +- updated information on resource type and claim field names +- updated mappings between CCLF and BCDA data +- new data available in v3 + +## Requesting data from new v3 endpoints + +### Point your app to v3 instead of v2 + +BCDA v3 will be located at the same domain as v1 and v2. However, it will have new endpoints for metadata requests and the Group and Patient $export operations. BCDA v3 will still support the [Bulk Data FHIR Implementation Guide](https://hl7.org/fhir/uv/bulkdata/STU2/). If you are currently connected to BCDA v1 or v2 endpoints, you can migrate to BCDA v3 by making the following changes. + +Point your app to **v3** instead of **v2** to connect to the new endpoint, complete the steps to export the data, check the job status, read the JSON manifest, and download the files. + +- v2 Patient $export operation:  + {% include copy_snippet.html code="GET /api/v2/Group/all/$export" language="shell" %} + +- v3 Patient $export operation:  + {% include copy_snippet.html code="GET /api/v3/Group/all/$export" language="shell" %} + +The /auth and /data URLs will remain the same between versions. You won't need to make any changes to how you request a token or download files. + +## Changes to claims representation + +In BCDA v3, claims data are still returned in FHIR R4 format, but there are changes to the Patient, Coverage, and ExplanationOfBenefit resources. These changes include: + - Conformity with [CARIN Blue Button Implementation Guide (v2.2.0)](https://hl7.org/fhir/us/carin-bb/STU2.2/en/) + - New StructureDefenition and CodeSystem URLs for CMS-specific extensions and terminologies + - New data elements + +Refer to the [v3 Data Dictionary]({{ '/api-documentation/how-to-migrate-v3.html#v3-data-dictionary' | relative_url }}) for a list of v3 supported data elements. + +### Changes to partially adjudicated claims + +BCDA v2 represents fully adjudicated claims by the ExplanationOfBenefit FHIR resource and partially adjudicated claims by the Claim and ClaimResponse resources. + +BCDA v3 represents all claims (fully and partially adjudicated) with the ExplanationOfBenefit resource. + + - v2 - `Claim`/`ClaimResponse` resource + - v3 - `ExplanationOfBenefit` resource + +This change in v3 lets you compare claims at different stages in the adjudication process without mapping between resource types. + +### Exporting claims based on adjudication status with v3 + +If you make requests for `Claim` and/or `ClaimResponse` resources using the `_type` parameter in v2, you'll need to update those requests in v3. + +#### How it works in v2 + +In BCDA v2 you exported claims based on adjudication status using the `_type` parameter, and passing the appropriate resource types (`Claim`, `ClaimResponse` for partially adjudicated claims and `ExplanationOfBenefit` for fully adjudicated claims) as the parameter value: + +- v2 $export for only partially adjudicated claims: + {% include copy_snippet.html code="GET /api/v2/Patient/$export?_type=Claim,ClaimResponse" language="shell" %} + +- v2 $export for only fully adjudicated claims: + {% include copy_snippet.html code="GET /api/v2/Patient/$export?_type=ExplanationOfBenefit" language="shell" %} + +In v2, BCDA differentiates "partially adjudicated" from "fully adjudicated" claims based on the data source rather than the claim status. BCDA's partially adjudicated data (Claim and ClaimResponse resources) are populated with data from the Medicare Shared Systems which often includes data for claims that had been fully processed and paid. + +#### How it works in v3 + +We've extended the API with the [`_typeFilter` parameter]({{ '/api-documentation/filter-claims-data.html#the-_typefilter-parameter' | relative_url }}) to filter export data more granularly. Because all claims in v3 are represented by the same resource type (`ExplanationOfBenefit`), use this parameter to specify the System-Type _tag, recreating your v2 filtering logic. + +Remember when using the _typeFilter parameter: +1. The _typeFilter parameter value must be URL-encoded +2. The _tag subquery parameter requires a token in the form `system|code` + +{% capture sampleRequest %}{% raw %} +GET /api/v3/Patient/$export + ?_type=ExplanationOfBenefit + &_typeFilter=ExplanationOfBenefit%3F_tag%3Dhttps%3A%2F%2Fbluebutton.cms.gov%2Ffhir%2FCodeSystem%2FSystem-Type%7CSharedSystem +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=sampleRequest language="shell" %} + +**If you received `ExplanationOfBenefit`, `Claim`, and `ClaimResponse` resources in v2,** you received all claims. + +In v3, specify all System-Type codes: +- SharedSystem +- NationalClaimsHistory +- DDPS + +{% capture sampleRequest %}{% raw %} +GET /api/v3/Patient/$export + ?_type=ExplanationOfBenefit + &_typeFilter=ExplanationOfBenefit%3F_tag%3Dhttps%3A%2F%2Fbluebutton.cms.gov%2Ffhir%2FCodeSystem%2FSystem-Type%7CSharedSystem%2Chttps%3A%2F%2Fbluebutton.cms.gov%2Ffhir%2FCodeSystem%2FSystem-Type%7CNationalClaimsHistory%2Chttps%3A%2F%2Fbluebutton.cms.gov%2Ffhir%2FCodeSystem%2FSystem-Type%7CDDPS +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=sampleRequest language="shell" %} + +**If you used the _type parameter for `_type=Claim,ClaimResponse` in v2 requests,** you received SharedSystem claims only. + +In v3, specify the SharedSystem System-Type code: +{% capture sampleRequest %}{% raw %} +GET /api/v3/Patient/$export + ?_type=ExplanationOfBenefit + &_typeFilter=ExplanationOfBenefit%3F_tag%3Dhttps%3A%2F%2Fbluebutton.cms.gov%2Ffhir%2FCodeSystem%2FSystem-Type%7CSharedSystem +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=sampleRequest language="shell" %} + +**If you used the _type parameter for `_type=ExplanationOfBenefit` for v2 requests,** you received NationalClaimsHistory claims and DDPS claims. + +In v3, specify the NationalClaimsHistory and DDPS System-Type codes: +{% capture sampleRequest %}{% raw %} +GET /api/v3/Patient/$export + ?_type=ExplanationOfBenefit + &_typeFilter=ExplanationOfBenefit%3F_tag%3Dhttps%3A%2F%2Fbluebutton.cms.gov%2Ffhir%2FCodeSystem%2FSystem-Type%7CNationalClaimsHistory%2Chttps%3A%2F%2Fbluebutton.cms.gov%2Ffhir%2FCodeSystem%2FSystem-Type%7CDDPS +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=sampleRequest language="shell" %} + +##### Omitting _typeFilter + +In v3, if you make a request without using the `_typeFilter` parameter to filter by System-Type, BCDA will return only NationalClaimsHistory and DDPS claims. + +{% capture sampleRequest %}{% raw %} +GET /api/v3/Patient/$export?_type=ExplanationOfBenefit +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=sampleRequest language="shell" %} + +## New extension and code system URLs + +BCDA v3 uses the StructureDefinition and CodeSystem FHIR resources. This is the standard way to represent code system URLs. It makes it easier to access the metadata. You can access the metadata for each Extension and CodeSystem by hitting the URL. This will return a StructureDefinition or CodeSystem FHIR resource. + +### URLs changing between v2 and v3 + +Make the following changes to migrate to v3 if you’ve mapped any bluebutton extension or code system URLs: + +#### Changes to Extension URLs + + + + + + + + + + + + + +
Changes to Extension URLs between v2 and v3
v2 URL + {% include copy_snippet.html code="https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd/" language="ruby" can_copy=true %} +
v3 URL + {% include copy_snippet.html code="https://bluebutton.cms.gov/fhir/StructureDefinition/CLM-NRLN-RIC-CD" language="ruby" can_copy=true %} +
+ +#### Changes to CodeSystem URLs + + + + + + + + + + + + + +
Changes to CodeSystem URLs between v2 and v3
v2 URL + {% include copy_snippet.html code="https://bluebutton.cms.gov/resources/variables/nch_near_line_rec_ident_cd/" language="ruby" can_copy=true %} +
v3 URL + {% include copy_snippet.html code="https://bluebutton.cms.gov/fhir/CodeSystem/CLM-NRLN-RIC-CD" language="ruby" can_copy=true %} +
+ +If your BCDA client is using any of the v2 URLs, you’ll need to update your code to look for the v3 version. + +## Different resource IDs between versions + +
+
+

Do not use FHIR IDs to match resources between versions.

+

To match beneficiaries, use MBI and demographics data. To match claims, use the claim control number.

+
+
+ +In BCDA v3, the FHIR ID (`Patient.id`, `ExplanationOfBenefit.id`, `Coverage.id`) for a given resource will not match the v2 FHIR resource ID. + +**Example:** + +If you receive the `Patient` resource from BCDA v3, do not look up an existing patient by expecting the v2 `Patient.id` to equal the v3 `Patient.id` value. These values will not match. They are a different set of identifiers. However, the range of v2 and v3 Patient IDs does overlap, so there are some IDs that represent a v2 Patient and v3 Patient. This would not mean they represent the same beneficiary in real life.   + +Instead of matching on `Patient.id`, use the relevant CMS identifier (MBI), along with additional supporting resource info (patient demographics like name, date of birth, etc.). Similarly, `ExplanationOfBenefit.id` and Coverage.id will not match between v2 and v3.  + +### Matching beneficiaries between v2 and v3 + +Because the v2 and v3 Patient IDs will not match, in order to match a v3 patient to a patient you already have in your database, the best way is to use the MBI and demographics. The MBI and demographics should match between versions of BCDA.  + +To find the MBI in the FHIR Patient resource, look at the `Patient.identifier` element for an identifier where the `identifier.system` is `http://hl7.org/fhir/sid/us-mbi`. The `identifier.value` will be the MBI. + +Example v3 `Patient.identifier` element: + +{% capture curlSnippet %}{% raw %} +{ + "identifier": [ + { + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "MB" + } + ] + }, + "system": "http://hl7.org/fhir/sid/us-mbi", + "value": "1S00E00AA08", + "period": { + "start": "2018-01-24T00:00:00+00:00" + } + } + ] +} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="json" can_copy=true %} + +It is always good practice to have some kind of validation based on patient demographics, like supplementing the MBI exact match with some subset of name, date of birth, and gender matching. + +Once you have matched an existing `Patient`, or created a new one, it is ok to store the BCDA v3 `Patient.id`. It is possible for two different MBIs to reference the same beneficiary, and therefore the same BCDA v3 `Patient`. If that happens, you will see multiple instances of `Patient.identifier` where the system is `http://hl7.org/fhir/sid/us-mbi`. This is rare, but you should be prepared for it to happen. + +Example of v3 `Patient.identifier` with two MBIs: + +{% capture curlSnippet %}{% raw %} +{ + "identifier": [ + { + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "MB" + } + ] + }, + "system": "http://hl7.org/fhir/sid/us-mbi", + "value": "1S00E00AA08", + "period": { + "start": "2022-04-05T00:00:00+00:00", + "end": "2022-04-12T00:00:00+00:00" + } + }, + { + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "MB" + } + ] + }, + "system": "http://hl7.org/fhir/sid/us-mbi", + "value": "1S00E00AE22", + "period": { + "start": "2022-04-12T00:00:00+00:00" + } + } + ] +} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="json" can_copy=true %} + +### Matching claims between v2 and v3 + +Because the v3 `ExplanationOfBenefit` FHIR ID will not match the v2 `ExplanationOfBenefit` or `Claim`/`ClaimResponse` FHIR IDs, in order to match a v3 claim to a claim you already have in your database, the best way is to use the claim control number. + +Because each version of a claim will have a unique claim ID, you should avoid using the unique claim ID to track a claim across versions, or even within a version across the adjudication journey. Instead, use the claim control number. It is the identifier where the `identifier.system` equals `https://bluebutton.cms.gov/identifiers/CLM-CNTL-NUM`. + +Example `ExplanationOfBenefit.identifier` element: + +{% capture curlSnippet %}{% raw %} +{ + "identifier": [ + { + "type": { + "coding": [ + { + "system": "http://hl7.org/fhir/us/carin-bb/CodeSystem/C4BBIdentifierType", + "code": "uc", + "display": "Unique Claim ID" + } + ] + }, + "value": "-9422919134190" + }, + { + "system": "https://bluebutton.cms.gov/identifiers/CLM-CNTL-NUM", + "value": "05968201612271KJS" + } + ] +} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="json" can_copy=true %} + +FHIRPath for claim identifier: + +{% include copy_snippet.html code="ExplanationOfBenefit.identifier.where(system = 'https://bluebutton.cms.gov/identifiers/CLM-CNTL-NUM').value" language="ruby" can_copy=true %} diff --git a/dev/vendor/bcda-static-site/src/_pages/bcda-data/comparison-bcda-cclf-files.md b/dev/vendor/bcda-static-site/src/_pages/bcda-data/comparison-bcda-cclf-files.md new file mode 100644 index 0000000..c9ab934 --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_pages/bcda-data/comparison-bcda-cclf-files.md @@ -0,0 +1,218 @@ +--- +layout: api-docs +page_title: "Comparison of BCDA v3 and CCLF Files" +seo_title: "" +description: "" +in-page-nav: true +tabs: + - label: v3 + url: /bcda-data/comparison-bcda-cclf-files.html + - label: v2 + url: /v2/bcda-data/comparison-bcda-cclf-files.html +--- + +# {{ page.page_title }} + +{% include tabs.html items=page.tabs %} + +Beneficiary Claims Data API (BCDA) and [Claim and Claim Line Feed (CCLF)](https://www.cms.gov/files/document/cclf-information-packet.pdf) files both offer Medicare Parts A, B, and D claims data. While the data sources overlap, there are some differences including their formatting and the frequency with which they make data available. + +## Overview of similarities and differences + +BCDA and CCLF files differ in how and when Medicare claims data is made available. + +### BCDA: frequent access and standardization + +As an API, BCDA lets you automate data requests and streamline workflows for more efficient, secure processes. The API uses Fast Healthcare Interoperability Resources ([FHIR](https://www.hl7.org/fhir/)) standards. It also supports frequent and early access to Medicare enrollees' claims data. This includes access to claims that would otherwise be unavailable during processing ("adjudication"). Earlier access to claims helps your organization build near real-time workflows and coordinate care for your patients. + +FHIR breaks claims data into discrete blocks of recognizable data called `Resources`. These use information categories such as Patient, Explanation of Benefit, and Coverage to organize files, which make them easy to share and interpret. BCDA uses the [CMS recommended Bulk FHIR Standards](https://www.cms.gov/priorities/key-initiatives/burden-reduction/interoperability/implementation-guides-and-standards/standards-and-igs-index-and-resources) to export larger amounts of data across systems and populations. + +### CCLF: ease of access and direct delivery + +CCLF files provide adjudicated claims data delivered on a regular cadence, which may be useful for organizations with established file-based workflows or analyses that depend on fields unavailable in BCDA. You can download CCLF files directly from your model-specific portal. + +## Who can use BCDA and CCLF files? + +BCDA and CCLF claims data are available to organizations participating in some [Alternative Payment Models](https://www.cms.gov/priorities/innovation/key-concepts/alternative-payment-models-apms). These are programs that reward health care providers for high-quality, coordinated care. BCDA and CCLF files are currently available to organizations in these models: + +- Medicare Shared Savings Program +- Realizing Equity, Access, and Community Health (ACO REACH) +- Kidney Care Choices (KCC) +- Increasing Organ Transplant Access (IOTA) + +## Differences in access, format, content, and timing + +{% assign unique = 1 %} + + + + + + + + + + + + + + + + +
BCDACCLF
Access method
Programmatically using API requestsManually using portals or programmatically using API requests or Command Line Interface
+ +{% assign unique = 2 %} + + + + + + + + + + + + + + + + +
BCDACCLF
Claims data format
Interoperable and machine-readable, using FHIR R4.Human-readable, using fixed-width tabular files.
+ +{% assign unique = 3 %} + + + + + + + + + + + + + + + + +
BCDACCLF
Claims data source
National Claims History and CMS Shared Systems data from the Integrated Data Repository (IDR)National Claims History data from the Integrated Data Repository (IDR)
+ +{% assign unique = 4 %} + + + + + + + + + + + + + + + + +
BCDACCLF
Historical data provided on newly attributed enrollees
+
    +
  • Shared Savings Program: all historical data available as far back as 2014
    • +
  • ACO REACH: 36 months of historical data from the start of the current performance year
    • +
  • KCC: 24 months of historical data from the start of the current performance year
    • +
+ 		+
    +
  • Shared Savings Program: 36 months prior to agreement start date
    • +
  • ACO REACH: 36 months of historical data from the start of the current performance year
    • +
  • KCC: 36 months of historical data from the start of the current performance year
    • +
+
+ +{% assign unique = 5 %} + + + + + + + + + + + + + + + + +
BCDACCLF
Update frequency
+
    +
  • Adjudicated claims (Medicare Part A and Part B): weekly
    • +
  • Adjudicated claims (Medicare Part D): 6x/week
    • +
  • Partially adjudicated claims: 4 to 5x/week
    • +
+ 		+
    +
  • Adjudicated claims: monthly
    • +
  • Adjudicated claim extracts: weekly*
    • +
+
+ +
    +
  • CCLF Self-Service is only available to ACOs in the Medicare Shared Savings Program. Visit the ACO-MS Knowledge Library to learn more about CCLF Self-Service.
    • +
+ +## Which data source is right for me? + +Both sources support model entities' use cases, and can even be used together. When choosing between them, consider the frequency, source, and level of automation most suited to your needs. You can also consider your organization's existing healthcare analytics and actuarial tools. A benefit of BCDA is its conformance with FHIR standards. This saves time and money on integrations with EHRs and off-the-shelf FHIR data clients. + +BCDA aligns with CCLF partly by using shared claim identifiers. This alignment makes it easy to use both CCLF files and BCDA's standards-based format. + +### Can I use both CCLF and BCDA as sources of Medicare claims data? + +Yes. Using both data sources provides many benefits: + +- Ensure data accuracy: Compare and cross-reference both sources to access a wider variety of historical data and identify any discrepancies. +- Evaluate care coordination and payment processes: Combine BCDA's insurance coverage details with CCLF files' payment data. This can evaluate the coordination of benefits between Medicare and other insurers, assess the financial impact on all parties, and identify primary or secondary payers. + +## How soon will I receive claims data from BCDA v3 and CCLF files? + +Depending on the model your organization participates in, claims data can be available: + +- 2-4 days after the claim is **submitted to CMS** via BCDA v3 +- 1-7 days after the claim is **adjudicated by CMS** via CCLF + +The following diagram illustrates these differences while providing more detail about the claims process. + +
+ Timeline of when claim data becomes available to users through BCDA and CCLF after providers submit a claim. +
+
    +
  • Through BCDA: a partially adjudicated claim is available 2 to 4 days after submission and continues to update every 2 to 4 days while CMS is processing. Once the claim is fully adjudicated, it is available through BCDA within 1 to 7 days.
    • +
  • Through CCLF: the fully adjudicated claim is available via CCLF Self-Service 1 to 7 days after adjudication and via the CCLF monthly feed 2 to 6 weeks after full adjudication.
    • +
+
+
+ +## Which data fields in BCDA v3 are not available from CCLF files? + +The BCDA v3 Data Dictionary contains a comprehensive list of all data fields available in BCDA v3 and includes a mapping to the equivalent or related CCLF file and element. + +

For an exhaustive list of fields available only in BCDA v3, look for rows where the cclfMapping column is blank in the "Data Dictionary" worksheet in the BCDA v3 Data Dictionary {% include sprite.html icon="file_download" class="text-middle" size="2" %}.

+ +## Which data fields in CCLF files are not available from BCDA v3? + +The majority of data elements found in CCLF files are also available from BCDA v3. There are some elements not available from BCDA v3 including: + +- Legacy HICN identifiers +- Certain beneficiary demographic information +- CCLF metadata +- Data fields from inactive programs + +Review the "CCLF fields absent from BCDA v3" worksheet in the BCDA v3 Data Dictionary for a comprehensive list of fields found in CCLF files which are unavailable in BCDA v3. diff --git a/dev/vendor/bcda-static-site/src/_pages/bcda-data/partially-adjudicated-claims-data.md b/dev/vendor/bcda-static-site/src/_pages/bcda-data/partially-adjudicated-claims-data.md new file mode 100644 index 0000000..191e03f --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_pages/bcda-data/partially-adjudicated-claims-data.md @@ -0,0 +1,231 @@ +--- +layout: api-docs +page_title: "Partially Adjudicated Claims Data" +seo_title: "" +description: "Get earlier access to Medicare enrollee data with BCDA partially adjudicated claims, now available as ExplanationOfBenefit resources in v3." +in-page-nav: true +feedback_id: "355f6ea6" +tabs: + - label: v3 + url: /bcda-data/partially-adjudicated-claims-data.html + - label: v2 + url: /v2/bcda-data/partially-adjudicated-claims-data.html +--- + +# {{ page.page_title }} + +{% include tabs.html items=page.tabs %} + +

A faster way to access claims data

+ +Medicare claims processing (adjudication) can take weeks. Through partially adjudicated claims, ACO REACH participants (REACH ACOs) receive patient data only 2-4 days after providers submit claims to Medicare. This helps them provide patients with more effective interventions, transitions, and coordination of care. + +Fully adjudicated claims offer the same rich insights, with some small differences including a longer frequency. + +## Comparison: partially and fully adjudicated claims + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Partially versus fully adjudicated claims data
Partially adjudicated claimsFully adjudicated claims
+ Update daily + + Update weekly +
+ Available to all eligible model + entities, except GUIDE participants + + Available to all eligible model + entities +
+ Requires BCDA v2* or v3 + + Available in all BCDA versions +
+ Sourced from Medicare Shared Systems + + Sourced from National Claims History (NCH) +
+ +
    +
  • Access to partially adjudicated claims data using BCDA v2 is limited to REACH ACOs. Participants in other CMS Innovation Models and the Medicare Shared Savings Program must use BCDA v3 to access partially adjudicated claims data.
    • +
+ +## What's in partially adjudicated claims data? + +- In BCDA v3, partially adjudicated claims data: + - Are represented with 1 FHIR resource type: + - ExplanationOfBenefit (EOB) - Information about the professional and institutional claims that providers submit for payment and information about a claim's adjudication status and processing results + - Contain Part A and Part B data from the current performance year and historical claims data from the previous 2 or more performance years, depending on your organization’s Alternative Payment Model. +- In BCDA v2, partially adjudicated claims data: + - Are represented with 2 FHIR resource types: + - Claim - Information about the professional and institutional claims that providers submit for payment (including the services that enrollees receive) + - ClaimResponse - Information about a claim's adjudication status and processing results + - Contain Part A and Part B data (excluding Durable Medical Equipment) with updates from the past 60 days + +## What's in fully adjudicated claims data? + +In all versions of BCDA, adjudicated claims data: +- Are represented with 3 FHIR resource types: + - Patient - Enrollees' demographic details and updates to their patient identifiers. + - Coverage - Enrollees' insurance coverage details, including dual coverage. + - ExplanationOfBenefit - Details for episodes of care, including where and when the service was performed, diagnosis codes, provider, and cost of care. +- Contain Part A, Part B, and Part D data from the current performance year and historical claims data from the previous 2 or more performance years, depending on your organization's Alternative Payment Model + + +## How do I know when a claim has been fully processed? + + +All EOBs contain the ExplanationOfBenefit.outcome element which will indicate if the claim is still processing. If you wish to include only EOBs that have been fully processed and paid in your job requests, see our guidance on filtering claims by `ExplanationOfBenefit.outcome` using _typeFilter. + +### Matching multiple EOBs to the same claim + +In v3, claims are automatically deduplicated within a source system. You won't need to deduplicate multiple versions of a claim within the same System-Type. + +However, if you are receiving claims with the `SharedSystem` System-Type AND claims with the `NationalClaimsHistory` System-Type, then it is possible to receive two versions of the same claim as two different ExplanationOfBenefit resources. + +To identify EOBs that represent the same claim, use the `CLM-CNTL-NUM` identifier `ExplanationOfBenefit.identifier.where('system'='https://bluebutton.cms.gov/fhir/CodeSystem/CLM-CNTL-NUM').value` to identify the matching pair. If two EOBs have the same Claim Control Number, then they represent the same claim. + +Example identifier element with claim control number: +{% capture curlSnippet %}{% raw %} +"identifier": [ + { + "system": "https://bluebutton.cms.gov/identifiers/CLM-CNTL-NUM", + "value": "-01614121123648SZF" + } +] +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="json" can_copy=true %} + +There may be some instances in which a claim can't be de-duplicated via `CLM-CNTL-NUM`: + +- #### Changes in Claim Control Number + If a claim receives a new claim control number, a previous claim control number will be available in `ExplanationOfBenefit.related`, and can be used to de-duplicate claims even if the claim control number has changed. + +- #### PDE Claims + Because Medicare does not process Part D / Prescription claims directly, the rules to identify related claims are a bit different than Part A / Part B claims. + + Part D claims are submitted to CMS by the Part D plan sponsors. As a result, not all Part D claims have a claim control number. However, those that do have a claim control number can be deduplicated using it. + + For claims without a claim control number, you can use a natural key to link together related Part D Events for a given patient. The natural key logic is outlined in CCLF Information Packet under Section 5.1.2, and repeated here: + + - Line Item From Date (`CLM_LINE_FROM_DT`) + - Provider Service Identifier Qualifier Code (`PRVDR_SRVC_ID_QLFYR_CD`) + - Service Provider Generic ID Number (`CLM_SRVC_PRVDR_GNRC_ID_NUM`) + - RX Number (`CLM_LINE_RX_SRVC_RFRNC_NUM`) + - RX Fill Number (`CLM_LINE_RX_FILL_NUM`) + +- #### DME Claims + When a claim from VMS is processed and released to the NCH, the leading digit (a 1) is dropped from the `CLM-CNTL-NUM`, due to a design decision made in preparation for Y2K. + + + + + + + + + + + + + + +
Example
VMS Claim Control NumberNCH Claim Control Number
1222222222222222222222222222222
+ +### Tracking a Claim After it is Fully Processed + +After a claim is fully adjudicated and released to the National Claims History system, it is still possible for updates to be made to the claim. Similar to matching ExplanationOfBenefit resources from different sources in the same export, you may need logic to match a claim from a current export to an existing claim from a previous export. Most of the time, you can still rely on the claim control number or the parent/original claim control number in `ExplanationOfBenefit.related`. + +A couple of quirks to call out: + +- **Professional claims:** NCH professional claims will never populate a new claim control number. Each new version of the claim will appear to have the original claim control number. +- **Institutional claims:** If an NCH institutional claim receives multiple rounds of claim control numbers, only the parent (the immediately-preceding claim control number) will be populated in `ExplanationOfBenefit.related`. + +## Get started with partially adjudicated claims data + +Access partially adjudicated claims data in the BCDA Sandbox. + +Review the Data Dictionary {% include sprite.html icon="file_download" class="text-middle"%} for a complete list and description of the data fields. + +## Use cases + +### Example 1: Improve transition of care + +

REACH ACOs can check if post-discharge processes are in place for + an attributed enrollee shortly after a hospital discharge. Benefits include reduced readmissions and healthcare costs. +

+ +On May 1, 2022, Mrs. Gonzales is doing better after a bout of pneumonia and is released home from the hospital. The hospital submits a claim for her stay to Medicare on May 3, 2022. + +1. BCDA receives partially adjudicated claims data that reflects this update. This is identified by a Document Control Number (DCN) within a few days. +2. A REACH ACO sets up a dashboard to track newly identified discharges. After receiving an automated care-coordination alert for Mrs. Gonzales, they're able to follow up on her post-discharge care in a timely manner. +3. Using supplemental data, the REACH ACO adds filters to the dashboard for parameters of interest like whether the discharging hospital is in-network. +4. BCDA will reflect any status changes to the claim, for example if Mrs. Gonzales' claim is returned to the provider or adjusted. +5. These changes may involve updates to key variables used for care coordination, like diagnosis codes. The claims data will indicate the final status once a claim finishes adjudication and is submitted for payment. +6. BCDA's adjudicated data will pick up this claim after several additional processing steps. This is typically up to 14 days after adjudication. + +REACH ACOs can also flag and identify patients who have a high risk of readmission based on factors like past diagnosis. Early access to claims data can help reduce readmissions and healthcare costs. + +### Example 2: Identify opportunities for intervention + +

REACH ACOs can monitor outpatient events for follow-up care and + track patterns that often indicate future utilization. The data can be used to target case management and deliver + clinically appropriate follow-up care. Additionally it can help reduce the volume of unnecessary procedures.

+ +On September 6, 2022, Mr. Fritz underwent a duplex scan to evaluate for carotid artery stenosis. The cardiology clinic submits a claim for this procedure to Medicare on September 8, 2022. + +1. BCDA receives partially adjudicated claims data that reflects this update. This is identified by a Claim Control Number in a few days. +2. A REACH ACO creates a dashboard to track procedures that merit review. Mr. Fritz's service is flagged because it signals the surgeon may be planning a carotid endarterectomy procedure. +3. However, if Mr. Fritz does not have neurological symptoms, this procedure may not be a recommended course of action. The REACH ACO is able to activate protocols to alert Mr. Fritz's primary care provider for intervention. +4. As with example 1, BCDA will reflect any status changes to the claim. +5. Similarly, BCDA's adjudicated data will pick up this claim. + +REACH ACOs can also track performance on cases where the care may be of limited value. For example, if Mr. Fritz does not have neurologic symptoms, then this kind of scan has been indicated to be more harmful than beneficial. This information could be used by the REACH ACO to reduce the volume of unnecessary procedures. + +### Example 3: Enhance care coordination + +

REACH ACOs can learn more about their patients' health histories, build their clinical profiles, and improve care coordination.

+ +Ms. Thompson began treatment for breast cancer on March 8, 2023. She is experiencing nausea and dehydration as a result of chemotherapy. While visiting family on March 14, she drove to an emergency room (ER) 50 miles from home. She was treated with IV fluids and nausea medication. + +1. BCDA receives partially adjudicated claims data that reflects this update. This is identified by a Claim Control Number in a few days. +2. This emergency room visit may provide useful insights into Ms. Thompson's health history and chemotherapy journey. If Ms. Thompson is experiencing symptoms, her plan of treatment may need to be adjusted moving forward. +3. REACH ACOs can track procedures and services, including those from a different system, to get a holistic view of patients' health histories. They can then follow up with the patient and/or provider to improve care coordination. + +REACH ACOs can also monitor the rate of emergency services used by patients undergoing active cancer treatment and following transplantation. + +In the past, it would take weeks to know whether patients received care outside the network. Partially adjudicated claims data allows REACH ACOs to more quickly identify and investigate out-of-network care. + +## Additional resources + +- Services covered by Medicare Parts A and B +- Original Medicare (OM) Claims Processing (requires 4i access) +- Reporting and Data Sharing Overview (requires 4i access) diff --git a/dev/vendor/bcda-static-site/src/_pages/support/faqs.md b/dev/vendor/bcda-static-site/src/_pages/support/faqs.md new file mode 100644 index 0000000..852d644 --- /dev/null +++ b/dev/vendor/bcda-static-site/src/_pages/support/faqs.md @@ -0,0 +1,156 @@ +--- +layout: support +page_title: "Frequently Asked Questions" +seo_title: "" +description: "Frequently asked questions about BCDA." +in-page-nav: true +--- + +# {{ page.page_title }} + +## About the API + +### Who is eligible to use Beneficiary Claims Data API (BCDA)? + +BCDA supports organizations (entities) participating in the following CMS [alternative payment models](https://www.cms.gov/priorities/innovation/about/alternative-payment-models){:target="_blank" rel="noopener"}: + +- [ACCESS (Advancing Chronic Care with Effective, Scalable Solutions)](https://www.cms.gov/priorities/innovation/innovation-models/access){:target="_blank" rel="noopener"} +- [ACO REACH (Accountable Care Organization Realizing Equity, Access, and Community Health)](https://www.cms.gov/priorities/innovation/innovation-models/aco-reach){:target="_blank" rel="noopener"} +- [GUIDE (Guiding an Improved Dementia Experience)](https://www.cms.gov/priorities/innovation/innovation-models/guide){:target="_blank" rel="noopener"} +- [IOTA (Increasing Organ Transplant Access)](https://www.cms.gov/priorities/innovation/innovation-models/iota){:target="_blank" rel="noopener"} +- [KCC (Kidney Care Choices)](https://www.cms.gov/priorities/innovation/innovation-models/kidney-care-choices-kcc-model){:target="_blank" rel="noopener"} +- [SSP (Shared Savings Program)](https://www.cms.gov/medicare/payment/fee-for-service-providers/shared-savings-program-ssp-acos){:target="_blank" rel="noopener"} + +### When do terminated and discontinued model entities lose API access? + +Terminated and discontinued organizations lose access to the API, including runouts data, the same day their participation in the model ends. + +### What's the difference between BCDA and CCLF files? + +CCLF files are automatically available monthly using 12 flat files, and can be downloaded weekly upon request. + +BCDA updates adjudicated claims weekly and partially adjudicated claims 4x/week using 3 NDJSON files (Coverage, Patient, ExplanationOfBenefit). + +Additionally, BCDA is an API that uses the [Bulk Fast Healthcare Interoperability Resources (FHIR)](https://hl7.org/fhir/uv/bulkdata/){:target="_blank" rel="noopener noreferrer"} format, as required by CMS. [Learn more about the differences]({{ '/bcda-data/comparison-bcda-cclf-files.html' | relative_url }}). + +### What's the difference between BCDA v2 and v3? + +BCDA v3 introduces a number of improvements, including: + +- More frequent and timely updates +- Easier claims tracking +- Enhanced filtering capabilities +- Simplified, reliable data mapping capabilities +- Improved conformance with select FHIR Implementation Guides +- Simplified linking between partially and fully adjudicated claims +- Uses consistent claim identifiers across all phases of adjudication + +We’ve also corrected issues some users encountered with v1 and v2 such as: + +- Mismatched data between BCDA resources and CCLF files +- Missing data for newly attributed enrollees +- Issues for enrollees assigned more than one BENE_ID + +You can learn more about v3 improvements and problems solved at [Introducing BCDA v3]({{ '/about/introducing-v3.html' | relative_url }}). + +## Compliance and restrictions +{: .margin-bottom-2} + +### How do I redact PHI and PII when sharing information? + +Completely cover or label [Personally Identifiable Information (PII)](https://www.hhs.gov/answers/hhs-administrative/what-is-pii/index.html) and [Protected Health Information (PHI)](https://www.hhs.gov/answers/hipaa/what-is-phi/index.html) as “REDACTED” in emails, screenshots, and documents. Ensure any masks are 100% opaque and in a format that supports layers. + +Examples of PII and PHI: + +- Medicare Beneficiary Identifier (MBI) +- Taxpayer Identification Number (TIN) +- National Provider Identifier (NPI) +- Social Security Number (SSN) +- API keys or access credentials (e.g., client ID and secret) +- authorization or bearer tokens + +PII and PHI are often found in API requests or response payloads referenced in: + +- the text of your Google Group posts or emails +- files attached to your Google Group posts or emails (e.g., XMLs, JSONs) +- screenshots attached to your Google Group posts or emails + +Example of a redacted response: + +{% capture curlSnippet %}{% raw %} +{ + "taxpayerIdentificationNumber": "REDACTED", + "nationalProviderIdentifier": "REDACTED" +} +{% endraw %}{% endcapture %} +{% include copy_snippet.html code=curlSnippet language="json" %} + +### Can I use 3rd party web-based REST clients or tools? + +Use caution with 3rd party web-based clients. By sharing your credentials, you may be allowing them to make REST calls on your behalf. This is a serious data privacy and security risk. We recommend using secure REST client tools. + +## Claims processing timeline + +### How long does it take BCDA to receive a claim after it is submitted? + +It typically takes 2-4 days after submission for BCDA to receive [partially adjudicated claims data]({{ '/bcda-data/partially-adjudicated-claims-data.html' | relative_url }}) and up to 7 days for adjudicated claims data. Even after adjudication, claims may go through additional processing. BCDA will continue to provide the latest updates available for each claim. + +According to Section 6404 of the Affordable Care Act, Original Medicare claims must be submitted within 12 months (1 calendar year) of the date of service. + +### Where does the data come from? + +BCDA v3 sources data from CMS's Integrated Data Repository (IDR). + +For earlier versions of BCDA, adjudicated claims data is loaded from the [Chronic Conditions Data Warehouse (CCW)](https://www2.ccwdata.org/web/guest/home/){:target="_blank" rel="noopener noreferrer"}. Partially adjudicated claims data is loaded from the Fiscal Intermediary Standard System (FISS) and Multi-Carrier System (MCS). + +### How often is data refreshed? + +The following table shows how frequently Medicare claims data is refreshed and potentially available to eligible organizations. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FHIR ResourceBCDA v3
Patient6x/week, Sunday to Friday
Coverage6x/week, Sunday to Friday
ExplanationOfBenefit - Part A/B (National Claims History)Weekly, on Monday1
ExplanationOfBenefit - Part D5x/week, Sunday to Thursday
ExplanationOfBenefit - Part A/B (Shared Systems)4x/week, Sunday and Tuesday to Thursday
+ +1 Data is loaded on the Monday after it's loaded into the source system, 0-7 days from processing or "adjudication." +{: #refresh-fn1 .font-body-2xs style="scroll-margin-top: 6.25rem;"} + +To stay up to date with the latest claims data for your enrollees, we recommend exporting data from National Claims History once per week, and daily for all other sources. Use the [_since parameter]({{ '/api-documentation/filter-claims-data.html#the-_since-parameter' | relative_url }}) when running jobs to avoid downloading duplicate data. + +## Troubleshooting + +### Why do I get connection timeout issues with my production credentials? + +BCDA requires that all production requests come from a registered IP address. Make sure the IP addresses you're using to request data have been added to the Allow List in your model-specific system. Visit [Production Access]({{ '/production-access.html' | relative_url }}) for more details. + +### Why am I getting a 429 error response? + +A status code of 429 indicates “Too Many Requests.” Wait until the period of time specified in the header has passed before making more requests. + +This makes sure your client can adapt without manual intervention, even if the rate-limiting parameters change. [Learn more about the 429 status code.]({{ '/api-documentation/access-claims-data.html' | relative_url }}#response-example-too-many-requests) diff --git a/docs/docs/cli/api-serve.md b/docs/docs/cli/api-serve.md index 9981b5f..deb7179 100644 --- a/docs/docs/cli/api-serve.md +++ b/docs/docs/cli/api-serve.md @@ -1,6 +1,6 @@ --- title: stack api serve -sidebar_position: 79 +sidebar_position: 80 --- # `stack api serve` diff --git a/docs/docs/cli/api.md b/docs/docs/cli/api.md index a86800d..c2de7f9 100644 --- a/docs/docs/cli/api.md +++ b/docs/docs/cli/api.md @@ -1,6 +1,6 @@ --- title: stack api -sidebar_position: 78 +sidebar_position: 79 --- # `stack api` diff --git a/docs/docs/cli/bcda-errors.md b/docs/docs/cli/bcda-errors.md new file mode 100644 index 0000000..105f81d --- /dev/null +++ b/docs/docs/cli/bcda-errors.md @@ -0,0 +1,20 @@ +--- +title: stack bcda errors +sidebar_position: 19 +--- + +# `stack bcda errors` + +``` +Usage: stack bcda errors [OPTIONS] KEY + + OperationOutcome issues from a job's error files (a partial export). + +╭─ Arguments ──────────────────────────────────────────────────────────────────╮ +│ * key TEXT Store job key. [required] │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --limit INTEGER Issues to print. [default: 50] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +``` diff --git a/docs/docs/cli/bcda-export.md b/docs/docs/cli/bcda-export.md index 5b2bd87..769f973 100644 --- a/docs/docs/cli/bcda-export.md +++ b/docs/docs/cli/bcda-export.md @@ -28,9 +28,13 @@ Usage: stack bcda export [OPTIONS] │ --system-type TEXT _typeFilter System-Type │ │ tag(s). │ │ --outcome TEXT _typeFilter outcome: partial │ -│ or complete. │ -│ --service-date-from TEXT _typeFilter service-date > │ -│ YYYY-MM-DD. │ +│ and/or complete. │ +│ --service-date-from TEXT _typeFilter earliest │ +│ service-date (YYYY[-MM[-DD]], │ +│ optional gt/ge/eq prefix). │ +│ --service-date-to TEXT _typeFilter latest │ +│ service-date (optional lt/le │ +│ prefix). │ │ --per-type --one-job One job per type (survives │ │ sandbox sweeps). │ │ [default: per-type] │ diff --git a/docs/docs/cli/bcda-load.md b/docs/docs/cli/bcda-load.md index 64f0e9c..37d5f06 100644 --- a/docs/docs/cli/bcda-load.md +++ b/docs/docs/cli/bcda-load.md @@ -1,6 +1,6 @@ --- title: stack bcda load -sidebar_position: 19 +sidebar_position: 20 --- # `stack bcda load` diff --git a/docs/docs/cli/bcda-sync.md b/docs/docs/cli/bcda-sync.md index ab613df..377ae33 100644 --- a/docs/docs/cli/bcda-sync.md +++ b/docs/docs/cli/bcda-sync.md @@ -1,6 +1,6 @@ --- title: stack bcda sync -sidebar_position: 21 +sidebar_position: 22 --- # `stack bcda sync` diff --git a/docs/docs/cli/bcda-vendor.md b/docs/docs/cli/bcda-vendor.md index a3fb754..d6312fd 100644 --- a/docs/docs/cli/bcda-vendor.md +++ b/docs/docs/cli/bcda-vendor.md @@ -1,6 +1,6 @@ --- title: stack bcda vendor -sidebar_position: 20 +sidebar_position: 21 --- # `stack bcda vendor` diff --git a/docs/docs/cli/bcda.md b/docs/docs/cli/bcda.md index 6c6178c..a73ae1b 100644 --- a/docs/docs/cli/bcda.md +++ b/docs/docs/cli/bcda.md @@ -25,6 +25,8 @@ Usage: stack bcda [OPTIONS] COMMAND [ARGS]... │ manifest. │ │ cancel Cancel an in-progress job. │ │ store Local store: jobs, files, bytes. │ +│ errors OperationOutcome issues from a job's error files (a partial │ +│ export). │ │ load Reconcile the store (or flatten the newest export per type), │ │ load, run the connector, publish the replica. │ │ vendor Vendored upstream (CMSgov/bcda-app, bcda.cms.gov): manifest, │ diff --git a/docs/docs/cli/bib-attach-iom.md b/docs/docs/cli/bib-attach-iom.md index 478fbf1..7742dcc 100644 --- a/docs/docs/cli/bib-attach-iom.md +++ b/docs/docs/cli/bib-attach-iom.md @@ -1,6 +1,6 @@ --- title: stack bib attach-iom -sidebar_position: 34 +sidebar_position: 35 --- # `stack bib attach-iom` diff --git a/docs/docs/cli/bib-attach-oig.md b/docs/docs/cli/bib-attach-oig.md index 1fdb4f5..9d56ef4 100644 --- a/docs/docs/cli/bib-attach-oig.md +++ b/docs/docs/cli/bib-attach-oig.md @@ -1,6 +1,6 @@ --- title: stack bib attach-oig -sidebar_position: 39 +sidebar_position: 40 --- # `stack bib attach-oig` diff --git a/docs/docs/cli/bib-backfill-comments.md b/docs/docs/cli/bib-backfill-comments.md index 4dd1f9e..d80ebf5 100644 --- a/docs/docs/cli/bib-backfill-comments.md +++ b/docs/docs/cli/bib-backfill-comments.md @@ -1,6 +1,6 @@ --- title: stack bib backfill-comments -sidebar_position: 32 +sidebar_position: 33 --- # `stack bib backfill-comments` diff --git a/docs/docs/cli/bib-cfr-jump.md b/docs/docs/cli/bib-cfr-jump.md index a1d239c..87edf68 100644 --- a/docs/docs/cli/bib-cfr-jump.md +++ b/docs/docs/cli/bib-cfr-jump.md @@ -1,6 +1,6 @@ --- title: stack bib cfr-jump -sidebar_position: 44 +sidebar_position: 45 --- # `stack bib cfr-jump` diff --git a/docs/docs/cli/bib-code-tags.md b/docs/docs/cli/bib-code-tags.md index a455b95..eccf715 100644 --- a/docs/docs/cli/bib-code-tags.md +++ b/docs/docs/cli/bib-code-tags.md @@ -1,6 +1,6 @@ --- title: stack bib code-tags -sidebar_position: 45 +sidebar_position: 46 --- # `stack bib code-tags` diff --git a/docs/docs/cli/bib-discover-pfs-rules.md b/docs/docs/cli/bib-discover-pfs-rules.md index 37b98e0..00d443f 100644 --- a/docs/docs/cli/bib-discover-pfs-rules.md +++ b/docs/docs/cli/bib-discover-pfs-rules.md @@ -1,6 +1,6 @@ --- title: stack bib discover-pfs-rules -sidebar_position: 28 +sidebar_position: 29 --- # `stack bib discover-pfs-rules` diff --git a/docs/docs/cli/bib-fetch-docket-comments.md b/docs/docs/cli/bib-fetch-docket-comments.md index a3ef0f5..0903bad 100644 --- a/docs/docs/cli/bib-fetch-docket-comments.md +++ b/docs/docs/cli/bib-fetch-docket-comments.md @@ -1,6 +1,6 @@ --- title: stack bib fetch-docket-comments -sidebar_position: 29 +sidebar_position: 30 --- # `stack bib fetch-docket-comments` diff --git a/docs/docs/cli/bib-fetch-pfs-comments.md b/docs/docs/cli/bib-fetch-pfs-comments.md index 45cf2b8..62cd3f8 100644 --- a/docs/docs/cli/bib-fetch-pfs-comments.md +++ b/docs/docs/cli/bib-fetch-pfs-comments.md @@ -1,6 +1,6 @@ --- title: stack bib fetch-pfs-comments -sidebar_position: 30 +sidebar_position: 31 --- # `stack bib fetch-pfs-comments` diff --git a/docs/docs/cli/bib-fr-grab.md b/docs/docs/cli/bib-fr-grab.md index b167aa2..2ca1534 100644 --- a/docs/docs/cli/bib-fr-grab.md +++ b/docs/docs/cli/bib-fr-grab.md @@ -1,6 +1,6 @@ --- title: stack bib fr-grab -sidebar_position: 42 +sidebar_position: 43 --- # `stack bib fr-grab` diff --git a/docs/docs/cli/bib-fr-jump.md b/docs/docs/cli/bib-fr-jump.md index d3b13d6..8dafaa9 100644 --- a/docs/docs/cli/bib-fr-jump.md +++ b/docs/docs/cli/bib-fr-jump.md @@ -1,6 +1,6 @@ --- title: stack bib fr-jump -sidebar_position: 43 +sidebar_position: 44 --- # `stack bib fr-jump` diff --git a/docs/docs/cli/bib-import-zotero.md b/docs/docs/cli/bib-import-zotero.md index e86d716..e2873a0 100644 --- a/docs/docs/cli/bib-import-zotero.md +++ b/docs/docs/cli/bib-import-zotero.md @@ -1,6 +1,6 @@ --- title: stack bib import-zotero -sidebar_position: 36 +sidebar_position: 37 --- # `stack bib import-zotero` diff --git a/docs/docs/cli/bib-ingest-iom.md b/docs/docs/cli/bib-ingest-iom.md index 1c8d2c7..deeb2b8 100644 --- a/docs/docs/cli/bib-ingest-iom.md +++ b/docs/docs/cli/bib-ingest-iom.md @@ -1,6 +1,6 @@ --- title: stack bib ingest-iom -sidebar_position: 33 +sidebar_position: 34 --- # `stack bib ingest-iom` diff --git a/docs/docs/cli/bib-ingest-mail.md b/docs/docs/cli/bib-ingest-mail.md index f1ea1d4..2e1ec78 100644 --- a/docs/docs/cli/bib-ingest-mail.md +++ b/docs/docs/cli/bib-ingest-mail.md @@ -1,6 +1,6 @@ --- title: stack bib ingest-mail -sidebar_position: 31 +sidebar_position: 32 --- # `stack bib ingest-mail` diff --git a/docs/docs/cli/bib-ingest-oig.md b/docs/docs/cli/bib-ingest-oig.md index 5b71bb1..fddff7d 100644 --- a/docs/docs/cli/bib-ingest-oig.md +++ b/docs/docs/cli/bib-ingest-oig.md @@ -1,6 +1,6 @@ --- title: stack bib ingest-oig -sidebar_position: 38 +sidebar_position: 39 --- # `stack bib ingest-oig` diff --git a/docs/docs/cli/bib-query.md b/docs/docs/cli/bib-query.md index d2e8602..ac85327 100644 --- a/docs/docs/cli/bib-query.md +++ b/docs/docs/cli/bib-query.md @@ -1,6 +1,6 @@ --- title: stack bib query -sidebar_position: 27 +sidebar_position: 28 --- # `stack bib query` diff --git a/docs/docs/cli/bib-refresh-iom.md b/docs/docs/cli/bib-refresh-iom.md index e587e3b..83c3342 100644 --- a/docs/docs/cli/bib-refresh-iom.md +++ b/docs/docs/cli/bib-refresh-iom.md @@ -1,6 +1,6 @@ --- title: stack bib refresh-iom -sidebar_position: 41 +sidebar_position: 42 --- # `stack bib refresh-iom` diff --git a/docs/docs/cli/bib-refresh-oig.md b/docs/docs/cli/bib-refresh-oig.md index 4b7895b..00ca821 100644 --- a/docs/docs/cli/bib-refresh-oig.md +++ b/docs/docs/cli/bib-refresh-oig.md @@ -1,6 +1,6 @@ --- title: stack bib refresh-oig -sidebar_position: 40 +sidebar_position: 41 --- # `stack bib refresh-oig` diff --git a/docs/docs/cli/bib-sync-zotero.md b/docs/docs/cli/bib-sync-zotero.md index 57e1973..76b9993 100644 --- a/docs/docs/cli/bib-sync-zotero.md +++ b/docs/docs/cli/bib-sync-zotero.md @@ -1,6 +1,6 @@ --- title: stack bib sync-zotero -sidebar_position: 37 +sidebar_position: 38 --- # `stack bib sync-zotero` diff --git a/docs/docs/cli/bib-sync.md b/docs/docs/cli/bib-sync.md index 4c2da1c..ab54893 100644 --- a/docs/docs/cli/bib-sync.md +++ b/docs/docs/cli/bib-sync.md @@ -1,6 +1,6 @@ --- title: stack bib sync -sidebar_position: 25 +sidebar_position: 26 --- # `stack bib sync` diff --git a/docs/docs/cli/bib-tag.md b/docs/docs/cli/bib-tag.md index a78f14f..7f1b9b9 100644 --- a/docs/docs/cli/bib-tag.md +++ b/docs/docs/cli/bib-tag.md @@ -1,6 +1,6 @@ --- title: stack bib tag -sidebar_position: 26 +sidebar_position: 27 --- # `stack bib tag` diff --git a/docs/docs/cli/bib-watch-iom.md b/docs/docs/cli/bib-watch-iom.md index 15c44ca..b80b28a 100644 --- a/docs/docs/cli/bib-watch-iom.md +++ b/docs/docs/cli/bib-watch-iom.md @@ -1,6 +1,6 @@ --- title: stack bib watch-iom -sidebar_position: 35 +sidebar_position: 36 --- # `stack bib watch-iom` diff --git a/docs/docs/cli/bib.md b/docs/docs/cli/bib.md index f31fb9e..286f03f 100644 --- a/docs/docs/cli/bib.md +++ b/docs/docs/cli/bib.md @@ -1,6 +1,6 @@ --- title: stack bib -sidebar_position: 24 +sidebar_position: 25 --- # `stack bib` diff --git a/docs/docs/cli/bls-ces.md b/docs/docs/cli/bls-ces.md index 32ecae0..cdabc92 100644 --- a/docs/docs/cli/bls-ces.md +++ b/docs/docs/cli/bls-ces.md @@ -1,6 +1,6 @@ --- title: stack bls ces -sidebar_position: 48 +sidebar_position: 49 --- # `stack bls ces` diff --git a/docs/docs/cli/bls-oews.md b/docs/docs/cli/bls-oews.md index dc2b4f6..55820cf 100644 --- a/docs/docs/cli/bls-oews.md +++ b/docs/docs/cli/bls-oews.md @@ -1,6 +1,6 @@ --- title: stack bls oews -sidebar_position: 47 +sidebar_position: 48 --- # `stack bls oews` diff --git a/docs/docs/cli/bls.md b/docs/docs/cli/bls.md index 417e796..14a0ea9 100644 --- a/docs/docs/cli/bls.md +++ b/docs/docs/cli/bls.md @@ -1,6 +1,6 @@ --- title: stack bls -sidebar_position: 46 +sidebar_position: 47 --- # `stack bls` diff --git a/docs/docs/cli/comments-dockets.md b/docs/docs/cli/comments-dockets.md index ed2f06f..35624d1 100644 --- a/docs/docs/cli/comments-dockets.md +++ b/docs/docs/cli/comments-dockets.md @@ -1,6 +1,6 @@ --- title: stack comments dockets -sidebar_position: 53 +sidebar_position: 54 --- # `stack comments dockets` diff --git a/docs/docs/cli/comments-extract-ocr.md b/docs/docs/cli/comments-extract-ocr.md index 7958670..8fc3592 100644 --- a/docs/docs/cli/comments-extract-ocr.md +++ b/docs/docs/cli/comments-extract-ocr.md @@ -1,6 +1,6 @@ --- title: stack comments extract-ocr -sidebar_position: 51 +sidebar_position: 52 --- # `stack comments extract-ocr` diff --git a/docs/docs/cli/comments-extract.md b/docs/docs/cli/comments-extract.md index d21f1ac..e97b2bc 100644 --- a/docs/docs/cli/comments-extract.md +++ b/docs/docs/cli/comments-extract.md @@ -1,6 +1,6 @@ --- title: stack comments extract -sidebar_position: 50 +sidebar_position: 51 --- # `stack comments extract` diff --git a/docs/docs/cli/comments-seal.md b/docs/docs/cli/comments-seal.md index 47fad6a..8151f5f 100644 --- a/docs/docs/cli/comments-seal.md +++ b/docs/docs/cli/comments-seal.md @@ -1,6 +1,6 @@ --- title: stack comments seal -sidebar_position: 54 +sidebar_position: 55 --- # `stack comments seal` diff --git a/docs/docs/cli/comments-stats.md b/docs/docs/cli/comments-stats.md index daf0817..32f994f 100644 --- a/docs/docs/cli/comments-stats.md +++ b/docs/docs/cli/comments-stats.md @@ -1,6 +1,6 @@ --- title: stack comments stats -sidebar_position: 52 +sidebar_position: 53 --- # `stack comments stats` diff --git a/docs/docs/cli/comments-unseal.md b/docs/docs/cli/comments-unseal.md index a2be37e..1cc4ca5 100644 --- a/docs/docs/cli/comments-unseal.md +++ b/docs/docs/cli/comments-unseal.md @@ -1,6 +1,6 @@ --- title: stack comments unseal -sidebar_position: 55 +sidebar_position: 56 --- # `stack comments unseal` diff --git a/docs/docs/cli/comments.md b/docs/docs/cli/comments.md index c4b15a7..5c6f7cd 100644 --- a/docs/docs/cli/comments.md +++ b/docs/docs/cli/comments.md @@ -1,6 +1,6 @@ --- title: stack comments -sidebar_position: 49 +sidebar_position: 50 --- # `stack comments` diff --git a/docs/docs/cli/db-comment.md b/docs/docs/cli/db-comment.md index 15462aa..a3ac5a7 100644 --- a/docs/docs/cli/db-comment.md +++ b/docs/docs/cli/db-comment.md @@ -1,6 +1,6 @@ --- title: stack db comment -sidebar_position: 72 +sidebar_position: 73 --- # `stack db comment` diff --git a/docs/docs/cli/db-inspect.md b/docs/docs/cli/db-inspect.md index 0ec38b5..28d5b2d 100644 --- a/docs/docs/cli/db-inspect.md +++ b/docs/docs/cli/db-inspect.md @@ -1,6 +1,6 @@ --- title: stack db inspect -sidebar_position: 73 +sidebar_position: 74 --- # `stack db inspect` diff --git a/docs/docs/cli/db.md b/docs/docs/cli/db.md index 7297962..309e7bb 100644 --- a/docs/docs/cli/db.md +++ b/docs/docs/cli/db.md @@ -1,6 +1,6 @@ --- title: stack db -sidebar_position: 71 +sidebar_position: 72 --- # `stack db` diff --git a/docs/docs/cli/docs-build.md b/docs/docs/cli/docs-build.md index 7fcd660..07c6175 100644 --- a/docs/docs/cli/docs-build.md +++ b/docs/docs/cli/docs-build.md @@ -1,6 +1,6 @@ --- title: stack docs build -sidebar_position: 75 +sidebar_position: 76 --- # `stack docs build` diff --git a/docs/docs/cli/docs-generate.md b/docs/docs/cli/docs-generate.md index e039afa..a7ea1a5 100644 --- a/docs/docs/cli/docs-generate.md +++ b/docs/docs/cli/docs-generate.md @@ -1,6 +1,6 @@ --- title: stack docs generate -sidebar_position: 77 +sidebar_position: 78 --- # `stack docs generate` diff --git a/docs/docs/cli/docs-serve.md b/docs/docs/cli/docs-serve.md index 9b7530f..4fede99 100644 --- a/docs/docs/cli/docs-serve.md +++ b/docs/docs/cli/docs-serve.md @@ -1,6 +1,6 @@ --- title: stack docs serve -sidebar_position: 76 +sidebar_position: 77 --- # `stack docs serve` diff --git a/docs/docs/cli/docs.md b/docs/docs/cli/docs.md index 246f84e..a05afb4 100644 --- a/docs/docs/cli/docs.md +++ b/docs/docs/cli/docs.md @@ -1,6 +1,6 @@ --- title: stack docs -sidebar_position: 74 +sidebar_position: 75 --- # `stack docs` diff --git a/docs/docs/cli/generate-models.md b/docs/docs/cli/generate-models.md index e603edc..5794474 100644 --- a/docs/docs/cli/generate-models.md +++ b/docs/docs/cli/generate-models.md @@ -1,6 +1,6 @@ --- title: stack generate models -sidebar_position: 23 +sidebar_position: 24 --- # `stack generate models` diff --git a/docs/docs/cli/generate.md b/docs/docs/cli/generate.md index 36be99c..fbbb526 100644 --- a/docs/docs/cli/generate.md +++ b/docs/docs/cli/generate.md @@ -1,6 +1,6 @@ --- title: stack generate -sidebar_position: 22 +sidebar_position: 23 --- # `stack generate` diff --git a/docs/docs/cli/lake-deploy.md b/docs/docs/cli/lake-deploy.md index 52bdbf9..796afb9 100644 --- a/docs/docs/cli/lake-deploy.md +++ b/docs/docs/cli/lake-deploy.md @@ -1,6 +1,6 @@ --- title: stack lake deploy -sidebar_position: 57 +sidebar_position: 58 --- # `stack lake deploy` diff --git a/docs/docs/cli/lake-load.md b/docs/docs/cli/lake-load.md index 392d76b..bfa4a06 100644 --- a/docs/docs/cli/lake-load.md +++ b/docs/docs/cli/lake-load.md @@ -1,6 +1,6 @@ --- title: stack lake load -sidebar_position: 58 +sidebar_position: 59 --- # `stack lake load` diff --git a/docs/docs/cli/lake-parity.md b/docs/docs/cli/lake-parity.md index 0a8b5f9..222d968 100644 --- a/docs/docs/cli/lake-parity.md +++ b/docs/docs/cli/lake-parity.md @@ -1,6 +1,6 @@ --- title: stack lake parity -sidebar_position: 60 +sidebar_position: 61 --- # `stack lake parity` diff --git a/docs/docs/cli/lake-validate.md b/docs/docs/cli/lake-validate.md index 80465cb..eddac98 100644 --- a/docs/docs/cli/lake-validate.md +++ b/docs/docs/cli/lake-validate.md @@ -1,6 +1,6 @@ --- title: stack lake validate -sidebar_position: 59 +sidebar_position: 60 --- # `stack lake validate` diff --git a/docs/docs/cli/lake.md b/docs/docs/cli/lake.md index 88e8777..2333909 100644 --- a/docs/docs/cli/lake.md +++ b/docs/docs/cli/lake.md @@ -1,6 +1,6 @@ --- title: stack lake -sidebar_position: 56 +sidebar_position: 57 --- # `stack lake` diff --git a/docs/docs/cli/llm-eval-tags.md b/docs/docs/cli/llm-eval-tags.md index 4241416..b0582af 100644 --- a/docs/docs/cli/llm-eval-tags.md +++ b/docs/docs/cli/llm-eval-tags.md @@ -1,6 +1,6 @@ --- title: stack llm eval-tags -sidebar_position: 68 +sidebar_position: 69 --- # `stack llm eval-tags` diff --git a/docs/docs/cli/llm-hosts.md b/docs/docs/cli/llm-hosts.md index 4a348a9..9b00348 100644 --- a/docs/docs/cli/llm-hosts.md +++ b/docs/docs/cli/llm-hosts.md @@ -1,6 +1,6 @@ --- title: stack llm hosts -sidebar_position: 64 +sidebar_position: 65 --- # `stack llm hosts` diff --git a/docs/docs/cli/llm-index.md b/docs/docs/cli/llm-index.md index 27bf742..f038d0e 100644 --- a/docs/docs/cli/llm-index.md +++ b/docs/docs/cli/llm-index.md @@ -1,6 +1,6 @@ --- title: stack llm index -sidebar_position: 62 +sidebar_position: 63 --- # `stack llm index` diff --git a/docs/docs/cli/llm-prune-rules.md b/docs/docs/cli/llm-prune-rules.md index c290e58..1792b91 100644 --- a/docs/docs/cli/llm-prune-rules.md +++ b/docs/docs/cli/llm-prune-rules.md @@ -1,6 +1,6 @@ --- title: stack llm prune-rules -sidebar_position: 69 +sidebar_position: 70 --- # `stack llm prune-rules` diff --git a/docs/docs/cli/llm-restamp.md b/docs/docs/cli/llm-restamp.md index df5e320..15a3774 100644 --- a/docs/docs/cli/llm-restamp.md +++ b/docs/docs/cli/llm-restamp.md @@ -1,6 +1,6 @@ --- title: stack llm restamp -sidebar_position: 63 +sidebar_position: 64 --- # `stack llm restamp` diff --git a/docs/docs/cli/llm-serve.md b/docs/docs/cli/llm-serve.md index 9fc321b..17b8fee 100644 --- a/docs/docs/cli/llm-serve.md +++ b/docs/docs/cli/llm-serve.md @@ -1,6 +1,6 @@ --- title: stack llm serve -sidebar_position: 65 +sidebar_position: 66 --- # `stack llm serve` diff --git a/docs/docs/cli/llm-stamp-themes.md b/docs/docs/cli/llm-stamp-themes.md index bb399a8..5742bbe 100644 --- a/docs/docs/cli/llm-stamp-themes.md +++ b/docs/docs/cli/llm-stamp-themes.md @@ -1,6 +1,6 @@ --- title: stack llm stamp-themes -sidebar_position: 70 +sidebar_position: 71 --- # `stack llm stamp-themes` diff --git a/docs/docs/cli/llm-tag.md b/docs/docs/cli/llm-tag.md index 97358fa..8235894 100644 --- a/docs/docs/cli/llm-tag.md +++ b/docs/docs/cli/llm-tag.md @@ -1,6 +1,6 @@ --- title: stack llm tag -sidebar_position: 67 +sidebar_position: 68 --- # `stack llm tag` diff --git a/docs/docs/cli/llm-vocab.md b/docs/docs/cli/llm-vocab.md index 7caac45..7ee5363 100644 --- a/docs/docs/cli/llm-vocab.md +++ b/docs/docs/cli/llm-vocab.md @@ -1,6 +1,6 @@ --- title: stack llm vocab -sidebar_position: 66 +sidebar_position: 67 --- # `stack llm vocab` diff --git a/docs/docs/cli/llm.md b/docs/docs/cli/llm.md index 069f1d2..c66c4cd 100644 --- a/docs/docs/cli/llm.md +++ b/docs/docs/cli/llm.md @@ -1,6 +1,6 @@ --- title: stack llm -sidebar_position: 61 +sidebar_position: 62 --- # `stack llm` diff --git a/docs/docs/cli/mail-attach-smarthost.md b/docs/docs/cli/mail-attach-smarthost.md index 1818125..f287be7 100644 --- a/docs/docs/cli/mail-attach-smarthost.md +++ b/docs/docs/cli/mail-attach-smarthost.md @@ -1,6 +1,6 @@ --- title: stack mail attach-smarthost -sidebar_position: 120 +sidebar_position: 121 --- # `stack mail attach-smarthost` diff --git a/docs/docs/cli/mail-dkim-export.md b/docs/docs/cli/mail-dkim-export.md index 43fcd94..40eb1e3 100644 --- a/docs/docs/cli/mail-dkim-export.md +++ b/docs/docs/cli/mail-dkim-export.md @@ -1,6 +1,6 @@ --- title: stack mail dkim-export -sidebar_position: 119 +sidebar_position: 120 --- # `stack mail dkim-export` diff --git a/docs/docs/cli/mail-dns.md b/docs/docs/cli/mail-dns.md index a13cd3d..725950a 100644 --- a/docs/docs/cli/mail-dns.md +++ b/docs/docs/cli/mail-dns.md @@ -1,6 +1,6 @@ --- title: stack mail dns -sidebar_position: 118 +sidebar_position: 119 --- # `stack mail dns` diff --git a/docs/docs/cli/mail-down.md b/docs/docs/cli/mail-down.md index f6cf2fb..9309ef8 100644 --- a/docs/docs/cli/mail-down.md +++ b/docs/docs/cli/mail-down.md @@ -1,6 +1,6 @@ --- title: stack mail down -sidebar_position: 116 +sidebar_position: 117 --- # `stack mail down` diff --git a/docs/docs/cli/mail-provision.md b/docs/docs/cli/mail-provision.md index 0cc4a87..b108d73 100644 --- a/docs/docs/cli/mail-provision.md +++ b/docs/docs/cli/mail-provision.md @@ -1,6 +1,6 @@ --- title: stack mail provision -sidebar_position: 114 +sidebar_position: 115 --- # `stack mail provision` diff --git a/docs/docs/cli/mail-rotate-creds.md b/docs/docs/cli/mail-rotate-creds.md index 86cba90..bf4e438 100644 --- a/docs/docs/cli/mail-rotate-creds.md +++ b/docs/docs/cli/mail-rotate-creds.md @@ -1,6 +1,6 @@ --- title: stack mail rotate-creds -sidebar_position: 121 +sidebar_position: 122 --- # `stack mail rotate-creds` diff --git a/docs/docs/cli/mail-seed-mailboxes.md b/docs/docs/cli/mail-seed-mailboxes.md index 3743602..d583b96 100644 --- a/docs/docs/cli/mail-seed-mailboxes.md +++ b/docs/docs/cli/mail-seed-mailboxes.md @@ -1,6 +1,6 @@ --- title: stack mail seed-mailboxes -sidebar_position: 122 +sidebar_position: 123 --- # `stack mail seed-mailboxes` diff --git a/docs/docs/cli/mail-status.md b/docs/docs/cli/mail-status.md index 72ae198..c5424de 100644 --- a/docs/docs/cli/mail-status.md +++ b/docs/docs/cli/mail-status.md @@ -1,6 +1,6 @@ --- title: stack mail status -sidebar_position: 117 +sidebar_position: 118 --- # `stack mail status` diff --git a/docs/docs/cli/mail-up.md b/docs/docs/cli/mail-up.md index e155979..78a2e78 100644 --- a/docs/docs/cli/mail-up.md +++ b/docs/docs/cli/mail-up.md @@ -1,6 +1,6 @@ --- title: stack mail up -sidebar_position: 115 +sidebar_position: 116 --- # `stack mail up` diff --git a/docs/docs/cli/mail-wire-git.md b/docs/docs/cli/mail-wire-git.md index 0f234a4..3b7a781 100644 --- a/docs/docs/cli/mail-wire-git.md +++ b/docs/docs/cli/mail-wire-git.md @@ -1,6 +1,6 @@ --- title: stack mail wire-git -sidebar_position: 123 +sidebar_position: 124 --- # `stack mail wire-git` diff --git a/docs/docs/cli/mail.md b/docs/docs/cli/mail.md index ccfa985..510a133 100644 --- a/docs/docs/cli/mail.md +++ b/docs/docs/cli/mail.md @@ -1,6 +1,6 @@ --- title: stack mail -sidebar_position: 113 +sidebar_position: 114 --- # `stack mail` diff --git a/docs/docs/cli/perf-show.md b/docs/docs/cli/perf-show.md index 04ac799..54dac5c 100644 --- a/docs/docs/cli/perf-show.md +++ b/docs/docs/cli/perf-show.md @@ -1,6 +1,6 @@ --- title: stack perf show -sidebar_position: 81 +sidebar_position: 82 --- # `stack perf show` diff --git a/docs/docs/cli/perf.md b/docs/docs/cli/perf.md index 40feb56..7524bd4 100644 --- a/docs/docs/cli/perf.md +++ b/docs/docs/cli/perf.md @@ -1,6 +1,6 @@ --- title: stack perf -sidebar_position: 80 +sidebar_position: 81 --- # `stack perf` diff --git a/docs/docs/cli/pfs-cpt-ingest.md b/docs/docs/cli/pfs-cpt-ingest.md index f5786f5..cc0eadc 100644 --- a/docs/docs/cli/pfs-cpt-ingest.md +++ b/docs/docs/cli/pfs-cpt-ingest.md @@ -1,6 +1,6 @@ --- title: stack pfs cpt-ingest -sidebar_position: 95 +sidebar_position: 96 --- # `stack pfs cpt-ingest` diff --git a/docs/docs/cli/pfs-elements.md b/docs/docs/cli/pfs-elements.md index d27fe84..0702a65 100644 --- a/docs/docs/cli/pfs-elements.md +++ b/docs/docs/cli/pfs-elements.md @@ -1,6 +1,6 @@ --- title: stack pfs elements -sidebar_position: 87 +sidebar_position: 88 --- # `stack pfs elements` diff --git a/docs/docs/cli/pfs-exposure.md b/docs/docs/cli/pfs-exposure.md index e2121b6..59bb8f9 100644 --- a/docs/docs/cli/pfs-exposure.md +++ b/docs/docs/cli/pfs-exposure.md @@ -1,6 +1,6 @@ --- title: stack pfs exposure -sidebar_position: 92 +sidebar_position: 93 --- # `stack pfs exposure` diff --git a/docs/docs/cli/pfs-families.md b/docs/docs/cli/pfs-families.md index 9c6b5c5..c7ea024 100644 --- a/docs/docs/cli/pfs-families.md +++ b/docs/docs/cli/pfs-families.md @@ -1,6 +1,6 @@ --- title: stack pfs families -sidebar_position: 89 +sidebar_position: 90 --- # `stack pfs families` diff --git a/docs/docs/cli/pfs-guidance.md b/docs/docs/cli/pfs-guidance.md index a463a90..8796860 100644 --- a/docs/docs/cli/pfs-guidance.md +++ b/docs/docs/cli/pfs-guidance.md @@ -1,6 +1,6 @@ --- title: stack pfs guidance -sidebar_position: 90 +sidebar_position: 91 --- # `stack pfs guidance` diff --git a/docs/docs/cli/pfs-lineage.md b/docs/docs/cli/pfs-lineage.md index 54cc2f8..4293131 100644 --- a/docs/docs/cli/pfs-lineage.md +++ b/docs/docs/cli/pfs-lineage.md @@ -1,6 +1,6 @@ --- title: stack pfs lineage -sidebar_position: 88 +sidebar_position: 89 --- # `stack pfs lineage` diff --git a/docs/docs/cli/pfs-reaction.md b/docs/docs/cli/pfs-reaction.md index 89b5370..0ddcc2a 100644 --- a/docs/docs/cli/pfs-reaction.md +++ b/docs/docs/cli/pfs-reaction.md @@ -1,6 +1,6 @@ --- title: stack pfs reaction -sidebar_position: 91 +sidebar_position: 92 --- # `stack pfs reaction` diff --git a/docs/docs/cli/pfs-review.md b/docs/docs/cli/pfs-review.md index 5aa3c34..4c5c6b1 100644 --- a/docs/docs/cli/pfs-review.md +++ b/docs/docs/cli/pfs-review.md @@ -1,6 +1,6 @@ --- title: stack pfs review -sidebar_position: 94 +sidebar_position: 95 --- # `stack pfs review` diff --git a/docs/docs/cli/pfs-utilization.md b/docs/docs/cli/pfs-utilization.md index a80fde9..86adcf2 100644 --- a/docs/docs/cli/pfs-utilization.md +++ b/docs/docs/cli/pfs-utilization.md @@ -1,6 +1,6 @@ --- title: stack pfs utilization -sidebar_position: 93 +sidebar_position: 94 --- # `stack pfs utilization` diff --git a/docs/docs/cli/pfs.md b/docs/docs/cli/pfs.md index 3b06635..adf65d5 100644 --- a/docs/docs/cli/pfs.md +++ b/docs/docs/cli/pfs.md @@ -1,6 +1,6 @@ --- title: stack pfs -sidebar_position: 86 +sidebar_position: 87 --- # `stack pfs` diff --git a/docs/docs/cli/prisma-eligible.md b/docs/docs/cli/prisma-eligible.md index b7dcc28..c1f98d6 100644 --- a/docs/docs/cli/prisma-eligible.md +++ b/docs/docs/cli/prisma-eligible.md @@ -1,6 +1,6 @@ --- title: stack prisma eligible -sidebar_position: 107 +sidebar_position: 108 --- # `stack prisma eligible` diff --git a/docs/docs/cli/prisma-export.md b/docs/docs/cli/prisma-export.md index ee424e7..e496332 100644 --- a/docs/docs/cli/prisma-export.md +++ b/docs/docs/cli/prisma-export.md @@ -1,6 +1,6 @@ --- title: stack prisma export -sidebar_position: 104 +sidebar_position: 105 --- # `stack prisma export` diff --git a/docs/docs/cli/prisma-extract.md b/docs/docs/cli/prisma-extract.md index 9d2650e..99f66bf 100644 --- a/docs/docs/cli/prisma-extract.md +++ b/docs/docs/cli/prisma-extract.md @@ -1,6 +1,6 @@ --- title: stack prisma extract -sidebar_position: 108 +sidebar_position: 109 --- # `stack prisma extract` diff --git a/docs/docs/cli/prisma-fetch.md b/docs/docs/cli/prisma-fetch.md index c75ebbb..c180b30 100644 --- a/docs/docs/cli/prisma-fetch.md +++ b/docs/docs/cli/prisma-fetch.md @@ -1,6 +1,6 @@ --- title: stack prisma fetch -sidebar_position: 110 +sidebar_position: 111 --- # `stack prisma fetch` diff --git a/docs/docs/cli/prisma-flow.md b/docs/docs/cli/prisma-flow.md index 2284fc8..ea4a1cd 100644 --- a/docs/docs/cli/prisma-flow.md +++ b/docs/docs/cli/prisma-flow.md @@ -1,6 +1,6 @@ --- title: stack prisma flow -sidebar_position: 109 +sidebar_position: 110 --- # `stack prisma flow` diff --git a/docs/docs/cli/prisma-init.md b/docs/docs/cli/prisma-init.md index 1defc78..6db6ce4 100644 --- a/docs/docs/cli/prisma-init.md +++ b/docs/docs/cli/prisma-init.md @@ -1,6 +1,6 @@ --- title: stack prisma init -sidebar_position: 103 +sidebar_position: 104 --- # `stack prisma init` diff --git a/docs/docs/cli/prisma-ping-llm.md b/docs/docs/cli/prisma-ping-llm.md index 87ad272..995c52c 100644 --- a/docs/docs/cli/prisma-ping-llm.md +++ b/docs/docs/cli/prisma-ping-llm.md @@ -1,6 +1,6 @@ --- title: stack prisma ping-llm -sidebar_position: 105 +sidebar_position: 106 --- # `stack prisma ping-llm` diff --git a/docs/docs/cli/prisma-run.md b/docs/docs/cli/prisma-run.md index 4077344..929c6cb 100644 --- a/docs/docs/cli/prisma-run.md +++ b/docs/docs/cli/prisma-run.md @@ -1,6 +1,6 @@ --- title: stack prisma run -sidebar_position: 111 +sidebar_position: 112 --- # `stack prisma run` diff --git a/docs/docs/cli/prisma-screen.md b/docs/docs/cli/prisma-screen.md index 2194dc5..b22d248 100644 --- a/docs/docs/cli/prisma-screen.md +++ b/docs/docs/cli/prisma-screen.md @@ -1,6 +1,6 @@ --- title: stack prisma screen -sidebar_position: 106 +sidebar_position: 107 --- # `stack prisma screen` diff --git a/docs/docs/cli/prisma-vpn.md b/docs/docs/cli/prisma-vpn.md index 389be23..430eb0e 100644 --- a/docs/docs/cli/prisma-vpn.md +++ b/docs/docs/cli/prisma-vpn.md @@ -1,6 +1,6 @@ --- title: stack prisma vpn -sidebar_position: 112 +sidebar_position: 113 --- # `stack prisma vpn` diff --git a/docs/docs/cli/prisma.md b/docs/docs/cli/prisma.md index 6ed6413..98d0e08 100644 --- a/docs/docs/cli/prisma.md +++ b/docs/docs/cli/prisma.md @@ -1,6 +1,6 @@ --- title: stack prisma -sidebar_position: 102 +sidebar_position: 103 --- # `stack prisma` diff --git a/docs/docs/cli/rec-list.md b/docs/docs/cli/rec-list.md index 585423f..1645b31 100644 --- a/docs/docs/cli/rec-list.md +++ b/docs/docs/cli/rec-list.md @@ -1,6 +1,6 @@ --- title: stack rec list -sidebar_position: 83 +sidebar_position: 84 --- # `stack rec list` diff --git a/docs/docs/cli/rec-opps.md b/docs/docs/cli/rec-opps.md index b7973e2..a8f40c1 100644 --- a/docs/docs/cli/rec-opps.md +++ b/docs/docs/cli/rec-opps.md @@ -1,6 +1,6 @@ --- title: stack rec opps -sidebar_position: 85 +sidebar_position: 86 --- # `stack rec opps` diff --git a/docs/docs/cli/rec-pfs.md b/docs/docs/cli/rec-pfs.md index d672025..42ccde5 100644 --- a/docs/docs/cli/rec-pfs.md +++ b/docs/docs/cli/rec-pfs.md @@ -1,6 +1,6 @@ --- title: stack rec pfs -sidebar_position: 84 +sidebar_position: 85 --- # `stack rec pfs` diff --git a/docs/docs/cli/rec.md b/docs/docs/cli/rec.md index dc92750..dc47b70 100644 --- a/docs/docs/cli/rec.md +++ b/docs/docs/cli/rec.md @@ -1,6 +1,6 @@ --- title: stack rec -sidebar_position: 82 +sidebar_position: 83 --- # `stack rec` diff --git a/docs/docs/cli/zot-dump-schema.md b/docs/docs/cli/zot-dump-schema.md index 9d52972..44e4001 100644 --- a/docs/docs/cli/zot-dump-schema.md +++ b/docs/docs/cli/zot-dump-schema.md @@ -1,6 +1,6 @@ --- title: stack zot dump-schema -sidebar_position: 97 +sidebar_position: 98 --- # `stack zot dump-schema` diff --git a/docs/docs/cli/zot-fix-dates.md b/docs/docs/cli/zot-fix-dates.md index b2c6fca..3890c74 100644 --- a/docs/docs/cli/zot-fix-dates.md +++ b/docs/docs/cli/zot-fix-dates.md @@ -1,6 +1,6 @@ --- title: stack zot fix-dates -sidebar_position: 98 +sidebar_position: 99 --- # `stack zot fix-dates` diff --git a/docs/docs/cli/zot-fix-fields.md b/docs/docs/cli/zot-fix-fields.md index 4ece6b7..fbbef71 100644 --- a/docs/docs/cli/zot-fix-fields.md +++ b/docs/docs/cli/zot-fix-fields.md @@ -1,6 +1,6 @@ --- title: stack zot fix-fields -sidebar_position: 100 +sidebar_position: 101 --- # `stack zot fix-fields` diff --git a/docs/docs/cli/zot-fix-keys.md b/docs/docs/cli/zot-fix-keys.md index f7cafbe..518b8fb 100644 --- a/docs/docs/cli/zot-fix-keys.md +++ b/docs/docs/cli/zot-fix-keys.md @@ -1,6 +1,6 @@ --- title: stack zot fix-keys -sidebar_position: 99 +sidebar_position: 100 --- # `stack zot fix-keys` diff --git a/docs/docs/cli/zot-verify-parity.md b/docs/docs/cli/zot-verify-parity.md index 864fb05..92eba08 100644 --- a/docs/docs/cli/zot-verify-parity.md +++ b/docs/docs/cli/zot-verify-parity.md @@ -1,6 +1,6 @@ --- title: stack zot verify-parity -sidebar_position: 101 +sidebar_position: 102 --- # `stack zot verify-parity` diff --git a/docs/docs/cli/zot.md b/docs/docs/cli/zot.md index 748c6d3..2f945ca 100644 --- a/docs/docs/cli/zot.md +++ b/docs/docs/cli/zot.md @@ -1,6 +1,6 @@ --- title: stack zot -sidebar_position: 96 +sidebar_position: 97 --- # `stack zot` diff --git a/docs/superpowers/specs/2026-10-25-bcda-since-reconciliation-design.md b/docs/superpowers/specs/2026-10-25-bcda-since-reconciliation-design.md index b5ad30e..bdf5272 100644 --- a/docs/superpowers/specs/2026-10-25-bcda-since-reconciliation-design.md +++ b/docs/superpowers/specs/2026-10-25-bcda-since-reconciliation-design.md @@ -14,6 +14,40 @@ choose the right `_since`, refuse the wrong one, ingest the delta, reconcile it with everything already on disk under explicit tiebreakers, and keep a ledger that proves the invariants held. +## What the server does (from `CMSgov/bcda-app`, read 2026-10-25) + +- The `_since` window is **`(since, transactionTime]`**: the worker queries + BFD with `_lastUpdated=gt` and `_lastUpdated=le` + (`bcda/client/bluebutton.go`). `transactionTime` is BFD's + `Bundle.meta.lastUpdated`, the instant the last ETL load completed + (PR #1483); environments with synthetic data fall back to `now()`, + which is why the sandbox stamps midnight. +- A baseline is bounded by **service date**: the model's lookback window + (`lookback_period`, 2–3 performance years for CKCC/KCF/DC/ACCESS; none + for SSP/CEC/NGACO, hence "as early as March 1, 2021") and, for runout or + a terminated entity, a claims-through date. A caller's `service-date` + filter is intersected with it. GUIDE receives no partially adjudicated + claims. +- **Silent in deltas:** enrollees who opted out (bene-prefs, SAMHSA + preference) are removed from the attribution set at job time and + sensitive claims are excluded at BFD (`_security:not=42CFRPart2`); + terminated entities lose access the day participation ends. None of + this produces a row. Only a new baseline shows the enrollee gone. +- **Re-loads:** CMS re-loads fields as the v3 dataset is refined; + corrected claims carry a newer `lastUpdated`. Expect `replaced` without + clinical change. +- **Partial exports:** the manifest's `error` list holds OperationOutcome + ndjson for chunks that failed at BFD. Such a job's window is not fully + covered, so it does not advance the watermark (`partial_job`). +- **Claim identity** (bcda.cms.gov, *Matching multiple EOBs to the same + claim*): one version per control number per source system; across + systems the control number, with the VMS→NCH leading-digit rule for + DME; a changed control number names its predecessor in + `ExplanationOfBenefit.related` (institutional claims carry only the + immediate parent); Part D events without a control number use the CCLF + Information Packet §5.1.2 natural key. `clm_cntl_num_match`, + `clm_natural_key` and `superseded_by` carry this in the tables. + ## What the API actually promises (sources of truth) From bcda.cms.gov (How to Filter Claims Data, How to Migrate to v3, FAQs) and @@ -80,7 +114,7 @@ Identity keys: |---|---|---| | `bcda.patient` | `bene_id` (falls back to `id`) | `Patient.id` is `BENE_SK`; MBIs rotate | | `bcda.coverage` | `id` | composite bene × plan × sequence | -| `bcda.explanation_of_benefit` | `clm_cntl_num`, else `clm_id`, else `id` | a claim version gets a new FHIR id; the control number is the claim | +| `bcda.explanation_of_benefit` | `clm_cntl_num_match` (control number, VMS leading `1` stripped on SharedSystem DME), else `clm_cntl_num`, else `clm_natural_key` (Part D), else `clm_id`, else `id` | a claim version gets a new FHIR id; the control number is the claim | | `bcda.eob_item` | follows its header (`clm_id` + job) | lines are replaced wholesale with their header | Candidate rows for a type = rows of the newest applied baseline carrying it @@ -100,6 +134,11 @@ Winner per key, in order, each applied only when the previous ties: 5. greatest FHIR `id` string (total order; never reached in practice, present so the result is a pure function of the inputs). +After winners are chosen, a claim whose control number another current +claim names in `related` is marked `superseded_by` that claim's key (the +chain is walked); it stays in the table as history, is excluded from the +connector and counted as `superseded`. + A winner tagged `NotFinalAction` stays in the table as a **tombstone**: it records that the claim's latest version is not final, and the connector (`_stg_claim_header`) already drops it from `input_layer`. Rows are never @@ -164,6 +203,10 @@ with levels `ok` / `warn` / `block`; the UI and CLI refuse to start on | `unapplied` | warn | downloaded jobs not yet reconciled | | `attribution_newer` | warn | `last_attribution_update` > baseline `transactionTime`: expect backfill for new enrollees; attrition stays invisible until the next baseline | | `stale_baseline` | warn | baseline older than 24 h × 30: files are gone, only a new baseline can retire rows | +| `partial_job` | warn | a job with OperationOutcome error files covers its window only partly; it does not advance the watermark; re-pull with the same `_since` | +| `attribution_expired` | block | `attribution_status` `expiration_dates` is in the past: the server answers 404 until the next attribution file (every January, about a month) | +| `attribution_floor` | warn | `_since` predates the oldest `last_attribution_update` this store recorded: the server has no attribution file that old (404) | +| `runout_window` | warn | `Group/runout` after July: runout for the previous PY stopped refreshing | `next_since()` formats the watermark as `YYYY-MM-DDThh:mm:ss.sss+00:00`. diff --git a/docs/superpowers/specs/2026-10-25-bcda-upstream-review.md b/docs/superpowers/specs/2026-10-25-bcda-upstream-review.md index d42252d..265765e 100644 --- a/docs/superpowers/specs/2026-10-25-bcda-upstream-review.md +++ b/docs/superpowers/specs/2026-10-25-bcda-upstream-review.md @@ -212,18 +212,18 @@ show as `replaced` without clinical change. real pull), matching "claims are automatically deduplicated within a source system". -## Tracker issues (milestone P52) +## Tracker issues (milestone P52) — all addressed on `feat/bcda-v3` -| Gap | Issue | -|---|---| -| A · error files / partial jobs and the watermark | #1293 | -| B · claim identity: control-number chain, DME digit, Part D natural key | #1294 | -| C · attribution expiration, January, oldest-attribution floor | #1295 | -| D · adopt only servable jobs; cancelled-while-adopting | #1296 | -| E · `_typeFilter` grammar | #1297 | -| F · runout calendar and population labels | #1298 | -| G · vendor manifest coverage | #1299 | -| H · design records quote the server | #1300 | +| Gap | Issue | Where | +|---|---|---| +| A · error files / partial jobs and the watermark | #1293 | `Store._download_outputs` (OperationOutcome files, `DownloadError`), `JobInfo.partial`, `watermarks()`, `partial_job` check, Store page, `stack bcda errors` | +| B · claim identity: control-number chain, DME digit, Part D natural key | #1294 | `clm_cntl_num_match`, `clm_natural_key`, `prvdr_srvc_id_qlfyr_cd`, `superseded_by`; `sync.KEY_EXPR`, `_mark_superseded`, connector `_stg_claim_header` | +| C · attribution expiration, January, oldest-attribution floor | #1295 | `attribution_expired`, `attribution_floor` checks; `attribution_update` on job rows; `explain_server_error` | +| D · adopt only servable jobs; cancelled-while-adopting | #1296 | `Client.completed_jobs(max_age_hours=24)`; every file attempted, failures recorded per file | +| E · `_typeFilter` grammar | #1297 | `build_type_filter` (outcome list, service-date bounds and prefixes, server validation rules), CLI/UI options | +| F · runout calendar and population labels | #1298 | `runout_window` check; population on Sync page ledger | +| G · vendor manifest coverage | #1299 | `dev/vendor/MANIFEST.toml`: bcda-app behaviour files, `bcda-static-site` docs pages, PAC and unmapped-fields dictionaries | +| H · design records quote the server | #1300 | `2026-10-25-bcda-since-reconciliation-design.md`, "What the server does" | ## Sources diff --git a/src/bcda/client.py b/src/bcda/client.py index b0f0220..ab1b5d5 100644 --- a/src/bcda/client.py +++ b/src/bcda/client.py @@ -677,9 +677,14 @@ class Client: return [e["resource"] for e in body.get("entry", []) if "resource" in e] return body - def completed_jobs(self) -> list[dict]: + def completed_jobs(self, *, max_age_hours: float | None = 24) -> list[dict]: """Completed jobs, newest first, as ``(job_id, request, end)`` dicts. + ``/jobs`` reports Archived and Expired jobs as ``completed`` too, + but their files are gone (410), so by default only jobs that + ended within *max_age_hours* (the server's 24 h retention) are + returned; ``None`` lists them all. + Convenience over :meth:`jobs` for "reuse the last export" logic: ``job_id`` is the numeric id, ``job_url`` the status URL for :meth:`poll`, ``request`` the originating ``$export`` @@ -690,11 +695,26 @@ class Client: downloadable for 24 hours after completion (then 410/404). """ out = [] + cutoff = None + if max_age_hours is not None: + from datetime import datetime, timedelta, timezone + + cutoff = datetime.now(tz=timezone.utc) - timedelta(hours=max_age_hours) # Filter client-side: the sandbox's ``_status=Completed`` answered # 404 for jobs the unfiltered Bundle lists as ``completed``. for task in self.jobs(): if str(task.get("status", "")).lower() != "completed": continue + if cutoff is not None: + end_txt = (task.get("executionPeriod") or {}).get("end", "") + try: + from datetime import datetime as _dt + + ended = _dt.fromisoformat(str(end_txt).replace("Z", "+00:00")) + except ValueError: + ended = None + if ended is None or ended < cutoff: + continue ident = (task.get("identifier") or [{}])[0] job_id = ident.get("value") request = ((task.get("input") or [{}])[0]).get("valueString", "") diff --git a/src/bcda/express/cclf.py b/src/bcda/express/cclf.py index 2e5616c..14e2edc 100644 --- a/src/bcda/express/cclf.py +++ b/src/bcda/express/cclf.py @@ -100,6 +100,9 @@ FAMILIES = ("institutional", "professional", "dme", "pharmacy") _HEADER_COLS: dict[str, Any] = { "clm_id": nw.String, "clm_cntl_num": nw.String, + "clm_cntl_num_match": nw.String, + "clm_natural_key": nw.String, + "superseded_by": nw.String, "bene_id": nw.String, "pde_id": nw.String, "eob_type": nw.String, @@ -294,22 +297,33 @@ def stg_claim_header( ) -> FrameT: """Every EOB header that belongs in input_layer, tagged with its family. - 1. Drop ``NotFinalAction`` claims (v3 tag; v2 rows carry no tag). + 1. Drop ``NotFinalAction`` claims (v3 tag; v2 rows carry no tag) and + claims ``bcda.sync`` marked ``superseded_by`` (a later claim named + them as its predecessor through ``ExplanationOfBenefit.related``). 2. Derive ``claim_family`` (v3 EOB-TYPE → CLM-TYPE-CD → pde_id → v2 near-line RIC). - 3. Collapse a claim seen in two sources: rows sharing a claim - control number keep the NationalClaimsHistory / DDPS row over - the SharedSystem one, then the newest ``last_updated``. Rows - without a control number are left alone. + 3. Collapse a claim seen in two sources. The identity is, in order, + ``clm_cntl_num_match`` (the control number with the VMS leading + ``1`` stripped on SharedSystem DME, so the two versions of one DME + claim meet), ``clm_cntl_num``, then the Part D natural key for + events without a control number — bcda.cms.gov, *Matching multiple + EOBs to the same claim*. Within an identity the + NationalClaimsHistory / DDPS row wins over the SharedSystem one, + then the newest ``last_updated``. Rows with no identity are left + alone. 4. Resolve the current MBI through the xref. Operations: filter, with_columns, sort, unique, concat, join Dependencies: bcda.explanation_of_benefit, bcda._stg_beneficiary_xref """ eob = _ensure(bcda__explanation_of_benefit, _HEADER_COLS) - final = eob.filter( - nw.col("final_action").is_null() | (nw.col("final_action") == "FinalAction") - ).with_columns(_claim_family()) + final = ( + eob.filter( + nw.col("final_action").is_null() | (nw.col("final_action") == "FinalAction") + ) + .filter(nw.col("superseded_by").is_null()) + .with_columns(_claim_family()) + ) rank = ( nw.when(nw.col("system_type") == "SharedSystem") @@ -317,18 +331,28 @@ def stg_claim_header( .otherwise(nw.lit(0)) .alias("_src_rank") ) + identity = ( + nw.coalesce( + nw.col("clm_cntl_num_match"), + nw.col("clm_cntl_num"), + nw.col("clm_natural_key"), + ) + .cast(nw.String) + .alias("_claim_key") + ) + final = final.with_columns(identity) keyed = ( - final.filter(~nw.col("clm_cntl_num").is_null()) + final.filter(~nw.col("_claim_key").is_null()) .with_columns(rank) .sort( - ["clm_cntl_num", "_src_rank", "last_updated"], + ["_claim_key", "_src_rank", "last_updated"], descending=[False, False, True], ) - .unique(subset=["clm_cntl_num"], keep="first", maintain_order=True) + .unique(subset=["_claim_key"], keep="first", maintain_order=True) .drop("_src_rank") ) - unkeyed = final.filter(nw.col("clm_cntl_num").is_null()) - headers = nw.concat([keyed, unkeyed], how="vertical") + unkeyed = final.filter(nw.col("_claim_key").is_null()) + headers = nw.concat([keyed, unkeyed], how="vertical").drop("_claim_key") xref = bcda___stg_beneficiary_xref return headers.join(xref, on="bene_id", how="left") diff --git a/src/bcda/express/v3.py b/src/bcda/express/v3.py index 614f225..0074e4c 100644 --- a/src/bcda/express/v3.py +++ b/src/bcda/express/v3.py @@ -181,6 +181,21 @@ def _ext_str(extensions: list[dict] | None, code: str) -> str | None: return None if v is None else str(v) +def _identifier_type_code( + identifiers: list[dict], system_suffix: str, type_system: str +) -> str | None: + """``identifier.type.coding.code`` of the identifier whose system ends with + *system_suffix*, restricted to codings in *type_system*.""" + for ident in identifiers or []: + if str(ident.get("system", "")).endswith(system_suffix): + for c in (ident.get("type") or {}).get("coding") or []: + # the samples spell the scheme http://, the dictionary https:// + sys_ = str(c.get("system", "")) + if sys_.endswith(type_system.split("://", 1)[-1]) and c.get("code"): + return c["code"] + return None + + def _identifier(identifiers: list[dict] | None, system: str) -> str | None: for ident in identifiers or []: sys_ = ident.get("system", "") @@ -1134,6 +1149,9 @@ def flatten_eob(resource: dict) -> tuple[ExplanationOfBenefit, list[EobItem]]: kwargs["clm_srvc_prvdr_gnrc_id_num"] = _identifier( ids, GENERIC_ID ) # CLM_SRVC_PRVDR_GNRC_ID_NUM + kwargs["prvdr_srvc_id_qlfyr_cd"] = _identifier_type_code( + ids, GENERIC_ID, CS + "PRVDR-ID-QLFYR-CD" + ) # CLM_PRVDR_ID_QLFYR_CD kwargs["clm_blg_prvdr_name"] = _display_name(org.get("name")) kwargs["clm_blg_prvdr_zip5_cd"] = _get( (org.get("address") or [{}])[0], "postalCode" @@ -1185,10 +1203,28 @@ def flatten_eob(resource: dict) -> tuple[ExplanationOfBenefit, list[EobItem]]: kwargs["clm_prcdr_prfrm_dt"] = _date(procs[0].get("date")) # CLM_PRCDR_PRFRM_DT billable = resource.get("billablePeriod") or {} + dx_by_seq = _diagnosis_by_sequence(dx_list) + lines = [_flatten_item(it, clm_id, dx_by_seq) for it in items] + system_type = _tag(resource, CS_SYSTEM_TYPE) + bene_id = _ref_id(_get(resource, "patient", "reference")) header = ExplanationOfBenefit( id=resource.get("id"), + clm_cntl_num_match=match_control_number(clm_cntl_num, system_type, eob_type), + clm_natural_key=( + pde_natural_key( + bene_id, + (lines[0].clm_line_from_dt if lines else None) + or _date(billable.get("start")), + kwargs.get("prvdr_srvc_id_qlfyr_cd"), + kwargs.get("clm_srvc_prvdr_gnrc_id_num"), + kwargs.get("clm_line_rx_srvc_rfrnc_num"), + kwargs.get("clm_line_rx_fill_num"), + ) + if eob_type == "PDE" and not clm_cntl_num + else None + ), last_updated=_get(resource, "meta", "lastUpdated"), - bene_id=_ref_id(_get(resource, "patient", "reference")), # BENE_SK on claim + bene_id=bene_id, # BENE_SK on claim clm_id=clm_id, clm_uniq_id=clm_id, pde_id=clm_id if eob_type == "PDE" else None, # ccwMapping PDE_ID (Pharmacy) @@ -1196,7 +1232,7 @@ def flatten_eob(resource: dict) -> tuple[ExplanationOfBenefit, list[EobItem]]: clm_sub_type=_coding(resource.get("subType")), clm_type_cd=_coding(resource.get("type"), CS_CLM_TYPE), # CLM_TYPE_CD eob_type=eob_type, - system_type=_tag(resource, CS_SYSTEM_TYPE), + system_type=system_type, final_action=_tag(resource, CS_FINAL_ACTION), # CLM_FINL_ACTN_IND clm_src_id=_get(resource, "meta", "source"), # CLM_SRC_ID outcome=resource.get("outcome"), @@ -1210,5 +1246,60 @@ def flatten_eob(resource: dict) -> tuple[ExplanationOfBenefit, list[EobItem]]: carr_clm_cntl_num=clm_cntl_num, # ccwMapping CARR_CLM_CNTL_NUM **kwargs, ) - dx_by_seq = _diagnosis_by_sequence(dx_list) - return header, [_flatten_item(it, clm_id, dx_by_seq) for it in items] + return header, lines + + +def match_control_number( + ccn: str | None, system_type: str | None, eob_type: str | None +) -> str | None: + """The control number to *match* on across source systems. + + bcda.cms.gov, *Partially adjudicated claims data → DME claims*: when + a VMS claim is released to NCH the leading digit (a ``1``) is dropped + from ``CLM-CNTL-NUM`` (a pre-Y2K design decision), so the SharedSystem + and NationalClaimsHistory versions of one DME claim differ by that + digit. Strip it on the SharedSystem side; everything else is returned + unchanged. + """ + if not ccn: + return None + if system_type == "SharedSystem" and eob_type == "DME" and ccn.startswith("1"): + return ccn[1:] + return ccn + + +def pde_natural_key( + bene_id: str | None, + from_dt: Any, + qualifier: str | None, + service_provider: str | None, + rx: str | None, + fill: float | int | None, +) -> str | None: + """Part D identity for an event without a control number. + + CCLF Information Packet §5.1.2 (quoted on bcda.cms.gov): line item + from date, provider service identifier qualifier code, service + provider generic id, RX number, RX fill number — "for a given + patient", so the beneficiary leads. ``None`` when the RX number and + fill number are both missing (nothing to key on). + """ + if rx is None and fill is None: + return None + fill_s = ( + "" + if fill is None + else str(int(fill)) + if float(fill).is_integer() + else str(fill) + ) + return "pde:" + "|".join( + [ + bene_id or "", + from_dt.isoformat() if hasattr(from_dt, "isoformat") else (from_dt or ""), + qualifier or "", + service_provider or "", + str(rx) if rx else "", + fill_s, + ] + ) diff --git a/src/bcda/store.py b/src/bcda/store.py index 5f6c8bc..2d872d2 100644 --- a/src/bcda/store.py +++ b/src/bcda/store.py @@ -75,9 +75,31 @@ JOBS_SCHEMA = pa.schema( # and read back as NULL (parquet schema evolution). ("transaction_time", pa.string()), ("request", pa.string()), + # last_attribution_update from /attribution_status when the job was + # started (bcda.sync uses the earliest one as the _since floor). + ("attribution_update", pa.string()), ] ) + +class DownloadError(Exception): + """One or more of a job's files could not be downloaded. + + Raised after *every* file was attempted, so the files ledger holds a + row per URL (``complete`` or ``failed`` with the server's text) and + the job can be resumed; the message summarises the failures. + """ + + def __init__(self, job_key: str, failures: list[tuple[str, str]]) -> None: + self.job_key = job_key + self.failures = failures + super().__init__( + f"job {job_key}: {len(failures)} file(s) failed — " + + "; ".join(f"{name}: {err}" for name, err in failures[:5]) + + (" …" if len(failures) > 5 else "") + ) + + FILES_SCHEMA = pa.schema( [ ("ts", pa.timestamp("us", tz="UTC")), @@ -313,6 +335,48 @@ class Store: with fs.open(path, "r") as f: return json.load(f) + def job_errors(self, job_key: str) -> list[dict]: + """File rows of the job's OperationOutcome (error) files.""" + return [ + f + for f in self.get_files(job_key) + if f.get("resource_type") == "OperationOutcome" + ] + + def error_diagnostics(self, job_key: str, *, limit: int = 200) -> list[dict]: + """Issues from the job's OperationOutcome files, flattened. + + One dict per ``issue``: ``severity``, ``code``, ``diagnostics``, + ``file``. Reads only completed error files; stops at *limit*. + """ + out: list[dict] = [] + for f in self.job_errors(job_key): + if f.get("status") != "complete" or not f.get("storage_path"): + continue + name = f["storage_path"].rsplit("/", 1)[-1] + with self.open(f["storage_path"], "r") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + res = json.loads(line) + except ValueError: + res = {"issue": [{"diagnostics": line[:500]}]} + for issue in res.get("issue", []) or []: + out.append( + { + "file": name, + "severity": issue.get("severity", ""), + "code": issue.get("code", ""), + "diagnostics": issue.get("diagnostics") + or str((issue.get("details") or {}).get("text", "")), + } + ) + if len(out) >= limit: + return out + return out + def open(self, storage_path: str, mode: str = "rb"): """Open a file from the storage backend.""" return self._fs().open(storage_path, mode) @@ -329,6 +393,7 @@ class Store: type_filter: str | None = None, poll_interval: float = 10.0, cancel_retries: int = 0, + attribution_update: str = "", ) -> str: """Run a full export with persistent state tracking. @@ -358,6 +423,9 @@ class Store: cancel every in-progress job happen several times an hour; each cancellation is recorded as a ``cancelled`` transition under the same key before the next attempt. + attribution_update : str + ``last_attribution_update`` from ``/attribution_status`` at + start time, kept on the job row. """ key = _generate_key() types_str = ",".join(types) if types else "" @@ -372,6 +440,7 @@ class Store: types=types_str, since=since or "", status="pending", + attribution_update=attribution_update, ) try: @@ -390,6 +459,7 @@ class Store: types=types_str, since=since or "", status="polling", + attribution_update=attribution_update, ) # 3. Poll until complete. @@ -405,6 +475,7 @@ class Store: since=since or "", status="cancelled", error=str(exc), + attribution_update=attribution_update, ) if attempt >= cancel_retries: raise @@ -425,6 +496,7 @@ class Store: status="complete", transaction_time=result.get("transactionTime", "") or "", request=result.get("request", "") or "", + attribution_update=attribution_update, ) self._write_manifest(key, result) @@ -445,6 +517,7 @@ class Store: since=since or "", status="failed", error=str(exc), + attribution_update=attribution_update, ) raise @@ -515,11 +588,25 @@ class Store: job_key: str, result: dict, ) -> None: - """Download all output files from a completed job result.""" - outputs = result.get("output", []) - type_counts: dict[str, int] = {} + """Download every file of a completed job: data files, then error files. - for entry in outputs: + The manifest's ``error`` list holds FHIR OperationOutcome ndjson + (one ``-error.ndjson`` per worker chunk that hit errors plus + the job's ``warnings-and-info.ndjson``); they land as + ``OperationOutcome[.N].ndjson`` with ``resource_type=OperationOutcome`` + so a *partial* export is visible in the files ledger and to + ``bcda.sync``. Every URL is attempted; failures are recorded per + file and raised together as :class:`DownloadError` at the end. + """ + outputs = list(result.get("output", []) or []) + error_entries = [ + {**e, "type": e.get("type") or "OperationOutcome"} + for e in (result.get("error", []) or []) + ] + type_counts: dict[str, int] = {} + failures: list[tuple[str, str]] = [] + + for entry in [*outputs, *error_entries]: url = entry["url"] resource_type = entry.get("type", "unknown") @@ -599,7 +686,9 @@ class Store: "resource_type": resource_type, }, ) - raise + failures.append((filename, str(exc))) + if failures: + raise DownloadError(job_key, failures) # ── Resume / retry ─────────────────────────────────────── diff --git a/src/bcda/sync.py b/src/bcda/sync.py index bf7dc78..80bc6f2 100644 --- a/src/bcda/sync.py +++ b/src/bcda/sync.py @@ -68,7 +68,13 @@ TYPE_OF = {v: k for k, v in TABLE_OF.items()} KEY_EXPR = { "patient": "coalesce(bene_id, id)", "coverage": "id", - "explanation_of_benefit": "coalesce(clm_cntl_num, clm_id, id)", + # bcda.cms.gov, "Matching multiple EOBs to the same claim": the control + # number (normalised for the VMS→NCH leading digit on DME), else the + # Part D natural key, else the FHIR id. + "explanation_of_benefit": ( + "coalesce(nullif(clm_cntl_num_match, ''), nullif(clm_cntl_num, ''), " + "nullif(clm_natural_key, ''), clm_id, id)" + ), } # Beneficiary of a row, for backfill/orphan classification. BENE_EXPR = { @@ -86,10 +92,13 @@ COUNT_FIELDS = ( "late", "tombstoned", "adjusted", + "superseded", "retired", "orphans", "current", ) +# Runout data for performance year N stops being refreshed after July of N+1. +RUNOUT_LAST_MONTH = 7 _INSTANT = re.compile( r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$" @@ -164,6 +173,12 @@ class JobInfo: transaction_time: datetime transaction_source: str # job_row | manifest | started_at ts: datetime + error_files: int = 0 # OperationOutcome files in the manifest → partial export + attribution_update: datetime | None = None + + @property + def partial(self) -> bool: + return self.error_files > 0 @property def kind(self) -> str: @@ -205,6 +220,12 @@ def job_info(store: Store, job: dict) -> JobInfo: if tt is None: tt = store.job_started_at(job["key"]) or _maybe_instant(job.get("ts")) source = "started_at" + manifest = store.manifest(job["key"]) + error_files = ( + len(manifest.get("error") or []) + if manifest + else len(store.job_errors(job["key"])) + ) return JobInfo( key=job["key"], endpoint=job.get("endpoint") or "Group/all", @@ -214,6 +235,8 @@ def job_info(store: Store, job: dict) -> JobInfo: transaction_time=tt, transaction_source=source, ts=_maybe_instant(job.get("ts")) or tt, + error_files=error_files, + attribution_update=_maybe_instant(job.get("attribution_update")), ) @@ -230,10 +253,10 @@ def complete_jobs(store: Store) -> list[JobInfo]: out: list[JobInfo] = [] for job in store.list_jobs(status="complete"): files = store.get_files(job["key"]) - if files: - if any(f["status"] != "complete" for f in files): - continue - elif "output" not in store.manifest(job["key"]): + data_files = [f for f in files if f.get("resource_type") != "OperationOutcome"] + if files and any(f["status"] != "complete" for f in files): + continue + if not data_files and "output" not in store.manifest(job["key"]): continue out.append(job_info(store, job)) out.sort(key=lambda j: (j.transaction_time, j.ts, j.key)) @@ -265,6 +288,11 @@ def watermarks( continue if type_filter is not None and j.type_filter != type_filter: continue + if j.partial: + # Some enrollees' resources failed server-side (OperationOutcome + # files): the window is not fully covered, so this job must not + # become the next _since. Re-pull with the same _since. + continue for t in j.types: k = (j.population, j.type_filter, t) if k not in out or j.transaction_time >= out[k].transaction_time: @@ -461,6 +489,25 @@ def check_since( t, ) ) + floor = min( + ( + jj.attribution_update + for jj in jobs + if jj.attribution_update and jj.population == pop and t in jj.types + ), + default=None, + ) + if floor and dt < floor: + checks.append( + Check( + "warn", + "attribution_floor", + f"{t}: {since} predates the oldest attribution file this store has seen " + f"({format_instant(floor)}); the server answers 404 'attribution file not " + "found for given _since parameter' when it has no file that old", + t, + ) + ) if attribution: last_attr = None for d in attribution.get("ingestion_dates", []): @@ -476,6 +523,45 @@ def check_since( t, ) ) + if attribution: + for d in attribution.get("expiration_dates", []) or []: + if d.get("type") == "attribution_access_expiration": + exp = _maybe_instant(d.get("timestamp")) + if exp and exp < now: + checks.append( + Check( + "block", + "attribution_expired", + f"attribution access expired {format_instant(exp)}: /Group/all and " + "/Patient answer 404 'No up-to-date attribution information' until " + "the next attribution file loads (every January, about a month)", + ) + ) + if pop == "runout" and now.month > RUNOUT_LAST_MONTH: + checks.append( + Check( + "warn", + "runout_window", + f"runout for PY {now.year - 1} stopped refreshing after July {now.year}; " + "a pull now re-delivers at most what you already have", + ) + ) + partial = [ + j + for j in jobs + if j.partial and j.population == pop and set(j.types) & set(types) + ] + if partial: + checks.append( + Check( + "warn", + "partial_job", + "partial export(s) with OperationOutcome error files — their windows are not " + "fully covered and they do not advance the watermark; re-pull with the same " + "_since: " + + ", ".join(f"{j.key} ({j.error_files} file(s))" for j in partial[:6]), + ) + ) # in-flight and unapplied work for job in store.list_jobs(): if job["status"] in ("pending", "polling", "downloading"): @@ -803,9 +889,13 @@ def _recompute( for table in TABLE_OF.values(): p = _current_path(store, table) if p.exists(): + con.execute( + f"create table prev_raw_{table} as select * from read_parquet({_sql_literal(str(p))})" + ) + _ensure_columns(con, f"prev_raw_{table}", table) con.execute( f"create table prev_{table} as select {KEY_EXPR[table]} as _key, id, last_updated, " - f"{BENE_EXPR[table]} as _bene from read_parquet({_sql_literal(str(p))})" + f"{BENE_EXPR[table]} as _bene from prev_raw_{table}" ) else: con.execute( @@ -828,12 +918,15 @@ def _recompute( rep.counts[table] = counts continue con.execute(f"create or replace table cand_{table} as {sql}") + _ensure_columns(con, f"cand_{table}", table) con.execute( f"create or replace table cur_{table} as " f"select * exclude (_rn) from (select *, row_number() over " f"(partition by {KEY_EXPR[table]} order by {_rank_order(table)}) as _rn " f"from cand_{table}) where _rn = 1" ) + if table == "explanation_of_benefit": + _mark_superseded(con) staged[table] = _write_current(con, store, table) counts["current"] = con.execute(f"select count(*) from cur_{table}").fetchone()[ 0 @@ -866,8 +959,11 @@ def _recompute( staged["eob_item"] = _write_current(con, store, "eob_item") con.execute( "create or replace table cur_eob_supersession as " - "select clm_cntl_num, clm_orig_cntl_num, id, last_updated, final_action, _job_key, _transaction_time " - "from cur_explanation_of_benefit where clm_orig_cntl_num is not null and clm_orig_cntl_num <> ''" + "select clm_cntl_num, clm_cntl_num_match, clm_orig_cntl_num, id, last_updated, " + "final_action, superseded_by, _job_key, _transaction_time " + "from cur_explanation_of_benefit " + "where (clm_orig_cntl_num is not null and clm_orig_cntl_num <> '') " + "or superseded_by is not null" ) staged["eob_supersession"] = _write_current(con, store, "eob_supersession") if "explanation_of_benefit" in rep.counts: @@ -896,6 +992,65 @@ def _recompute( os.replace(tmp, _current_path(store, table)) +def _mark_superseded(con: Any) -> None: + """Set ``superseded_by`` on current claims a later claim names as predecessor. + + bcda.cms.gov: when a claim receives a new control number, the previous + one is in ``ExplanationOfBenefit.related``; institutional claims carry + only the immediate parent, so the chain is walked by marking every + named predecessor (A←B←C marks both A and B). The predecessor is + matched on its raw or normalised control number. Superseded rows + stay in the table (they are the claim's history) but the connector + drops them and they are not counted as current claims. + """ + key = KEY_EXPR["explanation_of_benefit"] + con.execute("update cur_explanation_of_benefit set superseded_by = null") + con.execute( + "create or replace temp table _succ as " + f"select {key} as successor_key, clm_orig_cntl_num as pred " + "from cur_explanation_of_benefit " + "where clm_orig_cntl_num is not null and clm_orig_cntl_num <> ''" + ) + con.execute( + "create or replace temp table _sup as " + f"select {key} as own_key, id from cur_explanation_of_benefit" + ) + con.execute( + "update cur_explanation_of_benefit e set superseded_by = s.successor_key " + "from _succ s, _sup k " + "where k.id = e.id and (e.clm_cntl_num = s.pred or e.clm_cntl_num_match = s.pred) " + "and k.own_key <> s.successor_key" + ) + + +# Columns the key expressions and supersession logic read; partitions and +# current tables flattened before they existed get them as NULLs. +_EOB_IDENTITY_COLS = ( + "clm_cntl_num", + "clm_cntl_num_match", + "clm_natural_key", + "clm_orig_cntl_num", + "superseded_by", + "clm_id", + "system_type", + "final_action", + "bene_id", +) +_PATIENT_IDENTITY_COLS = ("bene_id",) +_COVERAGE_IDENTITY_COLS = ("beneficiary_reference",) + + +def _ensure_columns(con: Any, table_name: str, table: str) -> None: + """``ALTER TABLE ADD COLUMN IF NOT EXISTS`` for the identity columns of *table*.""" + cols = { + "explanation_of_benefit": _EOB_IDENTITY_COLS, + "patient": _PATIENT_IDENTITY_COLS, + "coverage": _COVERAGE_IDENTITY_COLS, + }.get(table, ()) + for c in cols: + con.execute(f"alter table {table_name} add column if not exists {c} varchar") + + def _table_exists(con: Any, name: str) -> bool: return bool( con.execute( @@ -931,10 +1086,14 @@ def _classify( """Second-order counts for *job*'s partition of *table* against the previous current state.""" key = KEY_EXPR[table] p = _partition(store, job.key, table) + con.execute( + f"create or replace table inc_raw_{table} as select * from read_parquet({_sql_literal(str(p))})" + ) + _ensure_columns(con, f"inc_raw_{table}", table) con.execute( f"create or replace table inc_{table} as select {key} as _key, id, last_updated, " f"{BENE_EXPR[table]} as _bene, try_cast(last_updated as timestamptz) as _lu " - f"from read_parquet({_sql_literal(str(p))})" + f"from inc_raw_{table}" ) con.execute( f"create or replace table win_{table} as select {key} as _key, id, last_updated, _job_key " @@ -989,6 +1148,10 @@ def _classify( f"select count(*) from cur_{table} where _job_key = {_sql_literal(job.key)} " "and clm_orig_cntl_num is not null and clm_orig_cntl_num <> ''" ) + counts["superseded"] = q( + f"select count(*) from cur_{table} where _job_key = {_sql_literal(job.key)} " + "and superseded_by is not null" + ) def _invariants( @@ -1017,6 +1180,19 @@ def _invariants( f"{table}: count conservation failed — {prev_counts[table]} + {c['new']} new − {c['retired']} retired " f"= {expected}, but current is {c['current']}" ) + if _table_exists(con, "cur_explanation_of_benefit"): + live_pred = con.execute( + "select count(*) from cur_explanation_of_benefit e where e.superseded_by is null " + "and exists (select 1 from cur_explanation_of_benefit s where s.superseded_by is null " + "and s.clm_orig_cntl_num is not null and s.clm_orig_cntl_num <> '' " + "and (s.clm_orig_cntl_num = e.clm_cntl_num or s.clm_orig_cntl_num = e.clm_cntl_num_match) " + "and s.id <> e.id)" + ).fetchone()[0] + if live_pred: + problems.append( + f"explanation_of_benefit: {live_pred} current claim(s) still named as " + "predecessor by another current claim" + ) if _table_exists(con, "cur_eob_item") and _table_exists( con, "cur_explanation_of_benefit" ): @@ -1082,6 +1258,11 @@ def status(store: Store) -> dict[str, Any]: "since": j.since, "transaction_time": format_instant(j.transaction_time), "transaction_source": j.transaction_source, + "partial": j.partial, + "error_files": j.error_files, + "attribution_update": format_instant(j.attribution_update) + if j.attribution_update + else "", "applied": j.key in applied, } for j in jobs diff --git a/src/bcda/table/eob.py b/src/bcda/table/eob.py index 3f7a61a..fc2df01 100644 --- a/src/bcda/table/eob.py +++ b/src/bcda/table/eob.py @@ -376,6 +376,26 @@ class ExplanationOfBenefit(SQLTable): """Original claim control number for adjusted claims (related[].reference where system CLM-CNTL-NUM).""" + prvdr_srvc_id_qlfyr_cd: str | None = None + """Service-provider identifier qualifier (CLM_PRVDR_ID_QLFYR_CD, v3 + Pharmacy: 01 NPI · 07 NCPDP · 08 state licence) — part of the Part D + natural key.""" + + clm_cntl_num_match: str | None = None + """Control number as it should be *matched* across systems: a VMS + (SharedSystem DME) number loses its leading ``1`` when NCH releases + the claim, so it is stripped here. Equal to ``clm_cntl_num`` otherwise.""" + + clm_natural_key: str | None = None + """Part D fallback identity when no control number is present + (CCLF Information Packet §5.1.2): beneficiary · line from-date · + provider qualifier · service provider id · RX number · fill number.""" + + superseded_by: str | None = None + """Set by ``bcda.sync``: the match key of the later claim whose + ``ExplanationOfBenefit.related`` names this claim's control number as + its predecessor. A superseded claim is history, not a current claim.""" + system_type: str | None = None """meta.tag System-Type: SharedSystem (partially adjudicated), NationalClaimsHistory, DDPS.""" diff --git a/src/bcda/ui/service.py b/src/bcda/ui/service.py index 23f0876..4f58f20 100644 --- a/src/bcda/ui/service.py +++ b/src/bcda/ui/service.py @@ -20,6 +20,7 @@ from __future__ import annotations import importlib.util import os +import re import sys from dataclasses import dataclass from datetime import datetime, timedelta, timezone @@ -32,6 +33,7 @@ from bcda.client import ( SANDBOX, SYSTEM_TYPES, BcdaError, + BcdaHttpError, Client, type_filter_tag, ) @@ -263,15 +265,51 @@ def _endpoint_from_request(request: str) -> str: return "?" +_SERVICE_DATE_RE = re.compile(r"^\d{4}(-\d{2}(-\d{2})?)?$") +OUTCOMES = ("partial", "complete") + + +def _service_date(value: str, prefix: str, allowed: tuple[str, ...]) -> str: + """One ``service-date`` param in the server's grammar. + + ``bcda/fhir/param.go``: prefixes ``eq gt ge lt le`` (BFD supports no + others), dates ``YYYY``, ``YYYY-MM`` or ``YYYY-MM-DD``; a value may + not be a comma list; at most one earliest (``gt``/``ge``) and one + latest (``lt``/``le``) bound (``eq`` counts as both). + """ + value = value.strip() + if value[:2] in ("eq", "gt", "ge", "lt", "le"): + prefix, value = value[:2], value[2:] + if prefix not in allowed: + raise ValueError( + f"service-date prefix must be one of {allowed}, got {prefix!r}" + ) + if "," in value: + raise ValueError("service-date does not accept comma-separated values") + if not _SERVICE_DATE_RE.match(value): + raise ValueError( + f"service-date must be YYYY, YYYY-MM or YYYY-MM-DD, got {value!r}" + ) + return f"service-date={prefix}{value}" + + def build_type_filter( *, system_types: list[str] | None = None, - outcome: str | None = None, + outcome: str | list[str] | None = None, service_date_from: str | None = None, + service_date_to: str | None = None, ) -> str | None: - """Compose a v3 ``_typeFilter`` subquery from the three filters the - transition deck documents: System-Type ``_tag``, ``outcome`` - (partial / complete) and ``service-date``. None when nothing is set.""" + """Compose a v3 ``_typeFilter`` subquery in the server's grammar. + + ``_tag`` takes several System-Type tokens (``system|code,system|code``), + ``outcome`` one or both of ``partial`` / ``complete`` (comma list), + ``service-date`` an earliest bound (*service_date_from*, default + prefix ``gt``; pass ``ge2026-01`` to choose the prefix) and a latest + bound (*service_date_to*, default ``le``). ``None`` when nothing is + set. Mirrors ``bcda/fhir/param.go`` so a bad filter fails here, not + with a 400 after the job was queued. + """ params: list[str] = [] if system_types: bad = [s for s in system_types if s not in SYSTEM_TYPES] @@ -279,11 +317,21 @@ def build_type_filter( raise ValueError(f"unknown System-Type {bad}; choose from {SYSTEM_TYPES}") params.append(type_filter_tag(*system_types).split("?", 1)[1]) if outcome: - if outcome not in ("partial", "complete"): - raise ValueError("outcome must be 'partial' or 'complete'") - params.append(f"outcome={outcome}") + values = ( + [o.strip() for o in outcome.split(",")] + if isinstance(outcome, str) + else list(outcome) + ) + bad = [o for o in values if o not in OUTCOMES] + if bad or not values: + raise ValueError(f"outcome must be among {OUTCOMES}, got {values}") + params.append("outcome=" + ",".join(dict.fromkeys(values))) if service_date_from: - params.append(f"service-date=gt{service_date_from}") + params.append(_service_date(service_date_from, "gt", ("gt", "ge", "eq"))) + if service_date_to: + if service_date_from and service_date_from.strip()[:2] == "eq": + raise ValueError("an eq service-date already sets both bounds") + params.append(_service_date(service_date_to, "le", ("lt", "le"))) return "ExplanationOfBenefit?" + "&".join(params) if params else None @@ -327,6 +375,7 @@ def start_export( ] ) since = min(v for v in resolved.values() if v is not None) + attribution = _attribution(client) if since: checks = since_checks( store, @@ -335,18 +384,63 @@ def start_export( types=types, type_filter=type_filter, client=client, + attribution=attribution, ) if sync.blocking([sync.Check(**c) for c in checks]) and not force: raise SinceBlocked(checks) - return store.export( - client, - endpoint=endpoint, - types=types or None, - since=since or None, - type_filter=type_filter or None, - poll_interval=poll_interval, - cancel_retries=cancel_retries, - ) + attribution_update = "" + for d in (attribution or {}).get("ingestion_dates", []) or []: + if d.get("type") == "last_attribution_update": + attribution_update = str(d.get("timestamp") or "") + try: + return store.export( + client, + endpoint=endpoint, + types=types or None, + since=since or None, + type_filter=type_filter or None, + poll_interval=poll_interval, + cancel_retries=cancel_retries, + attribution_update=attribution_update, + ) + except BcdaHttpError as exc: + raise BcdaError(explain_server_error(exc)) from exc + + +def explain_server_error(exc: BcdaHttpError) -> str: + """The server's OperationOutcome text with the operational meaning spelled out.""" + diag = exc.diagnostics or str(exc) + low = diag.lower() + if exc.status_code == 404 and "attribution" in low: + if "_since" in low or "since parameter" in low: + return ( + f"{diag} — BCDA has no attribution file as old as that _since; use a " + "later _since (the watermark) or run a baseline without _since." + ) + return ( + f"{diag} — no usable attribution for this entity right now: access " + "expired (see attribution_status expiration_dates) or the yearly January " + "reload is in progress (about a month); runout keeps working." + ) + if exc.status_code == 400 and ( + "typefilter" in low + or "service-date" in low + or "_tag" in low + or "outcome" in low + ): + return f"{diag} — the _typeFilter was rejected by the server; see build_type_filter's rules." + if exc.status_code == 429: + return f"{diag} — rate limited (3,000 requests / 5 min per IP, or a job identical to one In Progress); honour Retry-After." + return diag + + +def _attribution(client: Client | None) -> dict | None: + if client is None: + return None + try: + return client.attribution_status() + except Exception: # noqa: BLE001 — advisory + return None def export_per_type( @@ -411,14 +505,11 @@ def since_checks( types: list[str] | None = None, type_filter: str | None = None, client: Client | None = None, + attribution: dict | None = None, ) -> list[dict]: """:func:`bcda.sync.check_since` as plain dicts, with the server's attribution status when a client is given.""" - attribution = None - if client is not None: - try: - attribution = client.attribution_status() - except Exception: # noqa: BLE001 — the check is advisory - attribution = None + if attribution is None: + attribution = _attribution(client) checks = sync.check_since( store, since, @@ -485,6 +576,9 @@ def store_jobs(store: Store) -> list[dict]: "ts": job["ts"], "files": len(files), "bytes": sum(int(f.get("file_size") or 0) for f in files), + "error_files": sum( + 1 for f in files if f.get("resource_type") == "OperationOutcome" + ), "error": job.get("error", ""), } ) @@ -492,6 +586,11 @@ def store_jobs(store: Store) -> list[dict]: return rows +def job_error_diagnostics(store: Store, key: str, *, limit: int = 200) -> list[dict]: + """OperationOutcome issues of a job's error files (a partial export).""" + return store.error_diagnostics(key, limit=limit) + + def store_files(store: Store, key: str) -> list[dict]: return store.get_files(key) diff --git a/src/bcda/ui/views.py b/src/bcda/ui/views.py index afb214d..0e34984 100644 --- a/src/bcda/ui/views.py +++ b/src/bcda/ui/views.py @@ -215,8 +215,11 @@ def exports() -> None: system_types = f1.multiselect( "System-Type _tag", list(svc.SYSTEM_TYPES), key="exp_system_types" ) - outcome = f2.selectbox( - "outcome", ["", "complete", "partial"], key="exp_outcome" + outcome = f2.multiselect( + "outcome", + list(svc.OUTCOMES), + key="exp_outcome", + help="partial = still adjudicating, complete = fully adjudicated; GUIDE has no partial claims.", ) use_svc = f3.checkbox("service-date after", key="exp_use_svc_date") svc_date = f3.date_input( @@ -225,6 +228,13 @@ def exports() -> None: key="exp_svc_date", label_visibility="collapsed", ) + use_svc_to = f3.checkbox("service-date on/before", key="exp_use_svc_date_to") + svc_date_to = f3.date_input( + "service date to", + value=date.today(), + key="exp_svc_date_to", + label_visibility="collapsed", + ) submitted = st.form_submit_button("Start export", type="primary") if submitted: @@ -233,11 +243,16 @@ def exports() -> None: "watermark": svc.SINCE_AUTO, "date": f"{since_date.isoformat()}T00:00:00+00:00", }[since_mode] - type_filter = svc.build_type_filter( - system_types=system_types or None, - outcome=outcome or None, - service_date_from=svc_date.isoformat() if use_svc else None, - ) + try: + type_filter = svc.build_type_filter( + system_types=system_types or None, + outcome=outcome or None, + service_date_from=svc_date.isoformat() if use_svc else None, + service_date_to=svc_date_to.isoformat() if use_svc_to else None, + ) + except ValueError as exc: + st.error(f"_typeFilter: {exc}") + return store, client = _store(), _client() if since and not _since_ok( store, client, since, endpoint, types, type_filter, bool(force_since) @@ -360,8 +375,20 @@ def store() -> None: .alias("size") ) st.dataframe(df.drop("bytes", "job_url"), hide_index=True, width="stretch") + partial = [j for j in jobs if j.get("error_files")] + if partial: + st.warning( + "Partial export(s) — the server wrote OperationOutcome error files, so some " + "enrollees' resources are missing and these jobs do not advance the watermark; " + "re-pull with the same _since: " + + ", ".join(f"{j['key']} ({j['error_files']})" for j in partial) + ) key = st.selectbox("Files for job", [j["key"] for j in jobs], key="store_key") files = svc.store_files(s, key) + issues = svc.job_error_diagnostics(s, key, limit=100) + if issues: + with st.expander(f"{len(issues)} OperationOutcome issue(s) for {key}"): + st.dataframe(pl.DataFrame(issues), hide_index=True, width="stretch") if files: fdf = pl.DataFrame(files).with_columns( pl.col("file_size") @@ -467,6 +494,12 @@ def sync() -> None: st.subheader("Downloaded jobs") if status["jobs"]: st.dataframe(pl.DataFrame(status["jobs"]), hide_index=True, width="stretch") + partial = [j for j in status["jobs"] if j.get("partial")] + if partial: + st.warning( + "Partial (error files, watermark not advanced): " + + ", ".join(j["key"] for j in partial) + ) c1, c2 = st.columns([1, 2]) force = c1.checkbox("re-apply everything", key="sync_force") if c1.button( @@ -497,6 +530,7 @@ def sync() -> None: "seq": r["applied_seq"], "job": r["job_key"], "kind": r["kind"], + "population": svc.sync.population_of(r["endpoint"]), "ok": r["ok"], "since": r["since"], "transaction_time": r["transaction_time"], @@ -510,6 +544,7 @@ def sync() -> None: "seq": r["applied_seq"], "job": r["job_key"], "kind": r["kind"], + "population": svc.sync.population_of(r["endpoint"]), "ok": r["ok"], "since": r["since"], "transaction_time": r["transaction_time"], diff --git a/src/cli/bcda.py b/src/cli/bcda.py index cee6151..c6e6365 100644 --- a/src/cli/bcda.py +++ b/src/cli/bcda.py @@ -127,9 +127,15 @@ def export( system_type: list[str] = typer.Option( None, "--system-type", help="_typeFilter System-Type tag(s)." ), - outcome: str = typer.Option("", help="_typeFilter outcome: partial or complete."), + outcome: list[str] = typer.Option( + None, "--outcome", help="_typeFilter outcome: partial and/or complete." + ), service_date_from: str = typer.Option( - "", help="_typeFilter service-date > YYYY-MM-DD." + "", + help="_typeFilter earliest service-date (YYYY[-MM[-DD]], optional gt/ge/eq prefix).", + ), + service_date_to: str = typer.Option( + "", help="_typeFilter latest service-date (optional lt/le prefix)." ), per_type: bool = typer.Option( True, "--per-type/--one-job", help="One job per type (survives sandbox sweeps)." @@ -147,6 +153,7 @@ def export( system_types=system_type or None, outcome=outcome or None, service_date_from=service_date_from or None, + service_date_to=service_date_to or None, ) if type_filter: typer.echo(f"_typeFilter={type_filter}") @@ -260,9 +267,14 @@ def sync_check( system_type: list[str] = typer.Option( None, "--system-type", help="_typeFilter System-Type tag(s)." ), - outcome: str = typer.Option("", help="_typeFilter outcome."), + outcome: list[str] = typer.Option( + None, "--outcome", help="_typeFilter outcome(s)." + ), service_date_from: str = typer.Option( - "", help="_typeFilter service-date > YYYY-MM-DD." + "", help="_typeFilter earliest service-date (optional gt/ge/eq prefix)." + ), + service_date_to: str = typer.Option( + "", help="_typeFilter latest service-date (optional lt/le prefix)." ), offline: bool = typer.Option( False, "--offline", help="Skip the attribution_status call." @@ -276,6 +288,7 @@ def sync_check( system_types=system_type or None, outcome=outcome or None, service_date_from=service_date_from or None, + service_date_to=service_date_to or None, ) resolved = svc.resolve_since( store, since, endpoint=endpoint, types=types or None, type_filter=type_filter @@ -351,9 +364,14 @@ def sync_pull( system_type: list[str] = typer.Option( None, "--system-type", help="_typeFilter System-Type tag(s)." ), - outcome: str = typer.Option("", help="_typeFilter outcome."), + outcome: list[str] = typer.Option( + None, "--outcome", help="_typeFilter outcome(s)." + ), service_date_from: str = typer.Option( - "", help="_typeFilter service-date > YYYY-MM-DD." + "", help="_typeFilter earliest service-date (optional gt/ge/eq prefix)." + ), + service_date_to: str = typer.Option( + "", help="_typeFilter latest service-date (optional lt/le prefix)." ), force: bool = typer.Option(False, "--force", help="Start even on block."), apply: bool = typer.Option( @@ -372,6 +390,7 @@ def sync_pull( system_types=system_type or None, outcome=outcome or None, service_date_from=service_date_from or None, + service_date_to=service_date_to or None, ) store = svc.make_store() _echo_since_plan(store, "auto", endpoint, types or None, type_filter) @@ -460,12 +479,38 @@ def store() -> None: rows = svc.store_jobs(svc.make_store()) for r in rows: + partial = ( + f" PARTIAL ({r['error_files']} error file(s))" + if r.get("error_files") + else "" + ) typer.echo( - f"{r['key']} {r['status']:<10} {r['types']:<40} {r['files']:>4} file(s) {r['bytes']:>14,} B {r['ts']}" + f"{r['key']} {r['status']:<10} {r['types']:<40} {r['files']:>4} file(s) {r['bytes']:>14,} B {r['ts']}{partial}" ) typer.echo(f"{len(rows)} job(s) in {svc.store_path()}") +@app.command() +def errors( + key: str = typer.Argument(help="Store job key."), + limit: int = typer.Option(50, help="Issues to print."), +) -> None: + """OperationOutcome issues from a job's error files (a partial export).""" + from bcda.ui import service as svc + + issues = svc.job_error_diagnostics(svc.make_store(), key, limit=limit) + if not issues: + typer.echo(f"{key}: no error files") + return + for i in issues: + typer.echo( + f" {i['severity']:<8} {i['code']:<14} {i['diagnostics']} [{i['file']}]" + ) + typer.echo( + f"{len(issues)} issue(s) shown; the job's window is not fully covered — re-pull with the same _since" + ) + + @app.command() def load( pipeline: bool = typer.Option( diff --git a/tests/bcda/test_client_v3.py b/tests/bcda/test_client_v3.py index d63b1aa..cc78f36 100644 --- a/tests/bcda/test_client_v3.py +++ b/tests/bcda/test_client_v3.py @@ -386,3 +386,45 @@ class TestInfoEndpoints: assert c.server_version()["version"] == "r302" assert c.health()["database"] == "ok" c.close() + + +class TestCompletedJobsRetention: + def test_expired_jobs_are_dropped_by_default(self) -> None: + from datetime import datetime, timedelta, timezone + + now = datetime.now(tz=timezone.utc) + fresh = (now - timedelta(hours=2)).strftime("%Y-%m-%dT%H:%M:%SZ") + stale = (now - timedelta(hours=30)).strftime("%Y-%m-%dT%H:%M:%SZ") + tasks = [ + _task( + "20", + "completed", + f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Patient", + fresh, + ), + _task( + "21", + "completed", + f"GET {SANDBOX}/api/v3/Group/all/$export?_type=Patient", + stale, + ), + ] + + def handler(req: httpx.Request) -> httpx.Response: + if "/auth/token" in str(req.url): + return _token_response() + return httpx.Response( + 200, + json={ + "resourceType": "Bundle", + "entry": [{"resource": t} for t in tasks], + }, + ) + + c = _client(handler) + assert [j["job_id"] for j in c.completed_jobs()] == ["20"] + assert [j["job_id"] for j in c.completed_jobs(max_age_hours=None)] == [ + "20", + "21", + ] + c.close() diff --git a/tests/bcda/test_store.py b/tests/bcda/test_store.py index 802c358..a7619d0 100644 --- a/tests/bcda/test_store.py +++ b/tests/bcda/test_store.py @@ -352,7 +352,9 @@ class TestExport: } client._download_file.side_effect = RuntimeError("disk full") - with pytest.raises(RuntimeError, match="disk full"): + from bcda.store import DownloadError + + with pytest.raises(DownloadError, match="disk full"): store.export(client) files = store.get_files(store.list_jobs(status="failed")[0]["key"]) @@ -887,3 +889,132 @@ class TestAdoptAndTypeFilter: "https://s/api/v3/jobs/1" ) client.close() + + +class TestErrorFiles: + """Manifest ``error`` files (OperationOutcome ndjson) make a job *partial*.""" + + def _client(self, bodies: dict[str, str | Exception]): + client = MagicMock() + client.start_export.return_value = "https://x/jobs/9" + client.poll.return_value = { + "transactionTime": "2026-10-25T00:00:00Z", + "request": "https://x/api/v3/Group/all/$export?_type=Patient", + "output": [{"url": "https://x/data/9/p.ndjson", "type": "Patient"}], + "error": [ + { + "url": "https://x/data/9/abc-error.ndjson", + "type": "OperationOutcome", + }, + { + "url": "https://x/data/9/warnings-and-info.ndjson", + "type": "OperationOutcome", + }, + ], + } + + def fake_download(url, dest): + body = bodies.get(url.rsplit("/", 1)[-1], '{"resourceType":"Patient"}\n') + if isinstance(body, Exception): + raise body + dest.write_text(body) + + client._download_file.side_effect = fake_download + return client + + def test_error_files_recorded_and_parsed(self, tmp_path) -> None: + oo = ( + '{"resourceType":"OperationOutcome","issue":[{"severity":"error","code":"exception",' + '"diagnostics":"Error retrieving ExplanationOfBenefit for beneficiary MBI 1S00…"}]}\n' + '{"resourceType":"OperationOutcome","issue":[{"severity":"warning","code":"informational",' + '"details":{"text":"3 beneficiaries skipped"}}]}\n' + ) + store = Store(tmp_path / "store") + key = store.export( + self._client({"abc-error.ndjson": oo, "warnings-and-info.ndjson": ""}), + types=["Patient"], + ) + files = store.get_files(key) + names = sorted(Path(f["storage_path"]).name for f in files) + assert names == [ + "OperationOutcome.1.ndjson", + "OperationOutcome.ndjson", + "Patient.ndjson", + ] + assert all(f["status"] == "complete" for f in files) + assert [f["resource_type"] for f in store.job_errors(key)] == [ + "OperationOutcome", + "OperationOutcome", + ] + issues = store.error_diagnostics(key) + assert [(i["severity"], i["code"]) for i in issues] == [ + ("error", "exception"), + ("warning", "informational"), + ] + assert ( + "MBI" in issues[0]["diagnostics"] + and issues[1]["diagnostics"] == "3 beneficiaries skipped" + ) + assert store.error_diagnostics(key, limit=1) == issues[:1] + assert store.manifest(key)["error"][0]["type"] == "OperationOutcome" + assert store.get_job(key)["status"] == "complete" + + def test_every_file_attempted_then_one_error(self, tmp_path) -> None: + from bcda.store import DownloadError + + store = Store(tmp_path / "store") + client = self._client( + { + "p.ndjson": RuntimeError("410 gone"), + "warnings-and-info.ndjson": RuntimeError("404"), + } + ) + with pytest.raises(DownloadError) as exc: + store.export(client, types=["Patient"]) + assert len(exc.value.failures) == 2 and "410 gone" in str(exc.value) + key = store.list_jobs(status="failed")[0]["key"] + statuses = { + Path(f["storage_path"] or f["url"]).name: f["status"] + for f in store.get_files(key) + } + assert statuses["OperationOutcome.ndjson"] == "complete" # the one that worked + assert sorted(s for s in statuses.values()) == ["complete", "failed", "failed"] + + def test_attribution_update_on_job_rows(self, tmp_path) -> None: + store = Store(tmp_path / "store") + key = store.export( + self._client({}), + types=["Patient"], + attribution_update="2026-10-01T00:00:00Z", + ) + assert store.get_job(key)["attribution_update"] == "2026-10-01T00:00:00Z" + assert store.job_started_at(key) is not None + + +def test_error_diagnostics_skips_failed_files_and_tolerates_bad_lines(tmp_path) -> None: + store = Store(tmp_path / "store") + d = tmp_path / "store" / "exports" / "K" + d.mkdir(parents=True) + good = d / "OperationOutcome.ndjson" + good.write_text( + '\n{"resourceType":"OperationOutcome","issue":[{"severity":"error","code":"x"}]}\nnot json at all\n' + ) + store._append_file( + job_key="K", + resource_type="OperationOutcome", + url="u1", + storage_path=str(good), + status="complete", + ) + store._append_file( + job_key="K", + resource_type="OperationOutcome", + url="u2", + status="failed", + error="404", + ) + issues = store.error_diagnostics("K") + assert [i["code"] for i in issues] == ["x", ""] + assert ( + issues[1]["diagnostics"] == "not json at all" and issues[0]["diagnostics"] == "" + ) diff --git a/tests/bcda/test_sync.py b/tests/bcda/test_sync.py index 91e9677..80ced3e 100644 --- a/tests/bcda/test_sync.py +++ b/tests/bcda/test_sync.py @@ -728,7 +728,8 @@ class TestEdges: "create table cur_patient as select * from (values ('A','A'), ('A','A')) t(bene_id, id)" ) con.execute( - "create table cur_explanation_of_benefit (clm_cntl_num varchar, clm_id varchar, id varchar)" + "create table cur_explanation_of_benefit (clm_cntl_num varchar, clm_cntl_num_match varchar, " + "clm_natural_key varchar, clm_orig_cntl_num varchar, superseded_by varchar, clm_id varchar, id varchar)" ) con.execute("create table cur_eob_item as select 'X' as clm_id") job = sync.complete_jobs(base_store)[0] # the Patient baseline @@ -762,3 +763,449 @@ class TestEdges: assert any( "newest baseline stays authoritative" in n for n in rep.applied[0].notes ) + + +# ── upstream review gaps (#1293–#1298) ─────────────────────── + + +def _manifest_with_errors(store: Store, key: str, tt: datetime, n: int = 1) -> None: + """Mark *key* partial: manifest lists OperationOutcome files and the files ledger has them.""" + man = store.manifest(key) + man["error"] = [ + {"type": "OperationOutcome", "url": f"https://s/data/{key}/e{i}-error.ndjson"} + for i in range(n) + ] + store._write_manifest(key, man) + d = Path(store.root) / "exports" / key + for i in range(n): + p = d / ( + "OperationOutcome.ndjson" if i == 0 else f"OperationOutcome.{i}.ndjson" + ) + p.write_text( + '{"resourceType":"OperationOutcome","issue":[{"severity":"error","code":"exception","diagnostics":"BFD timeout for bene"}]}\n' + ) + store._append_file( + job_key=key, + resource_type="OperationOutcome", + url=man["error"][i]["url"], + storage_path=str(p), + file_size=p.stat().st_size, + status="complete", + ) + + +class TestPartialJobs: + def test_partial_delta_applies_but_does_not_advance( + self, base_store: Store + ) -> None: + sync.apply(base_store) + since = T0 + timedelta(minutes=2) + delta, _ = _delta_rows(_rows("ExplanationOfBenefit"), _rows("Patient"), since) + add_job( + base_store, + "D_PART", + {"ExplanationOfBenefit": delta}, + since=sync.format_instant(since), + transaction_time=since + timedelta(days=7), + ) + _manifest_with_errors(base_store, "D_PART", since + timedelta(days=7), n=2) + info = next(j for j in sync.complete_jobs(base_store) if j.key == "D_PART") + assert info.partial and info.error_files == 2 + # its rows still reconcile (they are real), but the watermark stays at the baseline + rep = sync.apply(base_store) + assert rep.ok and rep.applied[0].counts["explanation_of_benefit"]["new"] == 3 + assert sync.next_since(base_store)[ + "ExplanationOfBenefit" + ] == sync.format_instant(since) + checks = sync.check_since( + base_store, + sync.format_instant(since), + types=["ExplanationOfBenefit"], + now=since + timedelta(days=8), + ) + codes = {c.code: c for c in checks} + assert ( + "partial_job" in codes + and "D_PART (2 file(s))" in codes["partial_job"].message + ) + st = sync.status(base_store) + row = next(j for j in st["jobs"] if j["key"] == "D_PART") + assert row["partial"] and row["error_files"] == 2 + assert ( + base_store.error_diagnostics("D_PART")[0]["diagnostics"] + == "BFD timeout for bene" + ) + + +class TestClaimIdentity: + def _eob_with( + self, + base: dict, + *, + new_id: str, + ccn: str | None, + related_to: str | None, + lu: datetime, + system: str | None = None, + ) -> dict: + e = copy.deepcopy(base) + e["id"] = new_id + e["identifier"] = [i for i in e["identifier"] if not i.get("system") == CCN] + if ccn: + e["identifier"].append({"system": CCN, "value": ccn}) + if related_to: + e["related"] = [{"reference": {"system": CCN, "value": related_to}}] + if system: + for t in e["meta"]["tag"]: + if t.get("system", "").endswith("System-Type"): + t["code"] = system + _lu(e, lu) + return e + + def test_new_control_number_supersedes_the_old_claim( + self, base_store: Store + ) -> None: + sync.apply(base_store) + eobs = _rows("ExplanationOfBenefit") + finals = [ + e + for e in eobs + if any(t.get("code") == "FinalAction" for t in e["meta"]["tag"]) + ] + a = finals[0] + a_ccn = _ccn(a) + since = T0 + timedelta(minutes=2) + b = self._eob_with( + a, + new_id=a["id"] + "-B", + ccn=a_ccn + "B", + related_to=a_ccn, + lu=since + timedelta(hours=1), + ) + c = self._eob_with( + a, + new_id=a["id"] + "-C", + ccn=a_ccn + "C", + related_to=a_ccn + "B", + lu=since + timedelta(hours=2), + ) + add_job( + base_store, + "D_SUP", + {"ExplanationOfBenefit": [b, c]}, + since=sync.format_instant(since), + transaction_time=since + timedelta(days=1), + ) + rep = sync.apply(base_store) + cnt = rep.applied[0].counts["explanation_of_benefit"] + assert ( + cnt["new"] == 2 and cnt["adjusted"] == 2 and cnt["superseded"] == 1 + ) # B is superseded by C + import duckdb + + flat = Path(base_store.root) / "flat" + con = duckdb.connect() + rows = con.execute( + f"select id, superseded_by from read_parquet('{flat / 'explanation_of_benefit.parquet'}') " + f"where clm_cntl_num in ('{a_ccn}', '{a_ccn}B', '{a_ccn}C') order by id" + ).fetchall() + by_id = dict(rows) + assert by_id[a["id"]] == a_ccn + "B" and by_id[a["id"] + "-B"] == a_ccn + "C" + assert by_id[a["id"] + "-C"] is None + sup = con.execute( + f"select count(*) from read_parquet('{flat / 'eob_supersession.parquet'}')" + ).fetchone()[0] + assert sup >= 3 + # the connector sees one live claim for the chain + import polars as pl + + from bcda.express.cclf import stg_claim_header + + hdr = pl.read_parquet(flat / "explanation_of_benefit.parquet") + xref = pl.DataFrame( + {"bene_id": hdr["bene_id"].unique().to_list()} + ).with_columns(pl.lit("M").alias("current_bene_mbi_id")) + out = stg_claim_header( + bcda__explanation_of_benefit=hdr, bcda___stg_beneficiary_xref=xref + ) + live = out.filter( + pl.col("clm_cntl_num").is_in([a_ccn, a_ccn + "B", a_ccn + "C"]) + ) + assert live["id"].to_list() == [a["id"] + "-C"] + con.close() + + def test_dme_vms_digit_matches_the_nch_claim(self, base_store: Store) -> None: + sync.apply(base_store) + eobs = _rows("ExplanationOfBenefit") + dme = next( + e + for e in eobs + if any( + c.get("code") == "DME" for c in (e.get("type") or {}).get("coding", []) + ) + and any(t.get("code") == "FinalAction" for t in e["meta"]["tag"]) + ) + since = T0 + timedelta(minutes=2) + vms = self._eob_with( + dme, + new_id=dme["id"] + "-VMS", + ccn="1" + _ccn(dme), + related_to=None, + lu=since + timedelta(hours=1), + system="SharedSystem", + ) + add_job( + base_store, + "D_VMS", + {"ExplanationOfBenefit": [vms]}, + since=sync.format_instant(since), + transaction_time=since + timedelta(days=1), + type_filter="ExplanationOfBenefit?_tag=https://bluebutton.cms.gov/fhir/CodeSystem/System-Type|SharedSystem", + ) + rep = sync.apply(base_store) + cnt = rep.applied[0].counts["explanation_of_benefit"] + # same claim, newer version: replaced, not new + assert cnt["new"] == 0 and cnt["replaced"] == 1 and cnt["current"] == 269 + import duckdb + + flat = Path(base_store.root) / "flat" + con = duckdb.connect() + row = con.execute( + f"select id, clm_cntl_num, clm_cntl_num_match, system_type from read_parquet('{flat / 'explanation_of_benefit.parquet'}') " + f"where clm_cntl_num_match = '{_ccn(dme)}'" + ).fetchall() + assert row == [(dme["id"] + "-VMS", "1" + _ccn(dme), _ccn(dme), "SharedSystem")] + con.close() + + def test_pde_natural_key(self) -> None: + from datetime import date + + from bcda.express import v3 + + assert v3.pde_natural_key( + "B1", date(2026, 1, 2), "01", "1234567890", "RX9", 2.0 + ) == ("pde:B1|2026-01-02|01|1234567890|RX9|2") + assert v3.pde_natural_key("B1", None, None, None, None, None) is None + assert ( + v3.pde_natural_key("B1", "2026-01-02", "", "", "RX9", 1.5) + == "pde:B1|2026-01-02|||RX9|1.5" + ) + assert ( + v3.match_control_number("1222222222222222", "SharedSystem", "DME") + == "222222222222222" + ) + assert ( + v3.match_control_number("1222222222222222", "NationalClaimsHistory", "DME") + == "1222222222222222" + ) + assert v3.match_control_number("1999", "SharedSystem", "CARRIER") == "1999" + assert v3.match_control_number(None, "SharedSystem", "DME") is None + + def test_legacy_current_tables_without_identity_columns( + self, base_store: Store + ) -> None: + """Current tables and partitions flattened before the identity columns existed still reconcile.""" + import duckdb + + sync.apply(base_store) + flat = Path(base_store.root) / "flat" + con = duckdb.connect() + for p in [ + flat / "explanation_of_benefit.parquet", + *flat.glob("history/*/explanation_of_benefit.parquet"), + ]: + con.execute( + f"copy (select * exclude (clm_cntl_num_match, clm_natural_key, superseded_by, prvdr_srvc_id_qlfyr_cd) " + f"from read_parquet('{p}')) to '{p}.tmp' (format parquet)" + ) + Path(f"{p}.tmp").replace(p) + con.close() + since = T0 + timedelta(minutes=2) + delta, expect = _delta_rows( + _rows("ExplanationOfBenefit"), _rows("Patient"), since + ) + add_job( + base_store, + "D_EOB", + {"ExplanationOfBenefit": delta}, + since=sync.format_instant(since), + transaction_time=since + timedelta(days=7), + ) + rep = sync.apply(base_store) + c = rep.applied[0].counts["explanation_of_benefit"] + assert {k: c[k] for k in expect} == expect + + +class TestAttributionAndRunoutChecks: + def test_attribution_expired_floor_and_runout(self, base_store: Store) -> None: + sync.apply(base_store) + now = T0 + timedelta(days=10) + attribution = { + "ingestion_dates": [ + { + "type": "last_attribution_update", + "timestamp": sync.format_instant(T0 - timedelta(days=30)), + } + ], + "expiration_dates": [ + { + "type": "attribution_access_expiration", + "timestamp": sync.format_instant(now - timedelta(days=1)), + } + ], + } + checks = sync.check_since( + base_store, + "2026-10-01T12:00:00.000+00:00", + now=now, + attribution=attribution, + types=["Patient"], + ) + levels = {c.code: c.level for c in checks} + assert levels["attribution_expired"] == "block" + # floor: a job row recorded an attribution update newer than the proposed _since + base_store._append_job( + key="B_PAT", + endpoint="Group/all", + types="Patient", + status="complete", + transaction_time=sync.format_instant(T0), + attribution_update=sync.format_instant(T0 - timedelta(days=5)), + ) + checks = sync.check_since( + base_store, + sync.format_instant(T0 - timedelta(days=6)), + now=now, + types=["Patient"], + ) + levels = {c.code: c.level for c in checks} + assert levels["attribution_floor"] == "warn" and levels["overlap"] == "ok" + # runout after July: warn (and endpoint_mix block because the store has no runout baseline) + add_job( + base_store, + "R_PAT", + {"Patient": _rows("Patient")[:3]}, + transaction_time=T0, + endpoint="Group/runout", + ) + checks = sync.check_since( + base_store, + sync.format_instant(T0), + now=datetime(2026, 10, 25, tzinfo=timezone.utc), + endpoint="Group/runout", + types=["Patient"], + ) + levels = {c.code: c.level for c in checks} + assert levels["runout_window"] == "warn" and "endpoint_mix" not in levels + checks = sync.check_since( + base_store, + sync.format_instant(T0), + now=datetime(2026, 5, 1, tzinfo=timezone.utc), + endpoint="Group/runout", + types=["Patient"], + ) + assert "runout_window" not in {c.code for c in checks} + + +class TestPartDNaturalKey: + def test_pde_without_control_number_reconciles_by_natural_key( + self, base_store: Store + ) -> None: + sync.apply(base_store) + eobs = _rows("ExplanationOfBenefit") + pde = next( + e + for e in eobs + if any( + c.get("code") == "PDE" for c in (e.get("type") or {}).get("coding", []) + ) + and any(t.get("code") == "FinalAction" for t in e["meta"]["tag"]) + ) + since = T0 + timedelta(minutes=2) + # first version: no control number at all → keyed by the natural key + v1 = copy.deepcopy(pde) + v1["id"] = pde["id"] + "-nk1" + v1["identifier"] = [i for i in v1["identifier"] if i.get("system") != CCN] + _lu(v1, since + timedelta(hours=1)) + v2 = copy.deepcopy(v1) + v2["id"] = pde["id"] + "-nk2" + _lu(v2, since + timedelta(hours=2)) + add_job( + base_store, + "D_PDE1", + {"ExplanationOfBenefit": [v1]}, + since=sync.format_instant(since), + transaction_time=since + timedelta(days=1), + ) + add_job( + base_store, + "D_PDE2", + {"ExplanationOfBenefit": [v2]}, + since=sync.format_instant(since + timedelta(days=1)), + transaction_time=since + timedelta(days=2), + ) + rep = sync.apply(base_store) + c1 = rep.applied[0].counts["explanation_of_benefit"] + c2 = rep.applied[1].counts["explanation_of_benefit"] + assert c1["new"] == 1 and c2["new"] == 0 and c2["replaced"] == 1 + assert c2["current"] == 270 + import duckdb + + flat = Path(base_store.root) / "flat" + con = duckdb.connect() + rows = con.execute( + f"select id, clm_natural_key from read_parquet('{flat / 'explanation_of_benefit.parquet'}') " + "where clm_natural_key is not null" + ).fetchall() + assert rows == [(pde["id"] + "-nk2", rows[0][1])] and rows[0][1].startswith( + "pde:" + ) + con.close() + + +class TestMoreEdges: + def test_type_claimed_but_never_delivered_anywhere(self, tmp_path: Path) -> None: + """A job row naming a type no partition carries: counts recorded, nothing to reconcile.""" + store = Store(path=str(tmp_path / "s")) + add_job(store, "B1", {"Patient": _rows("Patient")[:5]}, transaction_time=T0) + # rewrite the job row to claim Coverage too + store._append_job( + key="B1", + endpoint="Group/all", + types="Patient,Coverage", + status="complete", + transaction_time=sync.format_instant(T0), + ) + rep = sync.apply(store) + c = rep.applied[0].counts + assert ( + c["patient"]["current"] == 5 + and c["coverage"]["current"] == 0 + and c["coverage"]["rows"] == 0 + ) + + def test_invariant_live_predecessor(self, base_store: Store) -> None: + import duckdb + + con = duckdb.connect() + con.execute( + "create table cur_explanation_of_benefit (clm_cntl_num varchar, clm_cntl_num_match varchar, " + "clm_natural_key varchar, clm_orig_cntl_num varchar, superseded_by varchar, clm_id varchar, id varchar)" + ) + con.execute( + "insert into cur_explanation_of_benefit values ('A','A',null,null,null,'a','a'), ('B','B',null,'A',null,'b','b')" + ) + job = sync.complete_jobs(base_store)[0] + rep = sync.JobReport( + job_key=job.key, + kind="baseline", + endpoint="Group/all", + type_filter="", + types=["Patient"], + since="", + transaction_time="", + counts={}, + ) + with pytest.raises(sync.SyncInvariantError, match="still named as predecessor"): + sync._invariants(con, base_store, job, rep, {}, {}) + con.close() diff --git a/tests/bcda/test_ui_service.py b/tests/bcda/test_ui_service.py index bf23415..99cf2ec 100644 --- a/tests/bcda/test_ui_service.py +++ b/tests/bcda/test_ui_service.py @@ -484,7 +484,7 @@ class TestLocalSide: status = svc.vendor_status() assert status["available"] and status["problems"] == [] names = {s["name"] for s in status["sources"]} - assert names == {"bcda-app", "bcda-site"} + assert names == {"bcda-app", "bcda-site", "bcda-static-site"} assert any(f["path"].endswith("swagger.json") for f in status["files"]) mod = svc._vendor_module() monkeypatch.setattr(mod, "check", lambda m: ["bcda-app: swagger.json changed"]) @@ -610,3 +610,136 @@ class TestSince: assert st["watermarks"] == [] and st["pending"] == [] and st["current"] == {} rep = svc.apply_sync() assert rep == {"ok": True, "applied": [], "skipped": []} + + +# ── _typeFilter grammar + server error texts ───────────────── + + +class TestTypeFilterGrammar: + def test_outcome_list_and_service_date_bounds(self) -> None: + assert svc.build_type_filter(outcome=["partial", "complete"]) == ( + "ExplanationOfBenefit?outcome=partial,complete" + ) + assert svc.build_type_filter(outcome="complete,partial,complete") == ( + "ExplanationOfBenefit?outcome=complete,partial" + ) + assert svc.build_type_filter(service_date_from="2026-01") == ( + "ExplanationOfBenefit?service-date=gt2026-01" + ) + assert svc.build_type_filter( + service_date_from="ge2026", service_date_to="lt2027" + ) == ("ExplanationOfBenefit?service-date=ge2026&service-date=lt2027") + assert svc.build_type_filter(service_date_to="2026-06-30") == ( + "ExplanationOfBenefit?service-date=le2026-06-30" + ) + assert ( + svc.build_type_filter( + system_types=["NationalClaimsHistory", "DDPS"], + outcome="complete", + service_date_from="2026-01-01", + ).count("&") + == 2 + ) + + def test_rejections_mirror_the_server(self) -> None: + with pytest.raises(ValueError, match="outcome"): + svc.build_type_filter(outcome="done") + with pytest.raises(ValueError, match="comma"): + svc.build_type_filter(service_date_from="2026-01-01,2026-02-01") + with pytest.raises(ValueError, match="YYYY"): + svc.build_type_filter(service_date_from="01/01/2026") + with pytest.raises(ValueError, match="prefix"): + svc.build_type_filter(service_date_to="gt2026-01-01") + with pytest.raises(ValueError, match="eq service-date"): + svc.build_type_filter( + service_date_from="eq2026-01-01", service_date_to="2026-12-31" + ) + + def test_explain_server_error(self) -> None: + from bcda.client import BcdaHttpError + + def err(status: int, diag: str) -> BcdaHttpError: + resp = httpx.Response( + status, + json={ + "resourceType": "OperationOutcome", + "issue": [{"diagnostics": diag}], + }, + request=httpx.Request("GET", "https://s/x"), + ) + return BcdaHttpError(resp, diag) + + assert "later _since" in svc.explain_server_error( + err( + 404, + "failed to start job; attribution file not found for given _since parameter.", + ) + ) + assert "January" in svc.explain_server_error( + err( + 404, "No up-to-date attribution information is available for ACOID 'A1'" + ) + ) + assert "_typeFilter was rejected" in svc.explain_server_error( + err( + 400, + "invalid service-date parameter value: conflicting prefix conditions", + ) + ) + assert "Retry-After" in svc.explain_server_error(err(429, "rate limit")) + assert svc.explain_server_error(err(500, "boom")) == "boom" + + def test_start_export_records_attribution_and_explains_404(self, tmp_path) -> None: + calls: list[str] = [] + + def handler(req: httpx.Request) -> httpx.Response: + url = str(req.url) + calls.append(url) + if "/auth/token" in url: + return _token() + if "/attribution_status" in url: + return httpx.Response( + 200, + json={ + "ingestion_dates": [ + { + "type": "last_attribution_update", + "timestamp": "2026-10-01T00:00:00Z", + } + ], + "expiration_dates": [], + }, + ) + if "$export" in url: + return httpx.Response( + 404, + json={ + "resourceType": "OperationOutcome", + "issue": [ + { + "diagnostics": "No up-to-date attribution information is available" + } + ], + }, + ) + return httpx.Response(200, json={}) + + client = _client(handler, tmp_path=tmp_path) + store = Store(path=str(tmp_path)) + from bcda.client import BcdaError + + with pytest.raises(BcdaError, match="January") as exc: + svc.start_export(store, client, types=["Patient"], poll_interval=0) + assert not isinstance(exc.value, svc.SinceBlocked) + job = store.list_jobs()[0] + assert ( + job["status"] == "failed" + and job["attribution_update"] == "2026-10-01T00:00:00Z" + ) + client.close() + + def test_job_error_diagnostics(self, tmp_path) -> None: + store = Store(path=str(tmp_path)) + assert svc.job_error_diagnostics(store, "NOPE") == [] + rows = svc.store_jobs(store) + assert rows == [] diff --git a/tests/bcda/test_ui_views.py b/tests/bcda/test_ui_views.py index d342b71..50bc8ce 100644 --- a/tests/bcda/test_ui_views.py +++ b/tests/bcda/test_ui_views.py @@ -164,7 +164,7 @@ class TestExports: at = _page("exports").run() at.multiselect(key="exp_types").set_value(["Patient"]) at.multiselect(key="exp_system_types").set_value(["DDPS"]) - at.selectbox(key="exp_outcome").set_value("complete") + at.multiselect(key="exp_outcome").set_value(["complete"]) at.radio(key="exp_since_mode").set_value("date") # an empty store has no baseline: the checks block and nothing starts _button(at, "Start export").click().run() @@ -183,6 +183,29 @@ class TestExports: ].endswith("T00:00:00+00:00") assert calls[0]["force"] is True assert any("_typeFilter=" in c.value for c in at.caption) + # a filter the builder rejects stops before anything starts + monkeypatch.setattr( + svc, + "build_type_filter", + lambda **kw: (_ for _ in ()).throw(ValueError("conflicting prefix")), + ) + n_runs = len(fresh_runner.all()) + _button(at, "Start export").click().run() + assert any("_typeFilter: conflicting prefix" in e.value for e in at.error) + assert len(fresh_runner.all()) == n_runs + monkeypatch.undo() + monkeypatch.setattr( + svc, + "export_per_type", + lambda store, client, **kw: ( + calls.append({"per_type": True, **kw}) or {"Patient": "K1"} + ), + ) + monkeypatch.setattr( + svc, + "start_export", + lambda store, client, **kw: calls.append({"per_type": False, **kw}) or "K2", + ) # watermark mode hands the service the sentinel at.radio(key="exp_since_mode").set_value("watermark") _button(at, "Start export").click().run() @@ -272,6 +295,31 @@ class TestExports: class TestStoreAndLoad: + def test_partial_job_warning_and_diagnostics(self, tmp_store, tmp_path) -> None: + d = Path(tmp_store.root) / "exports" / "P1" + d.mkdir(parents=True) + oo = d / "OperationOutcome.ndjson" + oo.write_text( + '{"resourceType":"OperationOutcome","issue":[{"severity":"error","code":"exception","diagnostics":"BFD 502"}]}\n' + ) + tmp_store._append_file( + job_key="P1", + resource_type="OperationOutcome", + url="u", + storage_path=str(oo), + file_size=1, + status="complete", + ) + tmp_store._append_job( + key="P1", endpoint="Group/all", types="Patient", status="complete" + ) + at = _page("store").run() + assert not at.exception + assert any( + "Partial export" in w.value and "P1 (1)" in w.value for w in at.warning + ) + assert any("OperationOutcome issue" in e.label for e in at.expander) + def test_store_empty_and_populated(self, tmp_store, tmp_path) -> None: at = _page("store").run() assert any("No store jobs" in i.value for i in at.info) @@ -463,9 +511,22 @@ class TestSync: t0 = datetime(2026, 10, 1, tzinfo=timezone.utc) add_job(tmp_store, "B1", {"Patient": _rows("Patient")[:5]}, transaction_time=t0) + from .test_sync import _manifest_with_errors + + add_job( + tmp_store, + "P2", + {"Patient": _rows("Patient")[5:6]}, + since="2026-10-01T00:00:00+00:00", + transaction_time=t0, + ) + _manifest_with_errors(tmp_store, "P2", t0) at = _page("sync").run() assert not at.exception - assert any("1 pending" in b.label for b in at.button) + assert any( + "Partial (error files" in w.value and "P2" in w.value for w in at.warning + ) + assert any("2 pending" in b.label for b in at.button) assert any("nothing applied" in c.value for c in at.caption) calls: list = [] monkeypatch.setattr( @@ -473,11 +534,11 @@ class TestSync: "load_and_publish", lambda **kw: calls.append(kw) or {"bcda.patient": 5}, ) - _button(at, "Reconcile now (1 pending)").click().run() + _button(at, "Reconcile now (2 pending)").click().run() run = fresh_runner.wait(fresh_runner.all()[0].id) - assert run.state == "done" and run.result["applied"] == ["B1"] + assert run.state == "done" and run.result["applied"] == ["B1", "P2"] assert calls == [{"run_pipeline": True, "skip_flatten": True}] - assert sync.read_ledger(tmp_store)[0]["job_key"] == "B1" + assert [r["job_key"] for r in sync.read_ledger(tmp_store)] == ["B1", "P2"] # the ledger renders, and the inflated page still has no pending work at = _page("sync").run() assert not at.exception @@ -487,7 +548,7 @@ class TestSync: sync._append_ledger( tmp_store, { - "applied_seq": 2, + "applied_seq": 3, "job_key": "BAD", "kind": "delta", "endpoint": "Group/all", diff --git a/tests/cli/test_cli_bcda.py b/tests/cli/test_cli_bcda.py index 0d83b1e..b2b5e1a 100644 --- a/tests/cli/test_cli_bcda.py +++ b/tests/cli/test_cli_bcda.py @@ -453,3 +453,75 @@ def test_sync_apply_and_pull(monkeypatch) -> None: monkeypatch.setattr(svc, "export_per_type", refuse) result = runner.invoke(app, ["bcda", "sync", "pull", "-t", "Patient"]) assert result.exit_code == 2 and "✗ gap" in result.output + + +def test_errors_command_and_filter_options(monkeypatch) -> None: + monkeypatch.setattr(svc, "make_store", lambda path=None: _FakeStore()) + monkeypatch.setattr(svc, "job_error_diagnostics", lambda store, key, limit=50: []) + result = runner.invoke(app, ["bcda", "errors", "K1"]) + assert result.exit_code == 0 and "no error files" in result.output + monkeypatch.setattr( + svc, + "job_error_diagnostics", + lambda store, key, limit=50: [ + { + "severity": "error", + "code": "exception", + "diagnostics": "BFD 500 for bene", + "file": "OperationOutcome.ndjson", + } + ], + ) + result = runner.invoke(app, ["bcda", "errors", "K1"]) + assert ( + result.exit_code == 0 + and "BFD 500 for bene" in result.output + and "re-pull" in result.output + ) + + monkeypatch.setattr( + svc, + "store_jobs", + lambda s: [ + { + "key": "K1", + "status": "complete", + "types": "Patient", + "files": 3, + "bytes": 1, + "ts": "t", + "error_files": 2, + } + ], + ) + monkeypatch.setattr(svc, "store_path", lambda: "/tmp") + result = runner.invoke(app, ["bcda", "store"]) + assert "PARTIAL (2 error file(s))" in result.output + + calls = [] + monkeypatch.setattr( + svc, + "export_per_type", + lambda store, client, **kw: calls.append(kw) or {"ExplanationOfBenefit": "K"}, + ) + result = runner.invoke( + app, + [ + "bcda", + "export", + "-t", + "ExplanationOfBenefit", + "--outcome", + "partial", + "--outcome", + "complete", + "--service-date-from", + "ge2026-01", + "--service-date-to", + "2026-06-30", + ], + ) + assert result.exit_code == 0, result.output + assert calls[0]["type_filter"] == ( + "ExplanationOfBenefit?outcome=partial,complete&service-date=ge2026-01&service-date=le2026-06-30" + )