7e53d9360a
dev/vendor/MANIFEST.toml pins bcda-app@7decdcc (v3 handler + tests, request validation middleware, OperationOutcome writer, v3 Postman collection, gzip fixtures, sandbox config, LICENSE) and the bcda.cms.gov v3 data dictionary, v2 dictionary and official v3 sample ndjson. vendor_bcda_app.py --verify | --check | --sync (stdlib only); the bcda-vendor workflow runs --check weekly and files an issue on drift — nothing auto-commits, a payload change upstream is something a human reads before the flattener is re-pointed. gen_bcda_v3_crosswalk.py writes dev/seeds/bcda_v3_data_dictionary.csv (fhirPath, IDR sourceColumn, ccwMapping, cclfMapping per v3 field), bcda_v3_cclf_crosswalk.csv (293 CCLF elements -> v3 field) and bcda_v3_cclf_absent.csv (52 CCLF fields v3 does not carry); the CCLF sheet has label/name swapped on data rows and is normalised. Tests assert the committed seeds match the vendored workbook. refs #1139
246 lines
8.9 KiB
Python
246 lines
8.9 KiB
Python
"""dev/scripts/vendor_bcda_app.py and the vendored tree it maintains."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "dev" / "scripts" / "vendor_bcda_app.py"
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def vendor():
|
|
spec = importlib.util.spec_from_file_location("vendor_bcda_app", SCRIPT)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
sys.modules["vendor_bcda_app"] = mod
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
class TestVendoredTree:
|
|
def test_tree_matches_manifest(self, vendor) -> None:
|
|
"""Offline: every manifest entry is present with its recorded sha256."""
|
|
assert vendor.verify(vendor.load_manifest()) == []
|
|
|
|
def test_manifest_pins_what_tests_rely_on(self, vendor) -> None:
|
|
m = vendor.load_manifest()
|
|
by_name = {s["name"]: s for s in m["source"]}
|
|
app = by_name["bcda-app"]
|
|
assert app["kind"] == "github" and app["repo"] == "CMSgov/bcda-app"
|
|
assert len(app["commit"]) == 40
|
|
for f in (
|
|
"LICENSE",
|
|
"bcda/api/v3/api.go",
|
|
"bcda/api/v3/api_test.go",
|
|
"bcda/api/requests.go",
|
|
"bcda/responseutils/v3/writer.go",
|
|
"test/postman_test/BCDA_V3_Tests.postman_collection.json",
|
|
"shared_files/gzip_feature_test/test_gzip_encoded.ndjson",
|
|
):
|
|
assert f in app["files"]
|
|
site = by_name["bcda-site"]
|
|
assert site["kind"] == "https" and site["base"].startswith(
|
|
"https://bcda.cms.gov/"
|
|
)
|
|
for f in (
|
|
"BCDA_v3_Data_Dictionary.xlsx",
|
|
"v3_Patient.ndjson",
|
|
"v3_Coverage.ndjson",
|
|
"v3_ExplanationOfBenefit.ndjson",
|
|
):
|
|
assert f in site["files"]
|
|
|
|
def test_v3_postman_collection_covers_our_endpoints(self) -> None:
|
|
"""The upstream smoke suite exercises every v3 endpoint the client calls."""
|
|
pm = json.loads(
|
|
(
|
|
ROOT
|
|
/ "dev/vendor/bcda-app/test/postman_test/BCDA_V3_Tests.postman_collection.json"
|
|
).read_text()
|
|
)
|
|
urls: list[str] = []
|
|
|
|
def walk(items):
|
|
for it in items:
|
|
if "item" in it:
|
|
walk(it["item"])
|
|
else:
|
|
u = it.get("request", {}).get("url")
|
|
urls.append(u.get("raw", "") if isinstance(u, dict) else str(u))
|
|
|
|
walk(pm["item"])
|
|
joined = "\n".join(urls)
|
|
for needle in (
|
|
"/api/v3/metadata",
|
|
"/api/v3/attribution_status",
|
|
"/api/v3/Patient/$export",
|
|
"/api/v3/Group/runout/$export",
|
|
"/api/v3/jobs?_status=Completed",
|
|
"/_version",
|
|
"/_health",
|
|
):
|
|
assert needle in joined
|
|
|
|
def test_v3_handler_headers_match_client(self) -> None:
|
|
"""bcda-app's request validation requires exactly the headers we send."""
|
|
src = (
|
|
ROOT / "dev/vendor/bcda-app/bcda/web/middleware/validation.go"
|
|
).read_text()
|
|
assert "respond-async" in src
|
|
assert "application/fhir+json" in src
|
|
assert "_typeFilter" in src
|
|
requests_go = (ROOT / "dev/vendor/bcda-app/bcda/api/requests.go").read_text()
|
|
assert "TypeFilter" in requests_go
|
|
|
|
def test_license_present(self) -> None:
|
|
assert (
|
|
"Public Domain" in (ROOT / "dev/vendor/bcda-app/LICENSE").read_text()
|
|
or (ROOT / "dev/vendor/bcda-app/LICENSE").stat().st_size > 0
|
|
)
|
|
|
|
|
|
class TestManifestIO:
|
|
def test_dump_load_roundtrip(self, vendor, tmp_path) -> None:
|
|
m = vendor.load_manifest()
|
|
out = tmp_path / "MANIFEST.toml"
|
|
vendor.dump_manifest(m, out)
|
|
again = vendor.load_manifest(out)
|
|
assert again["source"] == m["source"]
|
|
assert again["sha256"] == m["sha256"]
|
|
|
|
def test_git_blob_sha(self, vendor) -> None:
|
|
# `git hash-object` of an empty file
|
|
assert vendor._git_blob_sha(b"") == "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
|
|
|
|
|
|
def _manifest(tmp_vendor: Path) -> dict:
|
|
return {
|
|
"source": [
|
|
{
|
|
"name": "app",
|
|
"kind": "github",
|
|
"repo": "org/repo",
|
|
"ref": "main",
|
|
"commit": "old",
|
|
"files": ["a.txt", "dir/b.txt"],
|
|
},
|
|
{
|
|
"name": "site",
|
|
"kind": "https",
|
|
"base": "https://example.test/",
|
|
"files": ["c.bin"],
|
|
},
|
|
],
|
|
"sha256": {},
|
|
}
|
|
|
|
|
|
class TestCheckAndSync:
|
|
@pytest.fixture
|
|
def fake_upstream(self, vendor, tmp_path, monkeypatch):
|
|
"""Patch network + paths; returns a mutable dict describing upstream."""
|
|
state = {
|
|
"head": "new",
|
|
"trees": {
|
|
"old": {"a.txt": "sha-a1", "dir/b.txt": "sha-b1"},
|
|
"new": {
|
|
"a.txt": "sha-a1",
|
|
"dir/b.txt": "sha-b2",
|
|
"dir/extra.txt": "sha-x",
|
|
},
|
|
},
|
|
"raw": {"a.txt": b"A", "dir/b.txt": b"B2"},
|
|
"https": {"c.bin": b"CCC"},
|
|
"truncated": False,
|
|
}
|
|
|
|
def gh_json(url: str) -> dict:
|
|
if "/commits/" in url:
|
|
return {"sha": state["head"]}
|
|
commit = url.rsplit("/trees/", 1)[1].split("?")[0]
|
|
return {
|
|
"truncated": state["truncated"],
|
|
"tree": [
|
|
{"path": p, "sha": s, "type": "blob"}
|
|
for p, s in state["trees"][commit].items()
|
|
],
|
|
}
|
|
|
|
def get(url: str, *, accept=None) -> bytes:
|
|
if url.startswith("https://raw.githubusercontent.com/"):
|
|
return state["raw"][url.split("/", 6)[-1]]
|
|
return state["https"][url.rsplit("/", 1)[1]]
|
|
|
|
monkeypatch.setattr(vendor, "_gh_json", gh_json)
|
|
monkeypatch.setattr(vendor, "_get", get)
|
|
monkeypatch.setattr(vendor, "VENDOR", tmp_path / "vendor")
|
|
monkeypatch.setattr(vendor, "MANIFEST", tmp_path / "vendor" / "MANIFEST.toml")
|
|
(tmp_path / "vendor").mkdir()
|
|
vendor.dump_manifest(
|
|
_manifest(tmp_path / "vendor"), tmp_path / "vendor" / "MANIFEST.toml"
|
|
)
|
|
return state
|
|
|
|
def test_check_reports_each_kind_of_drift(self, vendor, fake_upstream) -> None:
|
|
m = vendor.load_manifest()
|
|
drift = vendor.check(m)
|
|
assert any("dir/b.txt changed" in d for d in drift)
|
|
assert any("new upstream file dir/extra.txt" in d for d in drift)
|
|
assert any("c.bin changed" in d for d in drift) # never synced → sha mismatch
|
|
assert not any("a.txt" in d and "changed" in d for d in drift)
|
|
|
|
def test_check_removed_and_quiet(self, vendor, fake_upstream) -> None:
|
|
fake_upstream["trees"]["new"].pop("a.txt")
|
|
drift = vendor.check(vendor.load_manifest())
|
|
assert any("a.txt removed upstream" in d for d in drift)
|
|
# pin == head and https digest matches → no drift
|
|
m = vendor.load_manifest()
|
|
m["source"][0]["commit"] = "new"
|
|
import hashlib
|
|
|
|
m["sha256"]["site/c.bin"] = hashlib.sha256(b"CCC").hexdigest()
|
|
assert vendor.check(m) == []
|
|
|
|
def test_sync_writes_files_and_manifest(self, vendor, fake_upstream) -> None:
|
|
m = vendor.load_manifest()
|
|
changed = vendor.sync(m)
|
|
assert sorted(changed) == ["app/a.txt", "app/dir/b.txt", "site/c.bin"]
|
|
assert (vendor.VENDOR / "app" / "dir" / "b.txt").read_bytes() == b"B2"
|
|
m2 = vendor.load_manifest()
|
|
assert m2["source"][0]["commit"] == "new"
|
|
assert vendor.verify(m2) == []
|
|
# second sync: nothing changes
|
|
assert vendor.sync(vendor.load_manifest()) == []
|
|
|
|
def test_truncated_tree_raises(self, vendor, fake_upstream) -> None:
|
|
fake_upstream["truncated"] = True
|
|
with pytest.raises(RuntimeError, match="truncated"):
|
|
vendor.check(vendor.load_manifest())
|
|
|
|
def test_verify_reports_missing_and_mismatch(self, vendor, fake_upstream) -> None:
|
|
m = vendor.load_manifest()
|
|
problems = vendor.verify(m)
|
|
assert any(p.startswith("missing app/a.txt") for p in problems)
|
|
vendor.sync(m)
|
|
(vendor.VENDOR / "site" / "c.bin").write_bytes(b"tampered")
|
|
problems = vendor.verify(vendor.load_manifest())
|
|
assert len(problems) == 1 and "sha256 mismatch site/c.bin" in problems[0]
|
|
|
|
def test_main_modes(self, vendor, fake_upstream, capsys) -> None:
|
|
assert vendor.main(["--verify"]) == 1 # nothing synced yet
|
|
assert vendor.main(["--check"]) == 1
|
|
assert vendor.main(["--sync"]) == 0
|
|
assert vendor.main(["--verify"]) == 0
|
|
assert vendor.main(["--check"]) == 0
|
|
out = capsys.readouterr().out
|
|
assert (
|
|
"vendor tree OK" in out
|
|
and "no upstream drift" in out
|
|
and "synced; 3 file(s) changed" in out
|
|
)
|