Files
stack/tests/cli/test_drugs_cli.py
kert aac023cc29
Some checks failed
CI / lint (push) Successful in 1m25s
CI / notebooks-smoke (push) Successful in 2m11s
Deploy / changes (push) Successful in 22s
Infra CI / notebooks (push) Successful in 1m0s
CI / test (push) Failing after 5m34s
Infra CI / zotero (push) Successful in 31s
Infra CI / docs (push) Successful in 54s
Infra CI / api (push) Successful in 2m5s
Infra CI / bcda (push) Successful in 2m21s
Infra CI / llm (push) Successful in 1m30s
Infra CI / mc (push) Successful in 35s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 2m14s
Deploy / bcda (push) Successful in 2m35s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m47s
Deploy / report (push) Successful in 16s
Harden / build-scan-report (push) Successful in 37m31s
Comment Pull / comment-pull (push) Failing after 1m2s
Notebooks Integration / notebooks-integration (push) Failing after 11m47s
LLM Golden / llm-golden (push) Successful in 9m47s
Renovate / renovate (push) Successful in 1m56s
Zotero Sync / zotero-sync (push) Successful in 2m56s
BCDA Vendor / bcda-vendor (push) Failing after 41s
Package Supply Chain / pkg-supply-chain (push) Successful in 1m23s
fix(drugs): re-posted ASP files link to what they supersede in the manifest; 8-word guard over served drugs text; ndc11 whitespace tolerance documented
2026-11-13 04:39:35 -05:00

547 lines
18 KiB
Python

"""stack drugs — status."""
from __future__ import annotations
import io
import json
import zipfile
import duckdb
import httpx
import pytest
from typer.testing import CliRunner
import cli.drugs as dg
from bib.store import Store
from cli import app
from drugs import asp as A
from drugs import http as dh
from drugs import tables as T
from market import tables as MT
runner = CliRunner()
class _Batch:
def __init__(self, c):
self.c = c
def __enter__(self):
return self.c
def __exit__(self, *a):
return False
@pytest.fixture
def env(monkeypatch):
c = duckdb.connect(":memory:")
monkeypatch.setattr(dg, "_batch", lambda: _Batch(c))
monkeypatch.setattr(dg, "_read", lambda: c.cursor())
published: list[bool] = []
monkeypatch.setattr(dg, "_publish", lambda: published.append(True))
yield c, published
c.close()
def test_status_prints_counts_and_gaps(env):
c, _ = env
T.ensure_tables(c)
c.execute("CREATE TABLE drugs.probe (a INTEGER)")
c.execute("INSERT INTO drugs.probe VALUES (1), (2)")
MT.write_gap(c, "cms-asp:asp", "quarter 2026-Q4 missing")
MT.write_gap(c, "tiingo:prices", "no key")
res = runner.invoke(app, ["drugs", "status"])
assert res.exit_code == 0, res.output
assert "drugs.probe: 2" in res.output
assert "gap: cms-asp:asp: quarter 2026-Q4 missing" in res.output
assert "tiingo" not in res.output
def test_status_without_tables_or_gaps(env, monkeypatch):
c, _ = env
res = runner.invoke(app, ["drugs", "status"])
assert res.exit_code == 0 and "no drugs tables yet" in res.output
T.ensure_tables(c)
res = runner.invoke(app, ["drugs", "status"])
assert "no gaps recorded" in res.output
def test_status_missing_market_table_and_errors(env, monkeypatch):
c, _ = env
T.ensure_tables(c)
res = runner.invoke(app, ["drugs", "status"])
assert res.exit_code == 0
class _Broken:
def execute(self, *a, **k):
raise RuntimeError("x")
def close(self):
pass
monkeypatch.setattr(dg, "_read", lambda: _Broken())
assert isinstance(runner.invoke(app, ["drugs", "status"]).exception, RuntimeError)
def test_gap_helper_writes_only_gap(env):
c, published = env
dg._gap("cms-asp:asp", "x", write=False)
assert MT.read_gaps(c) == [] if _has_gap_table(c) else True
assert published == []
dg._gap("cms-asp:asp", "x", write=True)
assert [g.source for g in MT.read_gaps(c)] == ["cms-asp:asp"]
assert published == [True]
assert (
c.execute("SELECT count(*) FROM cms.ingest_log").fetchone() == (0,)
if _has_log(c)
else True
)
def _has_gap_table(c):
return bool(
c.execute(
"SELECT 1 FROM information_schema.tables WHERE table_name='source_gap'"
).fetchall()
)
def _has_log(c):
return bool(
c.execute(
"SELECT 1 FROM information_schema.tables WHERE table_name='ingest_log'"
).fetchall()
)
def test_default_indirections_resolve_lazily(monkeypatch, capsys):
import conf.connect as cc
calls: list[str] = []
monkeypatch.setattr(cc, "duckdb_batch", lambda n: calls.append(f"batch {n}") or n)
monkeypatch.setattr(
cc,
"duckdb",
lambda n, read_only=False: calls.append(f"read {n} {read_only}") or n,
)
monkeypatch.setattr(cc, "bib", lambda: calls.append("bib") or "store")
monkeypatch.setattr(cc, "publish_replica", lambda n: f"/r/{n}")
assert dg._batch() == "aco" and dg._read() == "aco" and dg._store() == "store"
dg._publish()
assert calls == ["batch aco", "read aco True", "bib"]
assert "replica → /r/aco" in capsys.readouterr().out
def test_status_gap_read_error_propagates(env, monkeypatch):
c, _ = env
T.ensure_tables(c)
class _Con:
def execute(self, sql, *a):
if "source_gap" in sql:
raise RuntimeError("boom")
return c.execute(sql, *a)
def close(self):
pass
monkeypatch.setattr(dg, "_read", lambda: _Con())
assert isinstance(runner.invoke(app, ["drugs", "status"]).exception, RuntimeError)
# ── stack drugs asp ──────────────────────────────────────────────────
PRICING = (
"Payment Allowance Limits,,,,,,,,,,,\r\n"
"HCPCS Code,Short Description,HCPCS Code Dosage,Payment Limit,"
"Co-insurance Percentage,Vaccine AWP%,Vaccine Limit,Blood AWP%,Blood limit,"
"Clotting Factor,Notes\r\n"
"90371,x,1 ML,140.209,20.000,,,,,,\r\n"
"J9035,Bevacizumab injection,10 MG,66.123,20.000,,,,,,\r\n"
)
NP = (
"HCPCS Code,Long Description,Manufacturer\r\nJ7330,Chondrocytes implant,Vericel\r\n"
)
NOC = "Drug Generic Name (Trade Name),Dosage,Payment Limit,Notes\r\nExamplumab,10 MG,12.5,\r\n"
def _zip(members):
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
for n, b in members.items():
zf.writestr(n, b)
return buf.getvalue()
PAGE = """<html><body>
<a href="/files/zip/july-2026-medicare-part-b-payment-limit-files.zip">July 2026 Medicare Part B Payment Limit Files</a>
<a href="/files/zip/july-2026-ndc-hcpcs-crosswalk.zip">July 2026 NDC-HCPCS Crosswalk</a>
<a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/x/downloads/2018-April-ASP-Pricing-File.zip">April 2018 ASP Pricing File</a>
<a href="https://www.cms.gov/apps/ama/license.asp?file=/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip">April 2018 NOC Pricing File</a>
<a href="/files/zip/2026-2027-seasonal-vaccine-price-file.zip">2026-2027 Seasonal Vaccine Price File</a>
</body></html>"""
ROUTES = {
"/medicare/payment/part-b-drugs/asp-pricing-files": PAGE.encode(),
"/files/zip/july-2026-medicare-part-b-payment-limit-files.zip": _zip(
{
"section 508 version of July 2026 Payment Limit File.csv": PRICING,
"section 508 version of July 2026 Drugs Not Payable.csv": NP,
}
),
"/files/zip/july-2026-ndc-hcpcs-crosswalk.zip": _zip({"x.csv": "a"}),
"/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip": _zip({"NOC.csv": NOC}),
"/Medicare/x/downloads/2018-April-ASP-Pricing-File.zip": _zip({"ASP.csv": PRICING}),
}
class _NoWait:
def wait(self):
pass
@pytest.fixture
def asp_env(env, monkeypatch, tmp_path):
c, published = env
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
dh._MANIFESTS.clear()
store = Store(":memory:", storage_dir=tmp_path / "st")
monkeypatch.setattr(dg, "_store", lambda: store)
seen: list[httpx.Request] = []
def handler(req):
seen.append(req)
body = ROUTES.get(req.url.path)
return (
httpx.Response(200, content=body)
if body is not None
else httpx.Response(404)
)
monkeypatch.setattr(
dh,
"client",
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
)
monkeypatch.setattr(A, "today", lambda: __import__("datetime").date(2026, 8, 1))
return c, published, seen, tmp_path, store
def test_asp_cli_fetch_write(asp_env):
c, published, seen, root, store = asp_env
MT.write_gap(c, "cms-asp:asp", "stale")
res = runner.invoke(
app,
[
"drugs",
"asp",
"--fetch",
"--write",
"--quarter",
"2026-Q3",
"--quarter",
"2018-Q2",
],
)
assert res.exit_code == 0, res.output
pay = [r for r in T.read_asp(c) if r.quarter == "2026-Q3"]
assert {r.quarter for r in T.read_asp(c)} == {"2018-Q2", "2026-Q3"}
assert [(r.quarter, r.hcpcs, r.list_kind) for r in pay] == [
("2026-Q3", "90371", "payment"),
("2026-Q3", "J7330", "not-payable"),
("2026-Q3", "J9035", "payment"),
]
assert pay[0].short_desc == "" and pay[2].short_desc == "Bevacizumab injection"
assert pay[1].manufacturer == "Vericel"
assert all(
r.item_key
and r.source_url.startswith("https://www.cms.gov/files/zip/july-2026")
for r in pay
)
noc = T.read_noc(c)
assert [(n.quarter, n.drug_name, n.payment_limit) for n in noc] == [
("2018-Q2", "Examplumab", 12.5)
]
assert (
noc[0].source_url
== "https://www.cms.gov/Medicare/x/downloads/2018-April-NOC-Pricing-File.zip"
)
assert {
k
for (k,) in c.execute(
"SELECT DISTINCT table_name FROM cms.ingest_log"
).fetchall()
} == {
"drugs.asp_payment",
"drugs.asp_noc",
}
assert published == [True]
assert [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"] == []
assert "stale" not in " ".join(g.note for g in MT.read_gaps(c))
# every request carried the honest user agent through the real client factory
hoard = root / "cms" / "asp"
assert (hoard / "asp_2026-Q3.zip").exists()
assert (hoard / "noc" / "noc_2018-Q2.zip").exists()
assert (hoard / "xwalk" / "xwalk_2026-Q3.zip").exists()
assert (root / "drugs" / "cms" / "asp-pricing-files.html").exists()
manifest = [
json.loads(x) for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
]
assert len(manifest) == 5
# vaccine zip is never fetched
assert not any("vaccine" in str(r.url) for r in seen)
assert "2026-Q3" in res.output and "not-payable" in res.output
cites = [i for i in store.list_items() if "ASP" in i.title]
assert len(cites) == 3
# a second run fetches nothing but the page
n = len(seen)
again = runner.invoke(
app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3", "--quarter", "2018-Q2"]
)
assert again.exit_code == 0, again.output
assert len(seen) == n + 1
def test_asp_cli_gap_note_written_for_missing(asp_env):
c, published, seen, root, store = asp_env
res = runner.invoke(
app,
[
"drugs",
"asp",
"--fetch",
"--write",
"--quarter",
"2026-Q3",
"--quarter",
"2026-Q2",
],
)
assert res.exit_code == 0, res.output
gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"]
assert len(gaps) == 1 and "2026-Q2 pricing" in gaps[0].note
assert "gap: cms-asp:asp: no hoarded file for: 2026-Q2 pricing" in res.output
def test_asp_cli_dry_run_lists_links(asp_env):
c, published, seen, root, store = asp_env
first = runner.invoke(app, ["drugs", "asp", "--fetch", "--dry-run"])
assert first.exit_code == 0, first.output
assert [r.url.path for r in seen] == [
"/medicare/payment/part-b-drugs/asp-pricing-files"
]
out = first.output
assert (
"pricing 2026-Q3 fetch https://www.cms.gov/files/zip/july-2026-medicare-part-b-payment-limit-files.zip"
in out
)
assert "noc 2018-Q2 fetch" in out and "crosswalk 2026-Q3 fetch" in out
assert "vaccine" not in out
assert not (root / "cms").exists()
assert T.table_counts(c) == {}
assert published == []
# --write is ignored by a dry run
res = runner.invoke(
app, ["drugs", "asp", "--dry-run", "--write", "--quarter", "2018-Q2"]
)
assert res.exit_code == 0 and published == [] and "noc 2018-Q2 fetch" in res.output
assert "crosswalk 2026-Q3" not in res.output
def test_asp_cli_refusal_prints_gap(asp_env, monkeypatch):
c, published, seen, root, store = asp_env
monkeypatch.setattr(
dh,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(lambda r: httpx.Response(403))
),
)
res = runner.invoke(app, ["drugs", "asp", "--fetch"])
assert res.exit_code == 0, res.output
lines = [x for x in res.output.splitlines() if x.strip()]
assert len(lines) == 1 and lines[0].startswith("gap: cms-asp:asp:")
assert published == []
assert T.table_counts(c) == {} and not (root / "cms").exists()
res = runner.invoke(app, ["drugs", "asp", "--fetch", "--write"])
assert [g.source for g in MT.read_gaps(c)] == ["cms-asp:asp"]
assert published == [True]
assert T.table_counts(c) == {}
def test_asp_cli_refusal_on_a_zip_stops_fetching(asp_env, monkeypatch):
c, published, seen, root, store = asp_env
def handler(req):
if req.url.path.endswith("asp-pricing-files"):
return httpx.Response(200, content=PAGE.encode())
return httpx.Response(403)
monkeypatch.setattr(
dh,
"client",
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
)
res = runner.invoke(
app, ["drugs", "asp", "--fetch", "--write", "--quarter", "2026-Q3"]
)
assert res.exit_code == 0, res.output
assert "gap: cms-asp:asp: refused" in res.output
assert "2026-Q3 pricing" in res.output
assert T.read_asp(c) == []
def test_asp_cli_parse_only_reads_hoard_without_cites(asp_env):
c, published, seen, root, store = asp_env
assert (
runner.invoke(
app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3"]
).exit_code
== 0
)
assert T.table_counts(c) == {}
seen.clear()
res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q3"])
assert res.exit_code == 0 and seen == []
assert "2026-Q3" in res.output and "3 payment" not in res.output
assert "2 payment" in res.output and "1 not-payable" in res.output
assert store.list_items() == [] and published == []
def test_asp_cli_unreadable_zip_is_a_gap(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp"
d.mkdir(parents=True)
(d / "asp_2026-Q3.zip").write_bytes(b"not a zip")
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
assert res.exit_code == 0, res.output
gaps = [g for g in MT.read_gaps(c) if g.source == "cms-asp:asp"]
assert gaps and "2026-Q3 pricing unreadable" in gaps[0].note
assert T.read_asp(c) == []
def test_asp_cli_source_url_falls_back_to_csv(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp"
d.mkdir(parents=True)
(d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING}))
(root / "cms" / "asp_source_urls.csv").write_text(
"quarter,url\n2026-Q3,https://cms.test/canon.zip\n"
)
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
assert res.exit_code == 0, res.output
assert {r.source_url for r in T.read_asp(c)} == {"https://cms.test/canon.zip"}
assert all(r.item_key for r in T.read_asp(c))
def test_asp_cli_no_url_no_cite(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp"
d.mkdir(parents=True)
(d / "asp_2026-Q3.zip").write_bytes(_zip({"ASP Pricing File.csv": PRICING}))
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2026-Q3"])
assert res.exit_code == 0, res.output
assert {(r.source_url, r.item_key) for r in T.read_asp(c)} == {("", "")}
def test_asp_cli_refresh_refetches(asp_env):
c, published, seen, root, store = asp_env
assert (
runner.invoke(
app, ["drugs", "asp", "--fetch", "--quarter", "2018-Q2"]
).exit_code
== 0
)
n = len(seen)
ROUTES_BEFORE = dict(ROUTES)
res = runner.invoke(
app, ["drugs", "asp", "--fetch", "--refresh", "--quarter", "2018-Q2"]
)
assert (
res.exit_code == 0 and len(seen) == n + 3
) # page, the pricing zip and the NOC zip
assert ROUTES == ROUTES_BEFORE
def test_asp_cli_unfiltered_uses_expected_quarters(asp_env):
c, published, seen, root, store = asp_env
res = runner.invoke(app, ["drugs", "asp"])
assert res.exit_code == 0, res.output
assert "gap: cms-asp:asp: no hoarded file for: 2005-Q1 pricing" in res.output
assert "(+" in res.output and published == []
assert seen == []
def test_asp_today_is_real_date():
import datetime
assert A.today() == datetime.date.today()
def test_asp_cli_rejects_bad_quarter(asp_env):
res = runner.invoke(app, ["drugs", "asp", "--quarter", "2026-Q5"])
assert res.exit_code != 0 and "not a quarter" in res.output
def test_asp_cli_unreadable_noc_zip_is_a_gap(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp" / "noc"
d.mkdir(parents=True)
(d / "noc_2018-Q2.zip").write_bytes(b"not a zip")
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2018-Q2"])
assert res.exit_code == 0, res.output
assert "2018-Q2 noc unreadable" in res.output
assert T.read_noc(c) == []
def test_asp_cli_bundled_noc_from_pricing_zip(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp"
d.mkdir(parents=True)
(d / "asp_2006-Q2.zip").write_bytes(
_zip({"ASPbyHCPCS.csv": PRICING, "ASPNOC.csv": NOC})
)
res = runner.invoke(app, ["drugs", "asp", "--write", "--quarter", "2006-Q2"])
assert res.exit_code == 0, res.output
assert [n.drug_name for n in T.read_noc(c)] == ["Examplumab"]
assert "1 noc" in res.output
def test_asp_cli_dry_run_without_page(asp_env):
res = runner.invoke(app, ["drugs", "asp", "--dry-run"])
assert res.exit_code == 0 and "no page hoarded; run with --fetch" in res.output
def test_asp_cli_noc_only_quarter_is_listed(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp" / "noc"
d.mkdir(parents=True)
(d / "noc_2018-Q2.zip").write_bytes(_zip({"NOC.csv": NOC}))
res = runner.invoke(app, ["drugs", "asp", "--quarter", "2018-Q2"])
assert res.exit_code == 0, res.output
assert "2018-Q2: 1 noc" in res.output
def test_asp_cli_fetch_links_a_repost_to_what_it_supersedes(asp_env):
c, published, seen, root, store = asp_env
d = root / "cms" / "asp"
d.mkdir(parents=True)
old = d / "asp_2026-Q3.zip"
old.write_bytes(
_zip({"ASP Pricing File.csv": PRICING + "J0001,Extra,1 MG,1.0,20,,,,,,\r\n"})
)
(root / "cms" / "asp_source_urls.csv").write_text(
"quarter,url\n2026-Q3,https://cms.test/old.zip\n"
)
res = runner.invoke(app, ["drugs", "asp", "--fetch", "--quarter", "2026-Q3"])
assert res.exit_code == 0, res.output
recs = [
json.loads(x) for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
]
o = next(r for r in recs if r["path"] == "cms/asp/asp_2026-Q3.zip")
assert o["url"] == "https://cms.test/old.zip" and o["superseded_by"].startswith(
"cms/asp/asp_2026-Q3."
)
assert any(r.get("supersedes") == "cms/asp/asp_2026-Q3.zip" for r in recs)