Files
stack/tests/drugs/test_http.py
kert aac023cc29
Some checks failed
CI / lint (push) Successful in 1m25s
CI / notebooks-smoke (push) Successful in 2m11s
Deploy / changes (push) Successful in 22s
Infra CI / notebooks (push) Successful in 1m0s
CI / test (push) Failing after 5m34s
Infra CI / zotero (push) Successful in 31s
Infra CI / docs (push) Successful in 54s
Infra CI / api (push) Successful in 2m5s
Infra CI / bcda (push) Successful in 2m21s
Infra CI / llm (push) Successful in 1m30s
Infra CI / mc (push) Successful in 35s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 2m14s
Deploy / bcda (push) Successful in 2m35s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m47s
Deploy / report (push) Successful in 16s
Harden / build-scan-report (push) Successful in 37m31s
Comment Pull / comment-pull (push) Failing after 1m2s
Notebooks Integration / notebooks-integration (push) Failing after 11m47s
LLM Golden / llm-golden (push) Successful in 9m47s
Renovate / renovate (push) Successful in 1m56s
Zotero Sync / zotero-sync (push) Successful in 2m56s
BCDA Vendor / bcda-vendor (push) Failing after 41s
Package Supply Chain / pkg-supply-chain (push) Successful in 1m23s
fix(drugs): re-posted ASP files link to what they supersede in the manifest; 8-word guard over served drugs text; ndc11 whitespace tolerance documented
2026-11-13 04:39:35 -05:00

180 lines
5.9 KiB
Python

import json
import httpx
import pytest
from drugs import http as dh
from market import http as mh
class _NoWait:
def wait(self):
pass
@pytest.fixture
def root(monkeypatch, tmp_path):
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
dh._MANIFESTS.clear()
return tmp_path
def _client(body, status=200, seen=None):
def handler(req):
if seen is not None:
seen.append(req)
return httpx.Response(
status, content=body[0] if isinstance(body, list) else body
)
return httpx.Client(transport=httpx.MockTransport(handler))
def _fetch(cl, dest, **kw):
return dh.fetch(
cl, "https://x.test/a.csv", dest, provider="cms", source="cms-asp", **kw
)
def test_pace_and_pacer():
assert dh.PACE == {"cms": 1.0, "fda": 1.0, "cdc": 1.0, "mcd": 1.0}
p = dh.pacer("fda")
assert isinstance(p, mh.Pacer) and p.min_interval == 1.0
def test_data_root_and_path(monkeypatch, tmp_path):
import conf
monkeypatch.setattr(conf, "path", lambda k: tmp_path / "aco.duckdb")
assert dh.data_root() == tmp_path
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
assert dh.path("drugs", "a") == tmp_path / "drugs" / "a"
def test_fetch_hoards_and_reuses(root, monkeypatch):
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
seen: list = []
dest = root / "drugs" / "asp" / "a.csv"
cl = _client(b"one", seen=seen)
out = _fetch(cl, dest, note="n")
assert out == dest and dest.read_bytes() == b"one"
mf = root / "drugs" / "MANIFEST.jsonl"
lines = mf.read_text().splitlines()
assert len(lines) == 1
rec = json.loads(lines[0])
assert rec["url"] == "https://x.test/a.csv"
assert rec["path"] == "drugs/asp/a.csv" and rec["bytes"] == 3
assert rec["note"] == "n" and len(rec["sha256"]) == 64
# reuse: no request
assert _fetch(cl, dest) == dest
assert len(seen) == 1
# refresh with different bytes lands beside
cl2 = _client(b"two", seen=seen)
out2 = _fetch(cl2, dest, refresh=True)
assert out2 != dest and out2.suffix == ".csv" and out2.stem.startswith("a.")
assert out2.read_bytes() == b"two" and dest.read_bytes() == b"one"
assert len(mf.read_text().splitlines()) == 2
# newest copy is returned on reuse
assert _fetch(cl2, dest) == out2
# same bytes refreshed again: still two lines
_fetch(cl2, dest, refresh=True)
assert len(mf.read_text().splitlines()) == 2
def test_manifest_dedupes_across_processes_and_omits_empty_note(root):
p = root / "drugs" / "f.txt"
p.parent.mkdir(parents=True)
p.write_bytes(b"z")
dh.manifest(p, "u")
dh._MANIFESTS.clear()
dh.manifest(p, "u")
lines = (root / "drugs" / "MANIFEST.jsonl").read_text().splitlines()
assert len(lines) == 1 and "note" not in json.loads(lines[0])
def test_fetch_403_raises_refused(root, monkeypatch):
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
dest = root / "drugs" / "a.csv"
with pytest.raises(mh.Refused) as ei:
_fetch(_client(b"no", status=403), dest)
assert ei.value.status == 403
assert not dest.exists() and not (root / "drugs" / "MANIFEST.jsonl").exists()
def test_user_agent(root, monkeypatch):
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
seen: list = []
cl = mh.client("cms")
cl._transport = httpx.MockTransport(
lambda req: seen.append(req) or httpx.Response(200, content=b"x")
)
_fetch(cl, root / "drugs" / "u.csv")
assert (
seen[0].headers["user-agent"] == "stack-pfs-dossier/1.0 (ci@mail.fhirworx.io)"
)
class _Store:
def __init__(self):
self.items: dict[str, tuple] = {}
self.attached = 0
def upsert(self, item, *, tags=None, collection=""):
key = self.items.setdefault(item.url, (f"K{len(self.items)}", item, tags))[0]
return key
def attach_file(self, *a, **k):
self.attached += 1
def test_cite_file_dedupes_by_url_no_attachment():
st = _Store()
kw = dict(
url="https://x.test/a.csv", title="ASP", provider="cms", table="drugs.asp"
)
k1 = dh.cite_file(st, **kw)
k2 = dh.cite_file(st, **kw)
assert k1 == k2 and len(st.items) == 1 and st.attached == 0
_, item, tags = st.items["https://x.test/a.csv"]
assert item.item_type == "source"
assert {(t.namespace, t.value) for t in tags} == {
("module", "drugs"),
("source", "cms"),
("table", "drugs.asp"),
}
def test_link_superseded_links_both_ways_and_backfills(root):
d = root / "cms" / "asp"
d.mkdir(parents=True)
old, new = d / "asp_2026-Q2.zip", d / "asp_2026-Q2.ab12cd34.zip"
old.write_bytes(b"old")
new.write_bytes(b"new")
dh.manifest(new, "https://x.test/B.zip")
dh.link_superseded(old, new, "https://x.test/A.zip")
dh.link_superseded(old, new, "https://x.test/A.zip") # idempotent
recs = [
json.loads(x) for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
]
by_path = {}
for r in recs:
by_path.setdefault(r["path"], []).append(r)
(o,) = by_path["cms/asp/asp_2026-Q2.zip"]
assert o["url"] == "https://x.test/A.zip"
assert o["superseded_by"] == "cms/asp/asp_2026-Q2.ab12cd34.zip"
n = by_path["cms/asp/asp_2026-Q2.ab12cd34.zip"]
assert len(n) == 2 and n[0]["url"] == n[1]["url"] == "https://x.test/B.zip"
assert n[1]["supersedes"] == "cms/asp/asp_2026-Q2.zip"
# a later manifest() of the backfilled file adds nothing
dh.manifest(old, "https://x.test/A.zip")
assert len((root / "drugs/MANIFEST.jsonl").read_text().splitlines()) == 3
# an old file that already has a line keeps its url
dh._MANIFESTS.clear()
third = d / "asp_2026-Q2.ee00ee00.zip"
third.write_bytes(b"third")
dh.link_superseded(new, third)
last = json.loads((root / "drugs/MANIFEST.jsonl").read_text().splitlines()[-2])
assert (
last["path"].endswith("ab12cd34.zip") and last["url"] == "https://x.test/B.zip"
)