Some checks failed
CI / lint (push) Successful in 1m25s
CI / notebooks-smoke (push) Successful in 2m11s
Deploy / changes (push) Successful in 22s
Infra CI / notebooks (push) Successful in 1m0s
CI / test (push) Failing after 5m34s
Infra CI / zotero (push) Successful in 31s
Infra CI / docs (push) Successful in 54s
Infra CI / api (push) Successful in 2m5s
Infra CI / bcda (push) Successful in 2m21s
Infra CI / llm (push) Successful in 1m30s
Infra CI / mc (push) Successful in 35s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 2m14s
Deploy / bcda (push) Successful in 2m35s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m47s
Deploy / report (push) Successful in 16s
Harden / build-scan-report (push) Successful in 37m31s
Comment Pull / comment-pull (push) Failing after 1m2s
Notebooks Integration / notebooks-integration (push) Failing after 11m47s
LLM Golden / llm-golden (push) Successful in 9m47s
Renovate / renovate (push) Successful in 1m56s
Zotero Sync / zotero-sync (push) Successful in 2m56s
BCDA Vendor / bcda-vendor (push) Failing after 41s
Package Supply Chain / pkg-supply-chain (push) Successful in 1m23s
180 lines
5.9 KiB
Python
180 lines
5.9 KiB
Python
import json
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from drugs import http as dh
|
|
from market import http as mh
|
|
|
|
|
|
class _NoWait:
|
|
def wait(self):
|
|
pass
|
|
|
|
|
|
@pytest.fixture
|
|
def root(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
|
|
dh._MANIFESTS.clear()
|
|
return tmp_path
|
|
|
|
|
|
def _client(body, status=200, seen=None):
|
|
def handler(req):
|
|
if seen is not None:
|
|
seen.append(req)
|
|
return httpx.Response(
|
|
status, content=body[0] if isinstance(body, list) else body
|
|
)
|
|
|
|
return httpx.Client(transport=httpx.MockTransport(handler))
|
|
|
|
|
|
def _fetch(cl, dest, **kw):
|
|
return dh.fetch(
|
|
cl, "https://x.test/a.csv", dest, provider="cms", source="cms-asp", **kw
|
|
)
|
|
|
|
|
|
def test_pace_and_pacer():
|
|
assert dh.PACE == {"cms": 1.0, "fda": 1.0, "cdc": 1.0, "mcd": 1.0}
|
|
p = dh.pacer("fda")
|
|
assert isinstance(p, mh.Pacer) and p.min_interval == 1.0
|
|
|
|
|
|
def test_data_root_and_path(monkeypatch, tmp_path):
|
|
import conf
|
|
|
|
monkeypatch.setattr(conf, "path", lambda k: tmp_path / "aco.duckdb")
|
|
assert dh.data_root() == tmp_path
|
|
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
|
|
assert dh.path("drugs", "a") == tmp_path / "drugs" / "a"
|
|
|
|
|
|
def test_fetch_hoards_and_reuses(root, monkeypatch):
|
|
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
|
|
seen: list = []
|
|
dest = root / "drugs" / "asp" / "a.csv"
|
|
cl = _client(b"one", seen=seen)
|
|
out = _fetch(cl, dest, note="n")
|
|
assert out == dest and dest.read_bytes() == b"one"
|
|
mf = root / "drugs" / "MANIFEST.jsonl"
|
|
lines = mf.read_text().splitlines()
|
|
assert len(lines) == 1
|
|
rec = json.loads(lines[0])
|
|
assert rec["url"] == "https://x.test/a.csv"
|
|
assert rec["path"] == "drugs/asp/a.csv" and rec["bytes"] == 3
|
|
assert rec["note"] == "n" and len(rec["sha256"]) == 64
|
|
# reuse: no request
|
|
assert _fetch(cl, dest) == dest
|
|
assert len(seen) == 1
|
|
# refresh with different bytes lands beside
|
|
cl2 = _client(b"two", seen=seen)
|
|
out2 = _fetch(cl2, dest, refresh=True)
|
|
assert out2 != dest and out2.suffix == ".csv" and out2.stem.startswith("a.")
|
|
assert out2.read_bytes() == b"two" and dest.read_bytes() == b"one"
|
|
assert len(mf.read_text().splitlines()) == 2
|
|
# newest copy is returned on reuse
|
|
assert _fetch(cl2, dest) == out2
|
|
# same bytes refreshed again: still two lines
|
|
_fetch(cl2, dest, refresh=True)
|
|
assert len(mf.read_text().splitlines()) == 2
|
|
|
|
|
|
def test_manifest_dedupes_across_processes_and_omits_empty_note(root):
|
|
p = root / "drugs" / "f.txt"
|
|
p.parent.mkdir(parents=True)
|
|
p.write_bytes(b"z")
|
|
dh.manifest(p, "u")
|
|
dh._MANIFESTS.clear()
|
|
dh.manifest(p, "u")
|
|
lines = (root / "drugs" / "MANIFEST.jsonl").read_text().splitlines()
|
|
assert len(lines) == 1 and "note" not in json.loads(lines[0])
|
|
|
|
|
|
def test_fetch_403_raises_refused(root, monkeypatch):
|
|
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
|
|
dest = root / "drugs" / "a.csv"
|
|
with pytest.raises(mh.Refused) as ei:
|
|
_fetch(_client(b"no", status=403), dest)
|
|
assert ei.value.status == 403
|
|
assert not dest.exists() and not (root / "drugs" / "MANIFEST.jsonl").exists()
|
|
|
|
|
|
def test_user_agent(root, monkeypatch):
|
|
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
|
|
seen: list = []
|
|
cl = mh.client("cms")
|
|
cl._transport = httpx.MockTransport(
|
|
lambda req: seen.append(req) or httpx.Response(200, content=b"x")
|
|
)
|
|
_fetch(cl, root / "drugs" / "u.csv")
|
|
assert (
|
|
seen[0].headers["user-agent"] == "stack-pfs-dossier/1.0 (ci@mail.fhirworx.io)"
|
|
)
|
|
|
|
|
|
class _Store:
|
|
def __init__(self):
|
|
self.items: dict[str, tuple] = {}
|
|
self.attached = 0
|
|
|
|
def upsert(self, item, *, tags=None, collection=""):
|
|
key = self.items.setdefault(item.url, (f"K{len(self.items)}", item, tags))[0]
|
|
return key
|
|
|
|
def attach_file(self, *a, **k):
|
|
self.attached += 1
|
|
|
|
|
|
def test_cite_file_dedupes_by_url_no_attachment():
|
|
st = _Store()
|
|
kw = dict(
|
|
url="https://x.test/a.csv", title="ASP", provider="cms", table="drugs.asp"
|
|
)
|
|
k1 = dh.cite_file(st, **kw)
|
|
k2 = dh.cite_file(st, **kw)
|
|
assert k1 == k2 and len(st.items) == 1 and st.attached == 0
|
|
_, item, tags = st.items["https://x.test/a.csv"]
|
|
assert item.item_type == "source"
|
|
assert {(t.namespace, t.value) for t in tags} == {
|
|
("module", "drugs"),
|
|
("source", "cms"),
|
|
("table", "drugs.asp"),
|
|
}
|
|
|
|
|
|
def test_link_superseded_links_both_ways_and_backfills(root):
|
|
d = root / "cms" / "asp"
|
|
d.mkdir(parents=True)
|
|
old, new = d / "asp_2026-Q2.zip", d / "asp_2026-Q2.ab12cd34.zip"
|
|
old.write_bytes(b"old")
|
|
new.write_bytes(b"new")
|
|
dh.manifest(new, "https://x.test/B.zip")
|
|
dh.link_superseded(old, new, "https://x.test/A.zip")
|
|
dh.link_superseded(old, new, "https://x.test/A.zip") # idempotent
|
|
recs = [
|
|
json.loads(x) for x in (root / "drugs/MANIFEST.jsonl").read_text().splitlines()
|
|
]
|
|
by_path = {}
|
|
for r in recs:
|
|
by_path.setdefault(r["path"], []).append(r)
|
|
(o,) = by_path["cms/asp/asp_2026-Q2.zip"]
|
|
assert o["url"] == "https://x.test/A.zip"
|
|
assert o["superseded_by"] == "cms/asp/asp_2026-Q2.ab12cd34.zip"
|
|
n = by_path["cms/asp/asp_2026-Q2.ab12cd34.zip"]
|
|
assert len(n) == 2 and n[0]["url"] == n[1]["url"] == "https://x.test/B.zip"
|
|
assert n[1]["supersedes"] == "cms/asp/asp_2026-Q2.zip"
|
|
# a later manifest() of the backfilled file adds nothing
|
|
dh.manifest(old, "https://x.test/A.zip")
|
|
assert len((root / "drugs/MANIFEST.jsonl").read_text().splitlines()) == 3
|
|
# an old file that already has a line keeps its url
|
|
dh._MANIFESTS.clear()
|
|
third = d / "asp_2026-Q2.ee00ee00.zip"
|
|
third.write_bytes(b"third")
|
|
dh.link_superseded(new, third)
|
|
last = json.loads((root / "drugs/MANIFEST.jsonl").read_text().splitlines()[-2])
|
|
assert (
|
|
last["path"].endswith("ab12cd34.zip") and last["url"] == "https://x.test/B.zip"
|
|
)
|