Some checks failed
CI / lint (push) Successful in 57s
CI / notebooks-smoke (push) Successful in 2m45s
Deploy / changes (push) Successful in 18s
Infra CI / notebooks (push) Successful in 1m14s
CI / test (push) Failing after 5m26s
Infra CI / zotero (push) Successful in 21s
Infra CI / docs (push) Successful in 1m35s
Infra CI / api (push) Successful in 1m22s
Infra CI / llm (push) Successful in 1m14s
Infra CI / mc (push) Successful in 36s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
239 lines
7.4 KiB
Python
239 lines
7.4 KiB
Python
"""market.http: keys in headers only, one retry loop, pacer, hoard helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from market import http
|
|
from market.http import Pacer, Refused, api_key, client, get, key_headers, slug
|
|
|
|
|
|
def _cl(handler, **headers):
|
|
return httpx.Client(transport=httpx.MockTransport(handler), headers=headers)
|
|
|
|
|
|
class _Pacer:
|
|
def wait(self):
|
|
pass
|
|
|
|
|
|
def test_api_key_falls_back_to_api_data_gov():
|
|
env = {"API_DATA_GOV_KEY": "k1"}
|
|
assert api_key("fec", env) == api_key("congress", env) == "k1"
|
|
assert api_key("fec", {**env, "FEC_API_KEY": "k2"}) == "k2"
|
|
assert api_key("tiingo", {}) is None
|
|
assert api_key("nobody", {"X": "1"}) is None
|
|
|
|
|
|
def test_api_key_reads_process_env(monkeypatch):
|
|
monkeypatch.setenv("HUD_API_TOKEN", " tok ")
|
|
assert api_key("hud") == "tok"
|
|
|
|
|
|
def test_key_headers_per_source():
|
|
assert key_headers("tiingo", "k") == {"Authorization": "Token k"}
|
|
assert key_headers("lda", "k") == {"Authorization": "Token k"}
|
|
assert key_headers("fec", "k") == {"X-Api-Key": "k"}
|
|
assert key_headers("congress", "k") == {"X-Api-Key": "k"}
|
|
assert key_headers("hud", "k") == {"Authorization": "Bearer k"}
|
|
assert key_headers("sec", "k") == {}
|
|
assert key_headers("fec", None) == {}
|
|
|
|
|
|
def test_key_sent_in_header_never_in_url():
|
|
seen = []
|
|
|
|
def handler(request):
|
|
seen.append(request)
|
|
return httpx.Response(200, json={})
|
|
|
|
cl = _cl(handler, **key_headers("fec", "k1"))
|
|
get(cl, "https://api.open.fec.gov/v1/x", pacer=_Pacer(), source="fec")
|
|
assert seen[0].headers["X-Api-Key"] == "k1"
|
|
assert "k1" not in str(seen[0].url)
|
|
|
|
|
|
def test_user_agent_has_contact():
|
|
assert http.USER_AGENT == "stack-pfs-dossier/1.0 (ci@mail.fhirworx.io)"
|
|
cl = client("fec", "k")
|
|
assert cl.headers["user-agent"] == http.USER_AGENT
|
|
assert cl.headers["x-api-key"] == "k"
|
|
cl.close()
|
|
|
|
|
|
def test_pacer_spaces_requests():
|
|
times = iter([0.0, 1.0])
|
|
slept = []
|
|
p = Pacer(4.2, clock=lambda: next(times), sleep=slept.append)
|
|
p.wait()
|
|
p.wait()
|
|
assert slept == [pytest.approx(3.2)]
|
|
|
|
|
|
def test_pacer_no_sleep_when_slow_enough():
|
|
times = iter([0.0, 10.0])
|
|
slept = []
|
|
p = Pacer(1.0, clock=lambda: next(times), sleep=slept.append)
|
|
p.wait()
|
|
p.wait()
|
|
assert slept == []
|
|
|
|
|
|
def _seq(*responses):
|
|
it = iter(responses)
|
|
return lambda request: next(it)
|
|
|
|
|
|
def test_get_retries_5xx_then_ok():
|
|
slept = []
|
|
cl = _cl(_seq(httpx.Response(503), httpx.Response(200, text="ok")))
|
|
r = get(cl, "https://x.test/a", pacer=_Pacer(), source="s", sleep=slept.append)
|
|
assert r.text == "ok" and slept == [2.0]
|
|
|
|
|
|
def test_get_5xx_exhausted_raises_refused():
|
|
slept = []
|
|
cl = _cl(lambda request: httpx.Response(502))
|
|
with pytest.raises(Refused) as ei:
|
|
get(
|
|
cl,
|
|
"https://x.test/a",
|
|
pacer=_Pacer(),
|
|
source="s",
|
|
sleep=slept.append,
|
|
retries=2,
|
|
)
|
|
assert ei.value.status == 502 and slept == [2.0, 4.0]
|
|
|
|
|
|
def test_get_retries_transport_errors_then_refuses():
|
|
slept = []
|
|
calls = []
|
|
|
|
def boom(request):
|
|
calls.append(1)
|
|
raise httpx.ReadTimeout("slow", request=request)
|
|
|
|
with pytest.raises(Refused) as ei:
|
|
get(
|
|
_cl(boom),
|
|
"https://x.test/a",
|
|
pacer=_Pacer(),
|
|
source="s",
|
|
sleep=slept.append,
|
|
retries=2,
|
|
)
|
|
assert ei.value.status == 0 and "ReadTimeout" in str(ei.value)
|
|
assert slept == [2.0, 4.0] and len(calls) == 3
|
|
|
|
|
|
def test_get_transport_error_then_ok():
|
|
state = []
|
|
|
|
def flaky(request):
|
|
if not state:
|
|
state.append(1)
|
|
raise httpx.ConnectError("reset", request=request)
|
|
return httpx.Response(200, text="ok")
|
|
|
|
r = get(
|
|
_cl(flaky), "https://x.test/a", pacer=_Pacer(), source="s", sleep=lambda s: None
|
|
)
|
|
assert r.text == "ok"
|
|
|
|
|
|
def test_get_429_retry_after_waits_once():
|
|
slept = []
|
|
cl = _cl(
|
|
_seq(httpx.Response(429, headers={"Retry-After": "30"}), httpx.Response(200))
|
|
)
|
|
r = get(cl, "https://x.test/a", pacer=_Pacer(), source="s", sleep=slept.append)
|
|
assert r.status_code == 200 and slept == [30.0]
|
|
|
|
|
|
def test_get_429_twice_raises_refused():
|
|
cl = _cl(lambda request: httpx.Response(429, headers={"Retry-After": "1"}))
|
|
with pytest.raises(Refused) as ei:
|
|
get(cl, "https://x.test/a", pacer=_Pacer(), source="s", sleep=lambda s: None)
|
|
assert ei.value.status == 429
|
|
|
|
|
|
@pytest.mark.parametrize("headers", [{}, {"Retry-After": "9999"}, {"Retry-After": "x"}])
|
|
def test_get_429_without_usable_retry_after_raises(headers):
|
|
cl = _cl(lambda request: httpx.Response(429, headers=headers))
|
|
with pytest.raises(Refused):
|
|
get(cl, "https://x.test/a", pacer=_Pacer(), source="s", sleep=lambda s: None)
|
|
|
|
|
|
def test_refused_message_has_no_query_string():
|
|
cl = _cl(lambda request: httpx.Response(403))
|
|
with pytest.raises(Refused) as ei:
|
|
get(cl, "https://x.test/x?api_key=secret", pacer=_Pacer(), source="fec")
|
|
assert ei.value.status == 403
|
|
assert "secret" not in str(ei.value) and "?" not in str(ei.value)
|
|
assert ei.value.source == "fec"
|
|
|
|
|
|
def test_get_404_ok_when_allowed():
|
|
cl = _cl(lambda request: httpx.Response(404))
|
|
r = get(cl, "https://x.test/a", pacer=_Pacer(), source="s", ok_404=True)
|
|
assert r.status_code == 404
|
|
with pytest.raises(Refused):
|
|
get(cl, "https://x.test/a", pacer=_Pacer(), source="s", ok_404=False)
|
|
|
|
|
|
def test_slug():
|
|
assert slug("Medicare Physician Fee Schedule") == "medicare-physician-fee-schedule"
|
|
long = slug("a" * 70)
|
|
assert long.startswith("a" * 60 + "-") and len(long) == 69
|
|
|
|
|
|
def test_hoard_under_db_parent(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
|
|
assert http.hoard("sec", "x.json") == tmp_path / "market" / "sec" / "x.json"
|
|
|
|
|
|
def test_hoard_root_is_beside_the_database():
|
|
root = http.hoard_root()
|
|
assert root.name == "market" and root.parent.name == "data"
|
|
|
|
|
|
def test_save_never_overwrites(tmp_path):
|
|
p = tmp_path / "a.json"
|
|
assert http.save(p, b"1") == p
|
|
other = http.save(p, b"2")
|
|
assert other != p and p.read_bytes() == b"1" and other.read_bytes() == b"2"
|
|
assert http.newest(p).read_bytes() in (b"1", b"2")
|
|
|
|
|
|
def test_manifest_records_each_download_once(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
|
|
f = tmp_path / "market" / "sec" / "a.json"
|
|
f.parent.mkdir(parents=True)
|
|
f.write_bytes(b"abc")
|
|
http.manifest(f, "https://x/a.json")
|
|
http.manifest(f, "https://x/a.json")
|
|
f2 = f.with_name("a.1234abcd.json")
|
|
f2.write_bytes(b"abd")
|
|
http.manifest(f2, "https://x/a.json")
|
|
lines = [
|
|
json.loads(x)
|
|
for x in (tmp_path / "market" / "MANIFEST.jsonl").read_text().splitlines()
|
|
]
|
|
assert [l["path"] for l in lines] == ["sec/a.json", "sec/a.1234abcd.json"]
|
|
assert lines[0]["bytes"] == 3 and len(lines[0]["sha256"]) == 64
|
|
|
|
|
|
def test_manifest_dedupes_against_lines_from_an_earlier_run(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
|
|
f = tmp_path / "market" / "a.json"
|
|
f.parent.mkdir(parents=True)
|
|
f.write_bytes(b"abc")
|
|
http.manifest(f, "https://x/a.json")
|
|
http._MANIFESTS.clear() # a new process
|
|
http.manifest(f, "https://x/a.json")
|
|
assert len((tmp_path / "market" / "MANIFEST.jsonl").read_text().splitlines()) == 1
|