Some checks failed
CI / lint (push) Successful in 41s
CI / notebooks-smoke (push) Successful in 2m13s
Deploy / changes (push) Successful in 19s
Infra CI / notebooks (push) Successful in 1m0s
Infra CI / zotero (push) Successful in 22s
CI / test (push) Successful in 5m3s
Infra CI / docs (push) Successful in 20s
Infra CI / api (push) Successful in 1m21s
Infra CI / bcda (push) Successful in 1m56s
Infra CI / llm (push) Successful in 1m4s
Infra CI / mc (push) Successful in 30s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m58s
Deploy / bcda (push) Successful in 1m56s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m4s
Deploy / report (push) Successful in 14s
Comment Pull / comment-pull (push) Failing after 29s
Notebooks Integration / notebooks-integration (push) Failing after 12m58s
3247 lines
113 KiB
Python
3247 lines
113 KiB
Python
"""stack market — entities, gaps."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from datetime import date
|
||
from pathlib import Path
|
||
|
||
import duckdb
|
||
import httpx
|
||
import pytest
|
||
from typer.testing import CliRunner
|
||
|
||
import cli.market as mk
|
||
from bib.store import Store
|
||
from cli import app
|
||
from market import edgar, http
|
||
from market import tables as T
|
||
from market.seed import SEAT_LABEL_KEYS, load_seed
|
||
from pfs.codetables import SeatRow, write_seats
|
||
|
||
runner = CliRunner()
|
||
|
||
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
|
||
|
||
|
||
def _plain(text: str) -> str:
|
||
"""Strip ANSI. Rich highlights ``--option`` inside an error."""
|
||
return _ANSI_RE.sub("", text)
|
||
|
||
|
||
FX = Path(__file__).parent.parent / "market" / "fixtures"
|
||
|
||
|
||
class _Pacer:
|
||
def wait(self):
|
||
pass
|
||
|
||
|
||
def _seat(org, specialty, kind="specialty", year=2024):
|
||
return SeatRow("RUC", year, org, specialty, "permanent", "u", "K", "s", kind)
|
||
|
||
|
||
def _specialties() -> set[str]:
|
||
companies, orgs = load_seed()
|
||
out = {o.specialty for o in orgs if o.specialty}
|
||
out |= {
|
||
link.target
|
||
for e in (*companies, *orgs)
|
||
for link in e.links
|
||
if link.scope == "specialty"
|
||
}
|
||
return out - set(SEAT_LABEL_KEYS)
|
||
|
||
|
||
def _build_db() -> duckdb.DuckDBPyConnection:
|
||
c = duckdb.connect(":memory:")
|
||
c.execute("CREATE SCHEMA terminology")
|
||
c.execute(
|
||
"CREATE TABLE terminology.cms_acceptable_provider_specialty_codes "
|
||
"(specialty_description VARCHAR)"
|
||
)
|
||
c.executemany(
|
||
"INSERT INTO terminology.cms_acceptable_provider_specialty_codes VALUES (?)",
|
||
[(s.title(),) for s in sorted(_specialties())],
|
||
)
|
||
c.execute("CREATE SCHEMA pfs")
|
||
c.execute("CREATE TABLE pfs.cpt_code (code VARCHAR, descriptor VARCHAR)")
|
||
c.execute(
|
||
"INSERT INTO pfs.cpt_code VALUES ('99999', 'a licensed descriptor that no "
|
||
"filing quote should ever copy word for word')"
|
||
)
|
||
write_seats(
|
||
c,
|
||
"RUC",
|
||
[
|
||
_seat("Family Medicine", "FAMILY PRACTICE"),
|
||
_seat("Cardiology", "CARDIOLOGY"),
|
||
_seat("RUC Chair", "", "category"),
|
||
],
|
||
)
|
||
return c
|
||
|
||
|
||
class _Batch:
|
||
def __init__(self, c):
|
||
self.c = c
|
||
|
||
def __enter__(self):
|
||
return self.c
|
||
|
||
def __exit__(self, *a):
|
||
return False
|
||
|
||
|
||
@pytest.fixture
|
||
def env(monkeypatch, tmp_path):
|
||
c = _build_db()
|
||
monkeypatch.setattr(mk, "_batch", lambda: _Batch(c))
|
||
monkeypatch.setattr(mk, "_read", lambda: c.cursor())
|
||
store = Store(":memory:", storage_dir=tmp_path / "st")
|
||
monkeypatch.setattr(mk, "_store", lambda: store)
|
||
published: list[bool] = []
|
||
monkeypatch.setattr(mk, "_publish", lambda: published.append(True))
|
||
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
|
||
monkeypatch.setattr(edgar, "new_pacer", lambda: _Pacer())
|
||
try:
|
||
yield c, store, published, tmp_path / "market"
|
||
finally:
|
||
store.close()
|
||
c.close()
|
||
|
||
|
||
def _cache_offline(hoard: Path) -> None:
|
||
sec = hoard / "sec"
|
||
(sec / "submissions").mkdir(parents=True)
|
||
(sec / "company_tickers.json").write_bytes(
|
||
(FX / "company_tickers.json").read_bytes()
|
||
)
|
||
for name in (
|
||
"submissions_CIK0001564406.json",
|
||
"CIK0001564406-submissions-001.json",
|
||
):
|
||
target = name.replace("submissions_", "")
|
||
if name.startswith("submissions_"):
|
||
target = "CIK0001564406.json"
|
||
(sec / "submissions" / target).write_bytes((FX / name).read_bytes())
|
||
_cache_filings(hoard)
|
||
|
||
|
||
def _cache_filings(hoard: Path, companies=None) -> None:
|
||
"""A hoarded filing per cited accession, holding exactly the quotes the
|
||
seed cites to it."""
|
||
import html
|
||
|
||
companies = companies or load_seed()[0]
|
||
quotes: dict[tuple[str, str], list[str]] = {}
|
||
for _who, c, cite in edgar.filing_cites(companies):
|
||
if cite.adsh:
|
||
quotes.setdefault((c.cik, cite.adsh), []).append(cite.quote)
|
||
for (cik, adsh), qs in quotes.items():
|
||
p = hoard / "sec" / "filings" / cik / f"{adsh}.htm"
|
||
p.parent.mkdir(parents=True, exist_ok=True)
|
||
p.write_text(
|
||
"<html><body>"
|
||
+ "".join(f"<p>{html.escape(q)}</p>" for q in qs)
|
||
+ "</body></html>"
|
||
)
|
||
|
||
|
||
def _count(c, table):
|
||
return c.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
||
|
||
|
||
def test_entities_offline_writes_and_logs(env):
|
||
c, store, published, hoard = env
|
||
_cache_offline(hoard)
|
||
# a stale curated link the seed no longer carries
|
||
T.write_links(
|
||
c, [T.LinkRow("0001564406", "program", "QPP-MIPS", "curated", "x", "url", "u")]
|
||
)
|
||
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
companies, orgs = load_seed()
|
||
assert _count(c, "market.entity") == len(companies) + len(orgs)
|
||
osh = next(e for e in T.read_entities(c, "company") if e.key == "0001564406")
|
||
assert osh.status == "acquired" and osh.ended_on == date(2023, 5, 2)
|
||
assert osh.sic == "8000" and osh.item_key
|
||
assert "no submissions for AMED" in res.output
|
||
# the five fixture filings, and the seed's dated ticker
|
||
assert len(T.read_filings(c, "0001564406")) == 5
|
||
assert [s.source for s in T.read_securities(c, "0001564406")] == ["seed"]
|
||
links = T.read_links(c, basis="curated")
|
||
assert all(l.item_key for l in links)
|
||
assert ("program", "QPP-MIPS") not in {(l.scope, l.target) for l in links}
|
||
assert [l.target for l in T.read_links(c, basis="sic")] == ["8000"]
|
||
logs = c.execute(
|
||
"SELECT table_name, rows FROM cms.ingest_log WHERE module = 'market.edgar' "
|
||
"ORDER BY table_name"
|
||
).fetchall()
|
||
assert [t for t, _ in logs] == [
|
||
"market.entity",
|
||
"market.entity_link",
|
||
"market.filing",
|
||
"market.security",
|
||
]
|
||
assert published == [True]
|
||
assert "retired 1" in res.output
|
||
assert store.count() > 0
|
||
|
||
|
||
def test_entities_rerun_is_idempotent(env):
|
||
c, _, published, hoard = env
|
||
_cache_offline(hoard)
|
||
for _ in range(2):
|
||
assert (
|
||
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
||
== 0
|
||
)
|
||
companies, orgs = load_seed()
|
||
assert _count(c, "market.entity") == len(companies) + len(orgs)
|
||
assert _count(c, "market.filing") == 5
|
||
|
||
|
||
def test_entities_no_write_prints_counts(env, monkeypatch):
|
||
c, _, published, hoard = env
|
||
_cache_offline(hoard)
|
||
monkeypatch.setattr(
|
||
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
||
)
|
||
res = runner.invoke(app, ["market", "entities", "--offline"])
|
||
assert res.exit_code == 0, res.output
|
||
companies, orgs = load_seed()
|
||
assert f"entities {len(companies) + len(orgs)}" in res.output
|
||
assert (
|
||
"company/acquired" in res.output
|
||
and "organization/specialty_society" in res.output
|
||
)
|
||
assert "dry run" in res.output and published == []
|
||
assert "warning" not in res.output # every seat specialty has a society
|
||
|
||
|
||
def test_entities_warns_about_uncovered_seat_specialty(env):
|
||
c, _, _, hoard = env
|
||
_cache_offline(hoard)
|
||
write_seats(c, "RUC", [_seat("Podiatry", "PODIATRY", year=2025)])
|
||
res = runner.invoke(app, ["market", "entities", "--offline"])
|
||
assert res.exit_code == 0
|
||
assert "seat specialties without a society: ['PODIATRY']" in res.output
|
||
|
||
|
||
def _handler(routes):
|
||
def handler(request: httpx.Request):
|
||
url = str(request.url)
|
||
for pattern, make in routes:
|
||
m = re.search(pattern, url)
|
||
if m:
|
||
return make(m)
|
||
return httpx.Response(404)
|
||
|
||
return handler
|
||
|
||
|
||
def _online(monkeypatch, routes):
|
||
monkeypatch.setattr(
|
||
http,
|
||
"client",
|
||
lambda source, key=None: httpx.Client(
|
||
transport=httpx.MockTransport(_handler(routes))
|
||
),
|
||
)
|
||
|
||
|
||
def test_entities_refusal_prints_gap(env, monkeypatch):
|
||
c, _, published, hoard = env
|
||
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
|
||
res = runner.invoke(app, ["market", "entities", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: sec-edgar:entities:" in res.output and "403" in res.output
|
||
assert _count(c, "market.entity") == 0 and _count(c, "market.filing") == 0
|
||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:entities"]
|
||
assert published == [True]
|
||
|
||
|
||
def test_entities_refusal_without_write_records_nothing(env, monkeypatch):
|
||
c, _, published, _ = env
|
||
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
|
||
monkeypatch.setattr(
|
||
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
||
)
|
||
res = runner.invoke(app, ["market", "entities"])
|
||
assert res.exit_code == 0 and "gap: sec-edgar:entities:" in res.output
|
||
assert published == []
|
||
|
||
|
||
def _sub_for(cik: str) -> dict:
|
||
return {
|
||
"cik": cik,
|
||
"name": f"Filer {cik}",
|
||
"sic": "8082",
|
||
"sicDescription": "Home Health",
|
||
"tickers": [],
|
||
"formerNames": [],
|
||
"filings": {
|
||
"recent": {
|
||
"accessionNumber": ["A-1", "A-2"],
|
||
"filingDate": ["2024-01-01", "2024-01-02"],
|
||
"reportDate": ["", ""],
|
||
"acceptanceDateTime": ["", ""],
|
||
"form": ["10-K", "4"],
|
||
"items": ["", ""],
|
||
"primaryDocument": ["a.htm", "b.xml"],
|
||
},
|
||
"files": [],
|
||
},
|
||
}
|
||
|
||
|
||
def test_sic_sweep_registers_filers_and_sic_links(env, monkeypatch):
|
||
c, _, published, hoard = env
|
||
_cache_filings(hoard)
|
||
atom = (FX / "sic_8082.atom").read_bytes().replace(b"0001468328", b"0000000042")
|
||
seen: list[str] = []
|
||
|
||
def sub(m):
|
||
seen.append(m.group(1))
|
||
cik = m.group(1)
|
||
if cik == "0001564406":
|
||
return httpx.Response(
|
||
200, content=(FX / "submissions_CIK0001564406.json").read_bytes()
|
||
)
|
||
if cik == "0000000042":
|
||
return httpx.Response(404) # no submissions file: skipped
|
||
return httpx.Response(200, json=_sub_for(cik))
|
||
|
||
def page(m):
|
||
return httpx.Response(
|
||
200, content=(FX / "CIK0001564406-submissions-001.json").read_bytes()
|
||
)
|
||
|
||
_online(
|
||
monkeypatch,
|
||
[
|
||
(
|
||
r"company_tickers\.json",
|
||
lambda m: httpx.Response(
|
||
200, content=(FX / "company_tickers.json").read_bytes()
|
||
),
|
||
),
|
||
(r"SIC=\d+.*start=0", lambda m: httpx.Response(200, content=atom)),
|
||
(r"submissions-001\.json", page),
|
||
(r"CIK(\d{10})\.json", sub),
|
||
],
|
||
)
|
||
res = runner.invoke(app, ["market", "entities", "--sic-sweep", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert res.output.count("SIC ") == len(edgar.HEALTH_SICS)
|
||
keys = {e.key for e in T.read_entities(c, "company")}
|
||
assert {"0001725255", "0001468328"} <= keys and "0000000042" not in keys
|
||
# a sweep filer keeps only the sweep forms; the seed's filings keep everything
|
||
assert [f.form for f in T.read_filings(c, "0001725255")] == ["10-K"]
|
||
assert len(T.read_filings(c, "0001468328")) == 2
|
||
assert len(T.read_filings(c, "0001564406")) == 5
|
||
sic = T.read_links(c, basis="sic")
|
||
assert {l.target for l in sic} == {"8082", "8000"}
|
||
assert any(l.entity_key == "0001725255" for l in sic)
|
||
|
||
|
||
def test_entities_seed_errors_abort(env, monkeypatch):
|
||
c, _, published, _ = env
|
||
companies, orgs = load_seed()
|
||
bad = companies[0].__class__(
|
||
**{
|
||
**companies[0].__dict__,
|
||
"links": (
|
||
companies[0].links[0].__class__("family", "NOPE", companies[0].cite),
|
||
),
|
||
}
|
||
)
|
||
monkeypatch.setattr(mk, "load_seed", lambda: ((bad, *companies[1:]), orgs))
|
||
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
|
||
assert res.exit_code == 1 and "seed error: OSH" in res.output
|
||
assert c.execute(
|
||
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
|
||
).fetchone() == (0,)
|
||
assert published == []
|
||
|
||
|
||
def test_entities_missing_vocabulary_tables_abort(env, monkeypatch):
|
||
bare = duckdb.connect(":memory:")
|
||
monkeypatch.setattr(mk, "_read", lambda: bare)
|
||
res = runner.invoke(app, ["market", "entities", "--offline"])
|
||
assert res.exit_code == 1 and "cannot validate the seed" in res.output
|
||
bare.close()
|
||
|
||
|
||
def test_entities_other_read_errors_propagate(env, monkeypatch):
|
||
def boom():
|
||
raise RuntimeError("disk on fire")
|
||
|
||
monkeypatch.setattr(mk, "_read", boom)
|
||
res = runner.invoke(app, ["market", "entities", "--offline"])
|
||
assert isinstance(res.exception, RuntimeError)
|
||
|
||
|
||
def test_require_key_missing_writes_only_gap(env, monkeypatch):
|
||
c, _, published, _ = env
|
||
monkeypatch.delenv("TIINGO_API_KEY", raising=False)
|
||
assert mk._require_key("tiingo", "tiingo:prices", write=True) is None
|
||
assert [(g.source, g.note) for g in T.read_gaps(c)] == [
|
||
("tiingo:prices", "TIINGO_API_KEY not in .env")
|
||
]
|
||
assert all(_count(c, f"market.{t}") == 0 for t in ("entity", "security", "filing"))
|
||
assert published == [True]
|
||
|
||
|
||
def test_require_key_present_returns_it(env, monkeypatch):
|
||
monkeypatch.setenv("TIINGO_API_KEY", "abc")
|
||
assert mk._require_key("tiingo", "tiingo:prices", write=True) == "abc"
|
||
|
||
|
||
def test_require_key_missing_prints_gap_without_write(env, monkeypatch, capsys):
|
||
monkeypatch.delenv("HUD_API_TOKEN", raising=False)
|
||
assert mk._require_key("hud", "hud:hud", write=False) is None
|
||
assert "gap: hud:hud: HUD_API_TOKEN not in .env" in capsys.readouterr().out
|
||
c, _, published, _ = env
|
||
assert published == []
|
||
|
||
|
||
def test_gap_cleared_after_success(env):
|
||
c, _, _, hoard = env
|
||
_cache_offline(hoard)
|
||
T.write_gap(c, "sec-edgar:entities", "403")
|
||
T.write_gap(c, "tiingo:prices", "no key")
|
||
assert (
|
||
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
||
== 0
|
||
)
|
||
assert {g.source for g in T.read_gaps(c)} == {
|
||
"tiingo:prices",
|
||
"web:renalmd.org",
|
||
"web:asco.org",
|
||
"web:ahip.org",
|
||
}
|
||
|
||
|
||
def test_gaps_command_lists_rows(env):
|
||
c, _, _, _ = env
|
||
T.write_gap(c, "tiingo:prices", "TIINGO_API_KEY not in .env")
|
||
res = runner.invoke(app, ["market", "gaps"])
|
||
assert res.exit_code == 0
|
||
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
|
||
|
||
|
||
def test_gaps_command_empty_and_missing_tables(env, monkeypatch):
|
||
c, _, _, _ = env
|
||
T.ensure_tables(c)
|
||
assert "no gaps recorded" in runner.invoke(app, ["market", "gaps"]).output
|
||
bare = duckdb.connect(":memory:")
|
||
monkeypatch.setattr(mk, "_read", lambda: bare)
|
||
assert "no market tables yet" in runner.invoke(app, ["market", "gaps"]).output
|
||
|
||
class _Broken:
|
||
def execute(self, *a, **k):
|
||
raise RuntimeError("x")
|
||
|
||
def close(self):
|
||
pass
|
||
|
||
monkeypatch.setattr(mk, "_read", lambda: _Broken())
|
||
assert isinstance(runner.invoke(app, ["market", "gaps"]).exception, RuntimeError)
|
||
bare.close()
|
||
|
||
|
||
def test_default_indirections_resolve_lazily(monkeypatch):
|
||
import conf.connect as cc
|
||
|
||
calls: list[str] = []
|
||
monkeypatch.setattr(cc, "duckdb_batch", lambda n: calls.append(f"batch {n}") or n)
|
||
monkeypatch.setattr(
|
||
cc, "duckdb", lambda n, read_only=True: calls.append(f"read {n}") or n
|
||
)
|
||
monkeypatch.setattr(cc, "bib", lambda: calls.append("bib") or "store")
|
||
monkeypatch.setattr(
|
||
cc, "publish_replica", lambda n: calls.append(f"pub {n}") or "p"
|
||
)
|
||
assert mk._batch() == "aco" and mk._read() == "aco" and mk._store() == "store"
|
||
mk._publish()
|
||
assert calls == ["batch aco", "read aco", "bib", "pub aco"]
|
||
|
||
|
||
def test_entities_aborts_when_a_quote_is_not_in_the_filing(env):
|
||
c, _, published, hoard = env
|
||
_cache_offline(hoard)
|
||
victim = next(hoard.glob("sec/filings/*/*.htm"))
|
||
victim.write_text("<p>nothing the seed quotes</p>")
|
||
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
|
||
assert res.exit_code == 1 and "quote error:" in res.output
|
||
assert published == []
|
||
assert c.execute(
|
||
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
|
||
).fetchone() == (0,)
|
||
|
||
|
||
def test_entities_aborts_when_a_cited_filing_is_not_hoarded(env):
|
||
c, _, published, hoard = env
|
||
_cache_offline(hoard)
|
||
next(hoard.glob("sec/filings/*/*.htm")).unlink()
|
||
res = runner.invoke(app, ["market", "entities", "--offline"])
|
||
assert res.exit_code == 1 and "not hoarded" in res.output
|
||
|
||
|
||
def test_entities_fetches_cited_filings_online(env, monkeypatch):
|
||
c, _, _, hoard = env
|
||
_cache_offline(hoard)
|
||
victim = next(hoard.glob("sec/filings/*/*.htm"))
|
||
cik, adsh = victim.parent.name, victim.stem
|
||
body = victim.read_bytes()
|
||
victim.unlink()
|
||
(hoard / "sec" / "submissions" / f"CIK{cik}.json").write_text(
|
||
json.dumps(
|
||
_sub_for(cik)
|
||
| {
|
||
"filings": {
|
||
"recent": {
|
||
**_sub_for(cik)["filings"]["recent"],
|
||
"accessionNumber": [adsh, "Z"],
|
||
"primaryDocument": ["doc.htm", "z.htm"],
|
||
},
|
||
"files": [],
|
||
}
|
||
}
|
||
)
|
||
)
|
||
seen = []
|
||
|
||
def serve(m):
|
||
seen.append(m.group(0))
|
||
return httpx.Response(200, content=body)
|
||
|
||
_online(monkeypatch, [(r"/Archives/edgar/data/\d+/\d+/doc\.htm", serve)])
|
||
res = runner.invoke(app, ["market", "entities"])
|
||
assert res.exit_code == 0, res.output
|
||
assert len(seen) == 1 and victim.exists()
|
||
|
||
|
||
def test_entities_manifest_and_ingest_log_sources(env):
|
||
c, _, _, hoard = env
|
||
_cache_offline(hoard)
|
||
assert (
|
||
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
||
== 0
|
||
)
|
||
manifest = hoard / "MANIFEST.jsonl"
|
||
assert manifest.exists() and manifest.read_text().count("\n") >= 6
|
||
rows = c.execute(
|
||
"SELECT table_name, source_file, sha256 FROM cms.ingest_log WHERE module = 'market.edgar'"
|
||
).fetchall()
|
||
assert len(rows) == 4 and all(sf and sha and len(sha) == 64 for _, sf, sha in rows)
|
||
|
||
|
||
def test_entities_records_unreachable_site_gaps_and_pseudo_orgs(env):
|
||
c, _, _, hoard = env
|
||
_cache_offline(hoard)
|
||
assert (
|
||
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
||
== 0
|
||
)
|
||
gaps = {g.source for g in T.read_gaps(c)}
|
||
assert {"web:renalmd.org", "web:asco.org", "web:ahip.org"} <= gaps
|
||
assert "sec-edgar:entities" not in gaps
|
||
pseudo = {e.key for e in T.read_entities(c, "organization") if e.pseudo}
|
||
assert pseudo == {"PRIMARY-CARE-SEAT", "ONCOLOGY-SEAT"}
|
||
|
||
|
||
def test_entities_retires_organizations_the_seed_dropped(env):
|
||
c, _, _, hoard = env
|
||
_cache_offline(hoard)
|
||
T.write_entities(c, [T.EntityRow("ASCO", "ASCO", "organization")])
|
||
assert (
|
||
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
||
== 0
|
||
)
|
||
assert "ASCO" not in {e.key for e in T.read_entities(c, "organization")}
|
||
|
||
|
||
def test_entities_aborts_when_a_quote_copies_a_cpt_descriptor(env):
|
||
c, _, published, hoard = env
|
||
_cache_offline(hoard)
|
||
victim = load_seed()[0][0].cite.quote
|
||
c.execute("DELETE FROM pfs.cpt_code")
|
||
c.execute("INSERT INTO pfs.cpt_code VALUES ('1', ?)", [victim])
|
||
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
|
||
assert res.exit_code == 1 and "ama error: OSH" in res.output
|
||
assert published == []
|
||
assert c.execute(
|
||
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
|
||
).fetchone() == (0,)
|
||
|
||
|
||
def test_entities_stores_ticker_history_and_link_dates(env):
|
||
c, _, _, hoard = env
|
||
_cache_offline(hoard)
|
||
assert (
|
||
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
||
== 0
|
||
)
|
||
assert [
|
||
(s.ticker, s.valid_to, s.priced) for s in T.read_securities(c, "0001156039")
|
||
] == [("ANTM", date(2022, 6, 27), False), ("ELV", None, True)]
|
||
assert [s.priced for s in T.read_securities(c, "0001739940")] == [False]
|
||
(clov,) = [
|
||
l
|
||
for l in T.read_links(c, basis="curated", target="APM")
|
||
if l.entity_key == "0001801170"
|
||
]
|
||
assert clov.valid_to == date(2023, 12, 31)
|
||
|
||
|
||
# ── stack market prices ──────────────────────────────────────────────
|
||
import io # noqa: E402
|
||
import zipfile # noqa: E402
|
||
|
||
from market import tiingo # noqa: E402
|
||
|
||
DAY = date(2026, 11, 12)
|
||
|
||
|
||
def _supported_zip(*rows: str) -> bytes:
|
||
head = "ticker,exchange,assetType,priceCurrency,startDate,endDate\n"
|
||
buf = io.BytesIO()
|
||
with zipfile.ZipFile(buf, "w") as z:
|
||
z.writestr("supported_tickers.csv", head + "\n".join(rows) + "\n")
|
||
return buf.getvalue()
|
||
|
||
|
||
def _eod(day: str, close: float, adj: float | None = None) -> dict:
|
||
return {
|
||
"date": f"{day}T00:00:00.000Z",
|
||
"open": close,
|
||
"high": close,
|
||
"low": close,
|
||
"close": close,
|
||
"volume": 5,
|
||
"adjClose": close if adj is None else adj,
|
||
"divCash": 0.0,
|
||
"splitFactor": 1.0,
|
||
}
|
||
|
||
|
||
@pytest.fixture
|
||
def px(env, monkeypatch, tmp_path):
|
||
c, store, published, hoard = env
|
||
monkeypatch.setenv("TIINGO_API_KEY", "secret-key")
|
||
monkeypatch.setattr(tiingo, "state_dir", lambda: tmp_path / "state")
|
||
monkeypatch.setattr(tiingo, "new_pacer", lambda: _Pacer())
|
||
monkeypatch.setattr(tiingo, "today", lambda: DAY)
|
||
seen: list[httpx.Request] = []
|
||
series = {
|
||
"AAA": [
|
||
_eod("2024-01-02", 100),
|
||
_eod("2024-01-03", 110),
|
||
_eod("2024-01-04", 99),
|
||
],
|
||
"BBB": [_eod("2024-01-02", 10), _eod("2024-01-03", 11)],
|
||
"SPY": [_eod("2024-01-02", 400), _eod("2024-01-03", 404)],
|
||
"XLV": [_eod("2024-01-02", 100)],
|
||
"IHF": [_eod("2024-01-02", 50)],
|
||
}
|
||
z = _supported_zip(
|
||
"AAA,NYSE,Stock,USD,2000-01-01,2026-11-12",
|
||
"BBB,NYSE,Stock,USD,2000-01-01,2026-11-12",
|
||
"SPY,NYSE,ETF,USD,1993-01-29,2026-11-12",
|
||
"XLV,NYSE,ETF,USD,1998-12-22,2026-11-12",
|
||
"IHF,NYSE,ETF,USD,2006-05-05,2026-11-12",
|
||
"ZZZ,NYSE,Stock,USD,2000-01-01,2026-11-12",
|
||
)
|
||
|
||
def prices_for(m):
|
||
if m.group(1) not in series:
|
||
return httpx.Response(404, json={"detail": "Not found."})
|
||
return httpx.Response(200, json=series[m.group(1)])
|
||
|
||
routes = [
|
||
(r"supported_tickers\.zip", lambda m: httpx.Response(200, content=z)),
|
||
(r"/daily/(\w+)/prices", prices_for),
|
||
]
|
||
|
||
def handler(request):
|
||
seen.append(request)
|
||
return _handler(routes)(request)
|
||
|
||
monkeypatch.setattr(
|
||
http,
|
||
"client",
|
||
lambda source, key=None: httpx.Client(
|
||
transport=httpx.MockTransport(handler),
|
||
headers={"Authorization": f"Token {key}"} if key else {},
|
||
),
|
||
)
|
||
return c, store, published, hoard, seen, series, tmp_path / "state"
|
||
|
||
|
||
def _price_requests(seen):
|
||
return [r for r in seen if "/prices" in r.url.path]
|
||
|
||
|
||
def test_prices_cli_fetch_write(px):
|
||
c, _, published, hoard, seen, _, state = px
|
||
res = runner.invoke(
|
||
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert _count(c, "market.price_daily") == 5
|
||
assert [r.url.params["startDate"] for r in _price_requests(seen)] == [
|
||
"1993-01-29"
|
||
] * 2
|
||
assert all("secret-key" not in str(r.url) for r in seen)
|
||
keyed = [r for r in seen if "/prices" in r.url.path]
|
||
assert all(r.headers["authorization"] == "Token secret-key" for r in keyed)
|
||
free = [r for r in seen if "supported" in r.url.path]
|
||
assert free and "authorization" not in free[0].headers
|
||
assert {
|
||
(t, n)
|
||
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||
} == {("market.price_daily", 5), ("market.series", 2)}
|
||
assert [s.value for s in T.read_series(c, ["AAA"])] == [pytest.approx(99.0)]
|
||
assert published == [True]
|
||
used = json.loads((state / "tiingo-2026-11.json").read_text())
|
||
assert used["symbols"] == ["AAA", "BBB"] and used["day_counts"] == {"2026-11-12": 2}
|
||
assert (hoard / "tiingo" / "eod" / "AAA" / "1993-01-29_2026-11-12.json").is_file()
|
||
assert "priced 2 symbols, 5 rows" in res.output and "2/450" in res.output
|
||
assert T.read_gaps(c) == []
|
||
|
||
|
||
def test_prices_incremental_pull_and_same_day_skip(px, monkeypatch):
|
||
c, _, _, _, seen, series, _ = px
|
||
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
|
||
assert runner.invoke(app, cmd).exit_code == 0
|
||
before = len(_price_requests(seen))
|
||
again = runner.invoke(app, cmd)
|
||
assert len(_price_requests(seen)) == before # priced today: nothing to do
|
||
assert "priced 0 symbols" in again.output
|
||
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
|
||
series["AAA"] = [_eod("2024-01-04", 99, 49.5), _eod("2024-01-05", 108.9, 54.45)]
|
||
assert runner.invoke(app, cmd).exit_code == 0
|
||
last = _price_requests(seen)[-1]
|
||
assert last.url.params["startDate"] == "2024-01-04"
|
||
rows = {r.day: r for r in T.read_prices(c, "AAA")}
|
||
assert len(rows) == 4 and rows[date(2024, 1, 4)].adj_close == 99
|
||
assert rows[date(2024, 1, 5)].ret == pytest.approx(0.1)
|
||
assert [m.value for m in T.read_series(c, ["AAA"])] == [pytest.approx(108.9)]
|
||
|
||
|
||
def test_prices_benchmarks_start_at_listing(px):
|
||
c, _, _, _, seen, _, _ = px
|
||
res = runner.invoke(app, ["market", "prices", "--benchmarks", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
starts = {
|
||
r.url.path.split("/")[3]: r.url.params["startDate"]
|
||
for r in _price_requests(seen)
|
||
}
|
||
assert starts == {"SPY": "1993-01-29", "XLV": "1998-12-22", "IHF": "2006-05-05"}
|
||
|
||
|
||
def test_prices_dry_run_reports_budget(px, monkeypatch):
|
||
c, _, published, _, seen, _, state = px
|
||
T.ensure_tables(c)
|
||
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
|
||
T.write_links( # revenue_exposure passes the strength gate for any SIC
|
||
c, [T.LinkRow("K1", "program", "PFS", "fts", "revenue_exposure", "adsh", "a")]
|
||
)
|
||
state.mkdir(parents=True)
|
||
(state / "tiingo-2026-11.json").write_text(
|
||
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
|
||
)
|
||
monkeypatch.setattr(
|
||
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
||
)
|
||
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "1/450 symbols used, 449 remaining" in res.output
|
||
assert "1. SPY" in res.output and "used" in res.output
|
||
assert "AAA" in res.output and "K1" in res.output and "first pull" in res.output
|
||
assert _price_requests(seen) == [] and published == []
|
||
assert not (state / "tiingo-2026-11.json").read_text().count("AAA")
|
||
|
||
|
||
def test_prices_dry_run_without_market_tables_lists_benchmarks(px):
|
||
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "3 symbols ranked" in res.output
|
||
|
||
|
||
def test_prices_dry_run_shows_stored_span(px):
|
||
c, *_ = px
|
||
T.ensure_tables(c)
|
||
T.write_prices(
|
||
c,
|
||
"AAA",
|
||
tiingo.to_prices(
|
||
"AAA", [_eod("2024-01-02", 1)], pulled_on=DAY, source_file="f", overlap=None
|
||
),
|
||
)
|
||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--dry-run"])
|
||
assert "stored to 2024-01-02" in res.output
|
||
|
||
|
||
def test_prices_without_key_records_gap(px, monkeypatch):
|
||
c, _, published, _, seen, _, _ = px
|
||
monkeypatch.delenv("TIINGO_API_KEY")
|
||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
|
||
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"]
|
||
assert _count(c, "market.price_daily") == 0 and seen == [] and published == [True]
|
||
|
||
|
||
def test_prices_refusal_prints_gap(px, monkeypatch):
|
||
c, _, published, _, _, _, _ = px
|
||
monkeypatch.setattr(
|
||
http,
|
||
"client",
|
||
lambda source, key=None: httpx.Client(
|
||
transport=httpx.MockTransport(lambda r: httpx.Response(403))
|
||
),
|
||
)
|
||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: tiingo:prices:" in res.output and "403" in res.output
|
||
assert _count(c, "market.price_daily") == 0
|
||
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"] and published == [
|
||
True
|
||
]
|
||
|
||
|
||
def test_prices_refusal_on_prices_after_supported_ok(px, monkeypatch):
|
||
c, _, published, _, _, _, _ = px
|
||
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
|
||
routes = [
|
||
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
|
||
(r"/prices", lambda m: httpx.Response(401)),
|
||
]
|
||
_online(monkeypatch, routes)
|
||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA"])
|
||
assert (
|
||
res.exit_code == 0
|
||
and "gap: tiingo:prices:" in res.output
|
||
and "401" in res.output
|
||
)
|
||
assert published == []
|
||
|
||
|
||
def test_prices_nothing_selected_exits(px):
|
||
res = runner.invoke(app, ["market", "prices"])
|
||
assert res.exit_code == 2 and "nothing to price" in res.output
|
||
|
||
|
||
def test_prices_unknown_ticker_and_window_gap_become_gap_note(px):
|
||
c, _, _, _, _, series, _ = px
|
||
T.ensure_tables(c)
|
||
T.write_securities(
|
||
c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, date(2030, 1, 1), "s")]
|
||
)
|
||
res = runner.invoke(
|
||
app, ["market", "prices", "--ticker", "BBB", "--ticker", "ZZZ", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "ZZZ: not found at Tiingo" in res.output
|
||
(gap,) = T.read_gaps(c)
|
||
assert (
|
||
"ZZZ: not found at Tiingo" in gap.note
|
||
and "BBB: Tiingo ends 2026-11-12" in gap.note
|
||
)
|
||
# BBB later prices cleanly: its entry goes, ZZZ's stays
|
||
T.write_securities(c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, None, "s")])
|
||
runner.invoke(app, ["market", "prices", "--ticker", "BBB", "--refresh", "--write"])
|
||
(gap,) = T.read_gaps(c)
|
||
assert gap.note == "ZZZ: not found at Tiingo"
|
||
|
||
|
||
def test_prices_clean_run_clears_gap_and_window_clips(px):
|
||
c, _, _, _, _, _, _ = px
|
||
T.ensure_tables(c)
|
||
T.write_gap(c, "tiingo:prices", "AAA: not found at Tiingo")
|
||
T.write_securities(
|
||
c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", date(2024, 1, 3), None, "s")]
|
||
)
|
||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert [r.day for r in T.read_prices(c, "AAA")] == [
|
||
date(2024, 1, 3),
|
||
date(2024, 1, 4),
|
||
]
|
||
assert T.read_gaps(c) == []
|
||
|
||
|
||
def test_prices_limit_stops_downloads(px):
|
||
c, _, _, _, seen, _, _ = px
|
||
res = runner.invoke(
|
||
app, ["market", "prices", "--benchmarks", "--limit", "1", "--write"]
|
||
)
|
||
assert res.exit_code == 0 and len(_price_requests(seen)) == 1
|
||
assert [r.ticker for r in [*T.read_prices(c, "SPY")][:1]] == ["SPY"]
|
||
|
||
|
||
def test_prices_budget_exhausted_stops_and_writes_what_it_has(px):
|
||
c, _, published, _, seen, _, state = px
|
||
state.mkdir(parents=True)
|
||
(state / "tiingo-2026-11.json").write_text(
|
||
json.dumps(
|
||
{
|
||
"month": "2026-11",
|
||
"symbols": [f"S{i}" for i in range(449)] + ["AAA"],
|
||
"day_counts": {},
|
||
}
|
||
)
|
||
)
|
||
res = runner.invoke(
|
||
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "budget: stopped, BBB" in res.output
|
||
assert [r.ticker for r in T.read_prices(c, "AAA")][:1] == ["AAA"]
|
||
assert T.read_prices(c, "BBB") == [] and published == [True]
|
||
|
||
|
||
def test_prices_without_write_stores_nothing(px, monkeypatch):
|
||
c, _, published, _, _, _, _ = px
|
||
monkeypatch.setattr(
|
||
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
||
)
|
||
res = runner.invoke(app, ["market", "prices", "--ticker", "ZZZ"])
|
||
assert res.exit_code == 0, res.output
|
||
assert (
|
||
"dry run: pass --write" in res.output
|
||
and "gap: tiingo:prices: ZZZ" in res.output
|
||
)
|
||
assert published == []
|
||
|
||
|
||
def test_merge_gap_keeps_other_tickers_and_orders():
|
||
assert mk._merge_gap("", {}, set()) == ""
|
||
assert mk._merge_gap("A: x | B: y", {"C": "z"}, {"A"}) == "B: y | C: z"
|
||
|
||
|
||
def test_or_empty_reraises_other_errors():
|
||
def boom(_):
|
||
raise RuntimeError("disk")
|
||
|
||
with pytest.raises(RuntimeError):
|
||
mk._or_empty(boom, None, {})
|
||
|
||
|
||
def test_prices_ticker_refuses_unpriced_record_without_a_request(px):
|
||
c, _, _, _, seen, _, state = px
|
||
T.ensure_tables(c)
|
||
T.write_securities(
|
||
c, "K9", [T.SecurityRow("K9", "AAA", "NYSE", None, None, "s", False)]
|
||
)
|
||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "skipped AAA: symbol record only" in res.output
|
||
assert _price_requests(seen) == [] and _count(c, "market.price_daily") == 0
|
||
assert not (state / "tiingo-2026-11.json").exists()
|
||
assert T.read_gaps(c) == []
|
||
|
||
|
||
def test_prices_unsupported_named_ticker_is_gap_not_request(px):
|
||
c, _, _, _, seen, _, _ = px
|
||
res = runner.invoke(app, ["market", "prices", "--ticker", "QQQQ", "--write"])
|
||
assert "skipped QQQQ: not on Tiingo's supported list" in res.output
|
||
assert _price_requests(seen) == []
|
||
(gap,) = T.read_gaps(c)
|
||
assert gap.note == "QQQQ: not on Tiingo's supported list"
|
||
|
||
|
||
def test_prices_universe_records_unsupported_seed_ticker_gap(px):
|
||
c, _, _, _, seen, _, _ = px
|
||
T.ensure_tables(c)
|
||
T.write_securities(
|
||
c, "0001800682", [T.SecurityRow("0001800682", "CANO", "NYSE", None, None, "s")]
|
||
)
|
||
res = runner.invoke(app, ["market", "prices", "--universe", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "skipped CANO: not on Tiingo's supported list" in res.output
|
||
assert all("/CANO/" not in r.url.path for r in seen)
|
||
(gap,) = T.read_gaps(c)
|
||
assert gap.source == "tiingo:prices"
|
||
assert gap.note == "CANO: not on Tiingo's supported list"
|
||
# rerun with everything already priced today keeps the gap
|
||
again = runner.invoke(app, ["market", "prices", "--universe", "--write"])
|
||
assert again.exit_code == 0
|
||
assert [g.note for g in T.read_gaps(c)] == [gap.note]
|
||
|
||
|
||
def test_prices_missing_overlap_day_repulls_further_back(px, monkeypatch):
|
||
c, _, _, _, seen, _, _ = px
|
||
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
|
||
assert runner.invoke(app, cmd).exit_code == 0
|
||
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
|
||
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
|
||
|
||
def eod(m):
|
||
# the 2024-01-04 overlap day is absent from the narrow pull
|
||
start = next(r for r in seen[::-1] if "/prices" in r.url.path).url.params[
|
||
"startDate"
|
||
]
|
||
if start == "2024-01-04":
|
||
return httpx.Response(200, json=[_eod("2024-01-05", 54.45)])
|
||
return httpx.Response(
|
||
200, json=[_eod("2024-01-03", 55), _eod("2024-01-05", 60.5)]
|
||
)
|
||
|
||
handler = _handler(
|
||
[
|
||
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
|
||
(r"/daily/AAA/prices", eod),
|
||
]
|
||
)
|
||
|
||
def record(request):
|
||
seen.append(request)
|
||
return handler(request)
|
||
|
||
monkeypatch.setattr(
|
||
http,
|
||
"client",
|
||
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(record)),
|
||
)
|
||
res = runner.invoke(app, cmd)
|
||
assert res.exit_code == 0, res.output
|
||
assert "overlap day 2024-01-04 missing" in res.output
|
||
assert "re-pulling from 2023-12-25" in res.output
|
||
row = {r.day: r for r in T.read_prices(c, "AAA")}[date(2024, 1, 5)]
|
||
assert row.ret == pytest.approx(60.5 / 55 - 1)
|
||
|
||
|
||
# ── fts ──────────────────────────────────────────────────────────────
|
||
MPFS = '"Medicare Physician Fee Schedule"'
|
||
SNIPPET = (FX / "filing_snippet.htm").read_bytes()
|
||
LICENSED = "a licensed descriptor that no filing quote should ever copy word for word"
|
||
|
||
|
||
def _hit_json(cik, adsh, filename, sics, date_="2023-03-01"):
|
||
return {
|
||
"_id": f"{adsh}:{filename}",
|
||
"_source": {
|
||
"ciks": [cik],
|
||
"adsh": adsh,
|
||
"form": "10-K",
|
||
"file_date": date_,
|
||
"period_ending": "2022-12-31",
|
||
"sics": sics,
|
||
"display_names": [f"Filer {cik} (CIK {cik})"],
|
||
},
|
||
}
|
||
|
||
|
||
HITS = [
|
||
_hit_json("0000012345", "0000012345-23-000001", "a10k.htm", ["8071"]),
|
||
_hit_json("0000000002", "0000000002-23-000001", "reit.htm", ["6798"]),
|
||
_hit_json("0000012346", "0000012346-23-000001", "ex99.pdf", ["8082"]),
|
||
_hit_json("0000012347", "0000012347-23-000001", "gone.htm", ["8082"]),
|
||
]
|
||
|
||
|
||
def _fts_net(
|
||
monkeypatch, hits=HITS, docs=None, forbid_docs=False, status=200, broken=None
|
||
):
|
||
"""A mock EDGAR: the term "Medicare Physician Fee Schedule" has *hits* in
|
||
2023; every other query is empty."""
|
||
docs = docs if docs is not None else {"a10k.htm": SNIPPET}
|
||
requested: list[str] = []
|
||
|
||
def efts(m):
|
||
p = dict(httpx.URL(m.group(0)).params)
|
||
mine = p["q"] == MPFS and p["from"] == "0"
|
||
return httpx.Response(
|
||
status,
|
||
json={
|
||
"hits": {
|
||
"total": {"value": len(hits) if mine else 0},
|
||
"hits": hits if mine else [],
|
||
}
|
||
},
|
||
)
|
||
|
||
def doc(m):
|
||
requested.append(m.group(1))
|
||
if broken and m.group(1) in broken:
|
||
if broken[m.group(1)] == "timeout":
|
||
raise httpx.ReadTimeout("slow")
|
||
return httpx.Response(broken[m.group(1)])
|
||
body = docs.get(m.group(1))
|
||
if forbid_docs and body:
|
||
raise AssertionError("no document download expected")
|
||
return httpx.Response(200, content=body) if body else httpx.Response(404)
|
||
|
||
def sub(m):
|
||
return httpx.Response(200, json={**_sub_for(m.group(1)), "tickers": ["ZZZ"]})
|
||
|
||
_online(
|
||
monkeypatch,
|
||
[
|
||
(r"efts\.sec\.gov/.*", efts),
|
||
(r"/Archives/edgar/data/\d+/\d+/([^/?]+)$", doc),
|
||
(
|
||
r"company_tickers\.json",
|
||
lambda m: httpx.Response(
|
||
200, content=(FX / "company_tickers.json").read_bytes()
|
||
),
|
||
),
|
||
(r"CIK(\d{10})\.json", sub),
|
||
],
|
||
)
|
||
return requested
|
||
|
||
|
||
FTS = ["market", "fts", "--program", "PFS", "--from", "2023", "--to", "2023"]
|
||
|
||
|
||
def test_fts_cli_dry_run_lists_queries(env, monkeypatch):
|
||
c, _, published, hoard = env
|
||
monkeypatch.setattr(
|
||
http, "client", lambda *a, **k: (_ for _ in ()).throw(AssertionError("net"))
|
||
)
|
||
res = runner.invoke(app, [*FTS, "--dry-run"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "program PFS: 5 terms, 5 queries" in res.output
|
||
assert "5 queries (0 already hoarded)" in res.output
|
||
assert published == []
|
||
full = runner.invoke(app, ["market", "fts", "--dry-run"])
|
||
assert "program MSSP: 1 terms" in full.output
|
||
assert "program CARE-MANAGEMENT: 2 terms" in full.output
|
||
assert f"{(5 + 2 + 1 + 1 + 1 + 2 + 2) * (dt_year() - 2000)} queries" in full.output
|
||
|
||
|
||
def dt_year():
|
||
import datetime
|
||
|
||
return datetime.date.today().year
|
||
|
||
|
||
def test_fts_cli_family_scope_and_bad_keys(env, monkeypatch):
|
||
monkeypatch.setattr("pfs.families.refresh_from", lambda con: 0)
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"fts",
|
||
"--family",
|
||
"HOME-VISITS",
|
||
"--dry-run",
|
||
"--from",
|
||
"2023",
|
||
"--to",
|
||
"2023",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "family HOME-VISITS: 8 terms, 8 queries" in res.output
|
||
assert (
|
||
runner.invoke(app, ["market", "fts", "--family", "NOPE", "--dry-run"]).exit_code
|
||
== 2
|
||
)
|
||
assert (
|
||
runner.invoke(
|
||
app, ["market", "fts", "--program", "NOPE", "--dry-run"]
|
||
).exit_code
|
||
== 2
|
||
)
|
||
|
||
|
||
def test_fts_cli_without_write_stores_nothing(env, monkeypatch):
|
||
c, _, published, hoard = env
|
||
_fts_net(monkeypatch)
|
||
monkeypatch.setattr(
|
||
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
||
)
|
||
res = runner.invoke(app, FTS)
|
||
assert res.exit_code == 0, res.output
|
||
assert "Medicare Physician Fee Schedule 10-K 2023: 4 hits" in res.output
|
||
assert "passages kept 1," in res.output and "new entities 1" in res.output
|
||
assert "fts links 1 (PFS 1)" in res.output and "dry run" in res.output
|
||
assert (
|
||
"gap: sec-edgar:fts: 0000012346-23-000001:ex99.pdf: format not readable"
|
||
in res.output
|
||
)
|
||
assert (
|
||
"gap: sec-edgar:fts: 0000012347-23-000001:gone.htm: document not found"
|
||
in res.output
|
||
)
|
||
assert published == []
|
||
# hoarded: the pages and the one downloaded readable document
|
||
assert (
|
||
hoard / "sec/fts/medicare-physician-fee-schedule/10-K/2023.0.json"
|
||
).is_file()
|
||
assert (hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm").is_file()
|
||
|
||
|
||
def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch):
|
||
c, store, published, hoard = env
|
||
curated = T.LinkRow(
|
||
"0000012345", "program", "APM", "curated", "curated", "adsh", "x", "", "K"
|
||
)
|
||
T.write_links(c, [curated])
|
||
_fts_net(monkeypatch)
|
||
res = runner.invoke(app, [*FTS, "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
(ent,) = T.read_entities(c, "company")
|
||
assert ent.key == "0000012345" and ent.sic == "8082"
|
||
assert [s.ticker for s in T.read_securities(c, ent.key)] == ["ZZZ"]
|
||
assert [f.form for f in T.read_filings(c, ent.key)] == ["10-K"]
|
||
assert [l.target for l in T.read_links(c, basis="sic")] == ["8082"]
|
||
assert T.read_links(c, basis="curated") == [curated] # survives
|
||
(fts_link,) = T.read_links(c, basis="fts")
|
||
assert (fts_link.scope, fts_link.target, fts_link.strength) == (
|
||
"program",
|
||
"PFS",
|
||
"mention",
|
||
)
|
||
assert fts_link.item_key and fts_link.passage_id.startswith(
|
||
"0000012345-23-000001:a10k.htm:"
|
||
)
|
||
# the link's passage is verbatim in the hoarded filing, short, and cited
|
||
(p,) = [p for p in T.read_passages(c) if p.passage_id == fts_link.passage_id]
|
||
assert len(p.passage) <= 400 and p.item_key == fts_link.item_key
|
||
text = edgar.read_doc(hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm")
|
||
assert p.passage in text and p.label == "" and p.label_version == ""
|
||
assert len(T.read_hits(c)) == 4 and len(T.read_passages(c)) == 1
|
||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
|
||
assert "2 piece(s) missing" in T.read_gaps(c)[0].note
|
||
logs = {
|
||
t
|
||
for (t,) in c.execute(
|
||
"SELECT table_name FROM cms.ingest_log WHERE module = 'market.edgar'"
|
||
).fetchall()
|
||
}
|
||
assert {"market.filing_hit", "market.filing_passage", "market.entity_link"} <= logs
|
||
assert published == [True] and store.count() == 1
|
||
# a rerun is idempotent and downloads nothing new
|
||
n_docs = _fts_net(monkeypatch, forbid_docs=True)
|
||
res2 = runner.invoke(app, [*FTS, "--write"])
|
||
assert res2.exit_code == 0
|
||
assert "passages kept 0," in res2.output # stored passages are not re-read
|
||
assert (
|
||
_count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1
|
||
)
|
||
assert n_docs == [] # the 404 is remembered, not asked again
|
||
assert "gone.htm: document not found" in res2.output
|
||
|
||
|
||
def test_fts_cli_refusal_prints_gap(env, monkeypatch):
|
||
c, _, published, _ = env
|
||
_fts_net(monkeypatch, status=403)
|
||
res = runner.invoke(app, [*FTS, "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
|
||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
|
||
assert _count(c, "market.filing_hit") == 0 and _count(c, "market.entity") == 0
|
||
assert published == [True]
|
||
|
||
|
||
def test_fts_cli_withholds_ama_passages_and_drops_unsupported_links(env, monkeypatch):
|
||
c, _, _, _ = env
|
||
doc = f"<p>The Medicare Physician Fee Schedule: {LICENSED}.</p>".encode()
|
||
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": doc})
|
||
res = runner.invoke(app, [*FTS, "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "passages kept 0, withheld by the AMA guard 1" in res.output
|
||
assert "fts links 0 (none)" in res.output
|
||
assert T.read_links(c, basis="fts") == [] and T.read_passages(c) == []
|
||
assert len(T.read_hits(c)) == 1 # the hit stays
|
||
|
||
|
||
def test_fts_cli_unverifiable_quote_drops_link(env, monkeypatch):
|
||
c, _, _, hoard = env
|
||
_fts_net(monkeypatch, hits=HITS[:1])
|
||
real = mk._verified
|
||
|
||
def tamper(links, passages):
|
||
for p in passages.values():
|
||
p.__dict__["passage"] = "not in the filing"
|
||
return real(links, passages)
|
||
|
||
monkeypatch.setattr(mk, "_verified", tamper)
|
||
res = runner.invoke(app, [*FTS, "--write"])
|
||
assert "fts links 0" in res.output and "quote not found in the hoard" in res.output
|
||
assert T.read_links(c, basis="fts") == []
|
||
|
||
|
||
def test_verified_handles_missing_filing_and_unpassaged_links(env):
|
||
_, _, _, hoard = env
|
||
link = T.LinkRow(
|
||
"1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "A:b.htm:0"
|
||
)
|
||
p = T.PassageRow("A:b.htm:0", "1", "A", "b.htm", "t", 0, "x")
|
||
assert mk._verified([link], {("A:b.htm:0", "t"): p}) == ([], 1)
|
||
bare = T.LinkRow("1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "")
|
||
assert mk._verified([bare], {}) == ([], 1)
|
||
|
||
|
||
def test_fts_cli_limit_defers_downloads_and_writes_nothing(env, monkeypatch):
|
||
c, _, published, _ = env
|
||
hits = [
|
||
HITS[0],
|
||
_hit_json(
|
||
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
|
||
),
|
||
]
|
||
_fts_net(monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET, "b.htm": SNIPPET})
|
||
res = runner.invoke(app, [*FTS, "--limit", "1", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "1 filing(s) not downloaded yet" in res.output
|
||
assert "incomplete (limit or time): nothing written" in res.output
|
||
assert published == []
|
||
# the newest filing was fetched first
|
||
assert "a10k.htm" in str(list(http.hoard("sec", "docs").rglob("*.htm")))
|
||
|
||
|
||
def test_fts_cli_deadline_stops_fetching(env, monkeypatch):
|
||
c, _, published, _ = env
|
||
_fts_net(monkeypatch)
|
||
ticks = iter(range(0, 10_000, 100))
|
||
monkeypatch.setattr(mk.time, "monotonic", lambda: next(ticks))
|
||
res = runner.invoke(app, [*FTS, "--max-minutes", "1", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "incomplete" in res.output and published == []
|
||
|
||
|
||
def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch):
|
||
c, _, _, _ = env
|
||
_fts_net(monkeypatch)
|
||
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
|
||
(p,) = [p for p in T.read_passages(c) if p.passage.startswith("Substantially")]
|
||
c.execute(
|
||
"UPDATE market.filing_passage SET label = 'risk_factor', label_version = 'v' "
|
||
"WHERE passage_id = ?",
|
||
[p.passage_id],
|
||
)
|
||
_fts_net(monkeypatch) # a refresh reads the documents again
|
||
res = runner.invoke(app, [*FTS, "--refresh", "--write"])
|
||
assert res.exit_code == 0 and "new entities 0" in res.output
|
||
got = {q.passage_id: q for q in T.read_passages(c)}
|
||
assert (
|
||
got[p.passage_id].label == "risk_factor"
|
||
and got[p.passage_id].label_version == "v"
|
||
)
|
||
(link,) = T.read_links(c, basis="fts")
|
||
assert link.strength == "risk_factor" and link.passage_id == p.passage_id
|
||
|
||
|
||
def test_fts_cli_missing_submissions_is_a_gap(env, monkeypatch):
|
||
c, _, _, _ = env
|
||
_fts_net(monkeypatch, hits=HITS[:1])
|
||
_online(
|
||
monkeypatch,
|
||
[
|
||
(
|
||
r"efts\.sec\.gov/.*",
|
||
lambda m: (
|
||
httpx.Response(
|
||
200, json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
|
||
)
|
||
if "Medicare+Physician" in m.group(0)
|
||
or "Medicare%20Physician" in m.group(0)
|
||
else httpx.Response(
|
||
200, json={"hits": {"total": {"value": 0}, "hits": []}}
|
||
)
|
||
),
|
||
),
|
||
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
|
||
(
|
||
r"company_tickers\.json",
|
||
lambda m: httpx.Response(
|
||
200, content=(FX / "company_tickers.json").read_bytes()
|
||
),
|
||
),
|
||
],
|
||
)
|
||
res = runner.invoke(app, [*FTS, "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "CIK0000012345: no submissions, no entity" in res.output
|
||
assert T.read_entities(c) == []
|
||
|
||
|
||
# classification: the local model, resumable through the ledger
|
||
class _Ask:
|
||
model = "qwen-test"
|
||
|
||
def __init__(self, answers):
|
||
self.answers = answers
|
||
self.calls = 0
|
||
|
||
def __call__(self, system, user, schema):
|
||
self.calls += 1
|
||
return self.answers.pop(0)
|
||
|
||
|
||
TWO = [
|
||
HITS[0],
|
||
_hit_json("0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"),
|
||
]
|
||
TWO_DOCS = {"a10k.htm": SNIPPET, "b.htm": SNIPPET}
|
||
|
||
|
||
def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path):
|
||
c, _, _, _ = env
|
||
monkeypatch.setattr(
|
||
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / f"{task}-{fam}.jsonl"
|
||
)
|
||
# a passage of a term outside this run's scope is never sent to the model
|
||
T.write_passages(
|
||
c, [T.PassageRow("X:y.htm:1", "9", "X", "y.htm", "other term", 1, "z")]
|
||
)
|
||
ask = _Ask(
|
||
[
|
||
{"label": "revenue_exposure", "reason": "r"},
|
||
None,
|
||
{"label": "incidental", "reason": "i"},
|
||
]
|
||
)
|
||
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
||
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
|
||
assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first
|
||
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "classified in 1 round(s): {'done': 1" in res.output and ask.calls == 1
|
||
labelled = [p for p in T.read_passages(c) if p.label]
|
||
assert len(labelled) == 1 and labelled[0].label == "revenue_exposure"
|
||
assert labelled[0].label_version == "model-reading exposure-v1 qwen-test"
|
||
# resume: the done unit is skipped; the next call fails (None) and stays unlabelled
|
||
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
||
assert res.exit_code == 0 and ask.calls == 2
|
||
assert "'failed': 1" in res.output
|
||
# a rerun without --classify keeps what is stored
|
||
res = runner.invoke(app, [*FTS, "--write"])
|
||
assert len([p for p in T.read_passages(c) if p.label]) == 1
|
||
got = sorted(l.strength for l in T.read_links(c, basis="fts"))
|
||
assert got == ["mention", "revenue_exposure"]
|
||
|
||
|
||
def test_fts_cli_incidental_label_makes_no_link(env, monkeypatch, tmp_path):
|
||
c, _, _, _ = env
|
||
monkeypatch.setattr(
|
||
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
|
||
)
|
||
ask = _Ask([{"label": "incidental", "reason": "i"}] * 2)
|
||
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
||
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
|
||
res = runner.invoke(app, [*FTS, "--classify", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert T.read_links(c, basis="fts") == [] and len(T.read_passages(c)) == 2
|
||
|
||
|
||
def test_asker_builds_json_extractor(monkeypatch):
|
||
sentinel = object()
|
||
seen = {}
|
||
monkeypatch.setattr("llm.config.load", lambda: "cfg")
|
||
monkeypatch.setattr(
|
||
"llm.pool.HostPool.from_config", classmethod(lambda cls, cfg: ("pool", cfg))
|
||
)
|
||
|
||
def make(cfg, pool):
|
||
seen["args"] = (cfg, pool)
|
||
return sentinel
|
||
|
||
monkeypatch.setattr("llm.structured.json_extractor", make)
|
||
assert mk._asker() is sentinel
|
||
assert seen["args"] == ("cfg", ("pool", "cfg"))
|
||
|
||
|
||
def test_gap_note_truncates():
|
||
note = mk._gap_note([f"i{n}" for n in range(20)])
|
||
assert note.startswith("20 piece(s) missing: i0; i1") and "... 5 more" in note
|
||
|
||
|
||
def test_fts_cli_submissions_refusal_is_a_gap(env, monkeypatch):
|
||
c, _, published, _ = env
|
||
_fts_net(monkeypatch, hits=HITS[:1])
|
||
_online(
|
||
monkeypatch,
|
||
[
|
||
(
|
||
r"efts\.sec\.gov/.*",
|
||
lambda m: httpx.Response(
|
||
200,
|
||
json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
|
||
if "Medicare%20Physician" in m.group(0)
|
||
or "Medicare+Physician" in m.group(0)
|
||
else {"hits": {"total": {"value": 0}, "hits": []}},
|
||
),
|
||
),
|
||
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
|
||
(r"CIK\d{10}\.json", lambda m: httpx.Response(403)),
|
||
(r"company_tickers\.json", lambda m: httpx.Response(403)),
|
||
],
|
||
)
|
||
res = runner.invoke(app, [*FTS, "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
|
||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
|
||
assert _count(c, "market.filing_hit") == 0 and published == [True]
|
||
|
||
|
||
def test_fts_cli_long_gap_list_is_abridged(env, monkeypatch):
|
||
c, _, _, _ = env
|
||
hits = [
|
||
_hit_json("0000012346", f"0000012346-23-{n:06d}", "x.pdf", ["8082"])
|
||
for n in range(22)
|
||
]
|
||
_fts_net(monkeypatch, hits=hits)
|
||
res = runner.invoke(app, [*FTS, "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: sec-edgar:fts: ... 2 more" in res.output
|
||
assert "... 7 more" in T.read_gaps(c)[0].note
|
||
|
||
|
||
def test_fts_cli_flaky_document_is_skipped(env, monkeypatch):
|
||
hits = [
|
||
HITS[0],
|
||
_hit_json(
|
||
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
|
||
),
|
||
]
|
||
_fts_net(
|
||
monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET}, broken={"b.htm": "timeout"}
|
||
)
|
||
monkeypatch.setattr("time.sleep", lambda s: None)
|
||
res = runner.invoke(app, FTS)
|
||
assert res.exit_code == 0, res.output
|
||
assert "b.htm: sec-edgar: HTTP 0 ReadTimeout" in res.output
|
||
assert "passages kept 1," in res.output
|
||
|
||
|
||
def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch):
|
||
c, _, published, _ = env
|
||
_fts_net(monkeypatch, hits=HITS[:1], broken={"a10k.htm": 403})
|
||
res = runner.invoke(app, [*FTS, "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: sec-edgar:fts: sec-edgar: HTTP 403" in res.output
|
||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [
|
||
True
|
||
]
|
||
|
||
|
||
TWICE = (
|
||
b"<p>First, we are paid under the Medicare Physician Fee Schedule rates.</p>"
|
||
b"<p>Second, our risk comes from the Medicare Physician Fee Schedule cuts.</p>"
|
||
)
|
||
|
||
|
||
def test_fts_cli_classify_falls_back_to_the_next_passage(env, monkeypatch, tmp_path):
|
||
"""An incidental first passage retires it; the link moves to the next
|
||
passage, which the next round reads."""
|
||
c, _, _, _ = env
|
||
monkeypatch.setattr(
|
||
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
|
||
)
|
||
ask = _Ask(
|
||
[
|
||
{"label": "incidental", "reason": "i"},
|
||
{"label": "risk_factor", "reason": "r"},
|
||
]
|
||
)
|
||
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
||
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE})
|
||
res = runner.invoke(app, [*FTS, "--classify", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "classified in 2 round(s)" in res.output and ask.calls == 2
|
||
(link,) = T.read_links(c, basis="fts")
|
||
assert link.strength == "risk_factor"
|
||
labels = {p.passage_id: p.label for p in T.read_passages(c)}
|
||
assert sorted(labels.values()) == ["incidental", "risk_factor"]
|
||
text = {p.passage_id: p.passage for p in T.read_passages(c)}[link.passage_id]
|
||
assert text.startswith("Second")
|
||
|
||
|
||
def test_fts_cli_classify_limit_stops_between_rounds(env, monkeypatch, tmp_path):
|
||
c, _, _, _ = env
|
||
monkeypatch.setattr(
|
||
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
|
||
)
|
||
ask = _Ask([{"label": "incidental", "reason": "i"}] * 3)
|
||
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
||
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE})
|
||
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert ask.calls == 1 and "classified in 1 round(s)" in res.output
|
||
# the link now rests on the second, still unlabelled passage
|
||
(link,) = T.read_links(c, basis="fts")
|
||
assert link.strength == "mention"
|
||
|
||
|
||
# ── lda ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _lda_rec(**over):
|
||
rec = json.loads((FX / "lda_aafp_2024_q1.json").read_text())
|
||
rec.update(over)
|
||
return rec
|
||
|
||
|
||
def _lda_entities(c, *extra):
|
||
T.write_entities(
|
||
c,
|
||
[
|
||
T.EntityRow(
|
||
"AAFP",
|
||
"American Academy of Family Physicians",
|
||
"organization",
|
||
aliases="AMERICAN ACADEMY OF FAMILY PHYSICIANS",
|
||
),
|
||
T.EntityRow(
|
||
"PRIMARY-CARE-SEAT",
|
||
"Primary care rotating-seat societies",
|
||
"organization",
|
||
aliases="PRIMARY CARE ROTATING SEAT",
|
||
pseudo=True,
|
||
),
|
||
T.EntityRow(
|
||
"0000000001",
|
||
"SWEEP ONLY CO",
|
||
"company",
|
||
cik="0000000001",
|
||
aliases="SWEEP ONLY",
|
||
),
|
||
*extra,
|
||
],
|
||
)
|
||
|
||
|
||
@pytest.fixture
|
||
def lda(env, monkeypatch):
|
||
from market import lda as L
|
||
|
||
c, store, published, hoard = env
|
||
_lda_entities(c)
|
||
seen: list[httpx.Request] = []
|
||
monkeypatch.setattr(L, "new_pacer", lambda key=None: _Pacer())
|
||
monkeypatch.setattr(mk, "_lda_now", lambda: date(2026, 11, 13))
|
||
state = {"results": [_lda_rec()], "status": 200, "text": None}
|
||
|
||
def handler(request):
|
||
seen.append(request)
|
||
if state["status"] != 200:
|
||
return httpx.Response(state["status"], text="no")
|
||
if state["text"] is not None:
|
||
return httpx.Response(200, text=state["text"])
|
||
q = request.url.params
|
||
name = q.get("client_name") or ""
|
||
results = state["results"]
|
||
if name:
|
||
results = [
|
||
r for r in results if name.upper() in r["client"]["name"].upper()
|
||
]
|
||
year = q.get("filing_year")
|
||
if year:
|
||
results = [r for r in results if str(r["filing_year"]) == year]
|
||
return httpx.Response(
|
||
200,
|
||
json={
|
||
"count": len(results),
|
||
"next": None,
|
||
"previous": None,
|
||
"results": results,
|
||
},
|
||
)
|
||
|
||
monkeypatch.setattr(
|
||
http,
|
||
"client",
|
||
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
|
||
)
|
||
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
|
||
return c, store, published, hoard, seen, state
|
||
|
||
|
||
def test_lda_cli_entity_write(lda):
|
||
c, store, published, hoard, seen, state = lda
|
||
foundation = _lda_rec(
|
||
filing_uuid="f0000000-0000-0000-0000-000000000000",
|
||
client={
|
||
**_lda_rec()["client"],
|
||
"name": "AMERICAN ACADEMY OF FAMILY PHYSICIANS FOUNDATION",
|
||
},
|
||
)
|
||
state["results"] = [_lda_rec(), foundation]
|
||
T.write_gap(c, "lda:lda", "old")
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"lda",
|
||
"--entity",
|
||
"AAFP",
|
||
"--from",
|
||
"2024",
|
||
"--to",
|
||
"2024",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
rows = T.read_lda_filings(c)
|
||
assert [(r.filing_uuid[:8], r.entity_key, r.matched_by) for r in rows] == [
|
||
("df5f17c4", "AAFP", "client")
|
||
]
|
||
assert rows[0].amount == 825016.0 and rows[0].source_file.endswith(".1.json")
|
||
assert _count(c, "market.lda_issue") == 5
|
||
assert T.read_gaps(c) == []
|
||
assert published == [True]
|
||
assert {
|
||
(t, n)
|
||
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||
} == {("market.lda_filing", 1), ("market.lda_issue", 5)}
|
||
assert seen and all(r.url.host == "lda.gov" for r in seen)
|
||
assert seen[0].url.params["client_name"] == "American Academy of Family Physicians"
|
||
assert seen[0].url.params["filing_year"] == "2024"
|
||
assert "authorization" not in seen[0].headers
|
||
files = list((hoard / "lda" / "2024").glob("*.1.json"))
|
||
assert files and all(
|
||
"american-academy-of-family-physicians" in f.name for f in files
|
||
)
|
||
assert "1 filings" in res.output and "1 clients" in res.output
|
||
assert "HCR 1" in res.output and "MMM 1" in res.output
|
||
assert [i.title for i in store.list_items() if "Lobbying" in i.title] == [
|
||
"Lobbying Disclosure Act filings, 2024"
|
||
]
|
||
|
||
|
||
def test_lda_cli_without_write_stores_nothing(lda):
|
||
c, _, published, _, seen, _ = lda
|
||
res = runner.invoke(app, ["market", "lda", "--from", "2024", "--to", "2024"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "dry run: pass --write" in res.output
|
||
assert _count(c, "market.lda_filing") == 0
|
||
assert published == []
|
||
names = {r.url.params["client_name"] for r in seen}
|
||
assert "American Academy of Family Physicians" in names
|
||
assert not any("PRIMARY CARE" in n.upper() for n in names) # pseudo skipped
|
||
assert not any("SWEEP" in n.upper() for n in names) # not a seed company
|
||
|
||
|
||
def test_lda_cli_terms_offline(lda):
|
||
c, _, _, hoard, seen, state = lda
|
||
other = _lda_rec(
|
||
filing_uuid="a0000000-0000-0000-0000-000000000000",
|
||
client={**_lda_rec()["client"], "id": 5, "name": "SOME UNTRACKED CLIENT"},
|
||
)
|
||
state["results"] = [_lda_rec(), other]
|
||
cmd = [
|
||
"market",
|
||
"lda",
|
||
"--entity",
|
||
"AAFP",
|
||
"--terms",
|
||
"--from",
|
||
"2024",
|
||
"--to",
|
||
"2024",
|
||
]
|
||
res = runner.invoke(app, [*cmd, "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
terms = [r for r in seen if "filing_specific_lobbying_issues" in r.url.params]
|
||
assert [r.url.params["filing_specific_lobbying_issues"] for r in terms] == list(
|
||
__import__("market.lda", fromlist=["x"]).LDA_TERMS
|
||
)
|
||
got = {
|
||
r.filing_uuid[:2]: (r.entity_key, r.matched_by) for r in T.read_lda_filings(c)
|
||
}
|
||
assert got == {"df": ("AAFP", "client"), "a0": ("", "")}
|
||
# a second run reads only the hoard: not one request leaves
|
||
n = len(seen)
|
||
again = runner.invoke(app, cmd)
|
||
assert again.exit_code == 0 and len(seen) == n
|
||
assert "2 filings" in again.output and "0 requests" in again.output
|
||
assert "terms 5" in again.output
|
||
|
||
|
||
def test_lda_cli_range_uses_all_dir_and_filters_years(lda):
|
||
c, _, _, hoard, seen, state = lda
|
||
old = _lda_rec(filing_uuid="b0000000-0000-0000-0000-000000000000", filing_year=2015)
|
||
state["results"] = [_lda_rec(), old]
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"lda",
|
||
"--entity",
|
||
"AAFP",
|
||
"--from",
|
||
"2020",
|
||
"--to",
|
||
"2024",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "filing_year" not in seen[0].url.params
|
||
assert list((hoard / "lda" / "all").glob("*.1.json"))
|
||
assert [r.filing_year for r in T.read_lda_filings(c)] == [2024]
|
||
|
||
|
||
def test_lda_cli_refusal_prints_gap(lda):
|
||
c, _, published, _, seen, state = lda
|
||
state["status"] = 403
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"lda",
|
||
"--entity",
|
||
"AAFP",
|
||
"--from",
|
||
"2024",
|
||
"--to",
|
||
"2024",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: lda:lda: lda: HTTP 403" in res.output
|
||
assert [(g.source) for g in T.read_gaps(c)] == ["lda:lda"]
|
||
assert _count(c, "market.lda_filing") == 0
|
||
assert len(seen) == 1 # no workaround, no retry on a 403
|
||
state["status"], state["text"] = 200, "<html>Just a moment</html>"
|
||
res = runner.invoke(
|
||
app, ["market", "lda", "--entity", "AAFP", "--from", "2024", "--to", "2024"]
|
||
)
|
||
assert "gap: lda:lda:" in res.output and "not JSON" in res.output
|
||
assert len(T.read_gaps(c)) == 1 # no --write: not recorded again
|
||
|
||
|
||
def test_lda_cli_limit_and_minutes_stop_early_and_keep_the_gap(lda, monkeypatch):
|
||
c, _, _, _, seen, _ = lda
|
||
_lda_entities(
|
||
c,
|
||
T.EntityRow(
|
||
"ACP",
|
||
"American College of Physicians",
|
||
"organization",
|
||
aliases="AMERICAN COLLEGE OF PHYSICIANS",
|
||
),
|
||
)
|
||
T.write_gap(c, "lda:lda", "old")
|
||
res = runner.invoke(
|
||
app,
|
||
["market", "lda", "--from", "2024", "--to", "2024", "--limit", "1", "--write"],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert len(seen) == 1 and "stopped: request limit" in res.output
|
||
assert [g.note for g in T.read_gaps(c)] == ["old"]
|
||
assert _count(c, "market.lda_filing") == 1
|
||
ticks = iter([0.0])
|
||
monkeypatch.setattr(mk.time, "monotonic", lambda: next(ticks, 10_000.0))
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"lda",
|
||
"--from",
|
||
"2024",
|
||
"--to",
|
||
"2024",
|
||
"--max-minutes",
|
||
"1",
|
||
"--refresh",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stopped: time limit" in res.output
|
||
|
||
|
||
def test_lda_cli_bad_entity_and_empty_registry(lda, monkeypatch):
|
||
c, *_ = lda
|
||
res = runner.invoke(app, ["market", "lda", "--entity", "NOPE"])
|
||
assert res.exit_code == 2 and "unknown entity: NOPE" in res.output
|
||
res = runner.invoke(app, ["market", "lda", "--entity", "PRIMARY-CARE-SEAT"])
|
||
assert res.exit_code == 2
|
||
bare = duckdb.connect(":memory:")
|
||
monkeypatch.setattr(mk, "_read", lambda: bare)
|
||
res = runner.invoke(app, ["market", "lda"])
|
||
assert res.exit_code == 1 and "stack market entities --write" in res.output
|
||
|
||
|
||
def test_lda_cli_resupersedes_across_runs(lda):
|
||
c, _, _, _, _, state = lda
|
||
cmd = [
|
||
"market",
|
||
"lda",
|
||
"--entity",
|
||
"AAFP",
|
||
"--from",
|
||
"2024",
|
||
"--to",
|
||
"2024",
|
||
"--write",
|
||
]
|
||
assert runner.invoke(app, cmd).exit_code == 0
|
||
amend = _lda_rec(
|
||
filing_uuid="c0000000-0000-0000-0000-000000000000",
|
||
filing_type="1A",
|
||
dt_posted="2024-05-01T11:45:16-04:00",
|
||
expenses="900000.00",
|
||
)
|
||
state["results"] = [amend]
|
||
assert runner.invoke(app, [*cmd, "--refresh"]).exit_code == 0
|
||
got = {r.filing_uuid[:2]: r.superseded for r in T.read_lda_filings(c)}
|
||
assert got == {"df": True, "c0": False}
|
||
|
||
|
||
def test_lda_cli_year_order_and_seed_company_tracked(lda, monkeypatch):
|
||
c, _, _, _, seen, state = lda
|
||
monkeypatch.undo() # the real clock, only to read it
|
||
assert mk._lda_now() == date.today()
|
||
res = runner.invoke(app, ["market", "lda", "--from", "2025", "--to", "2024"])
|
||
assert res.exit_code == 2 and "is after --to" in res.output
|
||
|
||
|
||
def test_lda_cli_tracks_seed_companies_only(lda):
|
||
c, _, _, _, seen, state = lda
|
||
seed = load_seed()[0][0]
|
||
T.write_entities(
|
||
c,
|
||
[
|
||
T.EntityRow(
|
||
seed.cik,
|
||
seed.name,
|
||
"company",
|
||
cik=seed.cik,
|
||
aliases="|".join(seed.aliases),
|
||
)
|
||
],
|
||
)
|
||
res = runner.invoke(
|
||
app, ["market", "lda", "--entity", seed.cik, "--from", "2024", "--to", "2024"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert {r.url.params["client_name"] for r in seen} >= {seed.name}
|
||
assert "0 filings" in res.output
|
||
|
||
|
||
# ── fec ──────────────────────────────────────────────────────────────
|
||
|
||
FEC_SECRET = "fec-cli-key-do-not-leak"
|
||
|
||
|
||
def _fec_entities(c):
|
||
T.write_entities(
|
||
c,
|
||
[
|
||
T.EntityRow(
|
||
"AAFP",
|
||
"American Academy of Family Physicians",
|
||
"organization",
|
||
aliases="AMERICAN ACADEMY OF FAMILY PHYSICIANS",
|
||
),
|
||
T.EntityRow("AMA", "AMA", "organization", aliases="AMA"),
|
||
T.EntityRow("LOOKALIKE", "LOOKALIKE", "organization", aliases="LOOKALIKE"),
|
||
T.EntityRow(
|
||
"PRIMARY-CARE-SEAT",
|
||
"Primary care rotating-seat societies",
|
||
"organization",
|
||
aliases="PRIMARY CARE ROTATING SEAT",
|
||
pseudo=True,
|
||
),
|
||
],
|
||
)
|
||
|
||
|
||
@pytest.fixture
|
||
def fec(env, monkeypatch):
|
||
from market import fec as fec_api
|
||
|
||
c, store, published, hoard = env
|
||
_fec_entities(c)
|
||
seen: list[httpx.Request] = []
|
||
page1 = json.loads((FX / "fec_schedule_b_page1.json").read_text())
|
||
totals = json.loads((FX / "fec_totals_C00411553_2024.json").read_text())
|
||
committee = json.loads((FX / "fec_committees_aafp.json").read_text())
|
||
monkeypatch.setattr(fec_api, "new_pacer", lambda: _Pacer())
|
||
state = {"status": 200}
|
||
|
||
def handler(request):
|
||
seen.append(request)
|
||
if state["status"] != 200:
|
||
return httpx.Response(state["status"], text="no")
|
||
path = request.url.path
|
||
if path.endswith("/totals/"):
|
||
return httpx.Response(200, json=totals)
|
||
if path.endswith("/schedule_b/"):
|
||
if "last_index" in request.url.params:
|
||
return httpx.Response(200, json={"results": [], "pagination": {}})
|
||
return httpx.Response(200, json=page1)
|
||
if path.startswith("/v1/committees"):
|
||
return httpx.Response(
|
||
200,
|
||
json={
|
||
"results": [
|
||
{
|
||
"committee_id": "C00999999",
|
||
"name": "AMALGAMATED TRANSIT UNION PAC",
|
||
"committee_type": "Q",
|
||
"designation": "U",
|
||
}
|
||
]
|
||
},
|
||
)
|
||
return httpx.Response(200, json=committee)
|
||
|
||
monkeypatch.setattr(
|
||
http,
|
||
"client",
|
||
lambda source, key=None: httpx.Client(
|
||
transport=httpx.MockTransport(handler),
|
||
headers={"X-Api-Key": key} if key else {},
|
||
),
|
||
)
|
||
monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET)
|
||
return c, store, published, hoard, seen, state
|
||
|
||
|
||
def test_fec_cli_write_from_cache(fec):
|
||
c, _, published, hoard, seen, state = fec
|
||
T.write_gap(c, "fec:fec", "old")
|
||
dry = runner.invoke(app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024"])
|
||
assert dry.exit_code == 0, dry.output
|
||
assert "dry run: pass --write" in dry.output
|
||
assert "1 committees (1 curated, 0 exact-name, 0 hoarded)" in dry.output
|
||
assert _count(c, "market.fec_committee") == 0
|
||
assert published == []
|
||
assert seen and all("/committees" not in r.url.path for r in seen)
|
||
assert all(FEC_SECRET not in str(r.url) for r in seen)
|
||
n = len(seen)
|
||
state["status"] = 500
|
||
res = runner.invoke(
|
||
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert len(seen) == n # the hoard answered; the 500 was never requested
|
||
assert "stored: 1 committees, 1 totals, 3 schedule B rows" in res.output
|
||
(committee,) = T.read_fec_committees(c)
|
||
assert committee.basis == "curated" and committee.committee_id == "C00411553"
|
||
assert committee.source_file.endswith("fec/committee/C00411553.json")
|
||
(totals,) = T.read_fec_totals(c)
|
||
assert totals.receipts == 513798.01
|
||
assert totals.fed_candidate_committee_contributions == 683300.0
|
||
names = {r.recipient_name for r in T.read_fec_contributions(c)}
|
||
assert "JANE Q PUBLIC" in names and "MEMO OF THE 100" in names
|
||
from market.fec import contributions_by_cycle
|
||
|
||
assert contributions_by_cycle(c, "AAFP") == {2024: 125.0}
|
||
assert T.read_gaps(c) == []
|
||
assert published == [True]
|
||
assert {
|
||
(t, n)
|
||
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||
} == {
|
||
("market.fec_committee", 1),
|
||
("market.fec_totals", 1),
|
||
("market.fec_contribution", 3),
|
||
}
|
||
manifest = (hoard / "MANIFEST.jsonl").read_text()
|
||
assert FEC_SECRET not in manifest
|
||
assert (hoard / "fec" / "schedule_b" / "C00411553" / "2024" / "1.json").is_file()
|
||
|
||
|
||
def test_fec_cli_no_write_prints_candidates(fec):
|
||
c, _, published, _, seen, _ = fec
|
||
res = runner.invoke(
|
||
app, ["market", "fec", "--entity", "LOOKALIKE", "--cycles", "2024"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "candidate: LOOKALIKE C00999999 AMALGAMATED TRANSIT UNION PAC" in res.output
|
||
assert "0 committees" in res.output and "1 candidates" in res.output
|
||
assert "dry run: pass --write" in res.output
|
||
assert _count(c, "market.fec_committee") == 0
|
||
assert _count(c, "market.fec_contribution") == 0
|
||
assert published == []
|
||
assert seen[0].url.params["q"] == "LOOKALIKE"
|
||
assert all(
|
||
"/committee/" not in r.url.path or r.url.path.startswith("/v1/committees")
|
||
for r in seen
|
||
)
|
||
default = runner.invoke(app, ["market", "fec", "--entity", "LOOKALIKE"])
|
||
assert default.exit_code == 0, default.output
|
||
assert "1 candidates" in default.output
|
||
bad = runner.invoke(app, ["market", "fec", "--entity", "NOPE", "--cycles", "2024"])
|
||
assert bad.exit_code == 2 and "unknown entity: NOPE" in bad.output
|
||
cycles = runner.invoke(app, ["market", "fec", "--cycles", "nope"])
|
||
assert cycles.exit_code == 2 and "not a list of years" in cycles.output
|
||
|
||
|
||
def test_fec_cli_full_pull_keeps_hoard_ids(fec, monkeypatch):
|
||
from market import fec as fec_api
|
||
|
||
c, _, published, hoard, seen, _ = fec
|
||
monkeypatch.setattr(fec_api, "hoard_ids", lambda: ("C00765230",))
|
||
committee = json.loads((FX / "fec_committees_aafp.json").read_text())
|
||
totals = json.loads((FX / "fec_totals_C00411553_2024.json").read_text())
|
||
page1 = json.loads((FX / "fec_schedule_b_page1.json").read_text())
|
||
|
||
def handler(request):
|
||
seen.append(request)
|
||
path = request.url.path
|
||
if path == "/v1/committee/C00765230/":
|
||
return httpx.Response(
|
||
200,
|
||
json={
|
||
"results": [
|
||
{
|
||
"committee_id": "C00765230",
|
||
"name": "AMERICAN ANTHEM PAC",
|
||
"committee_type": "V",
|
||
"designation": "U",
|
||
}
|
||
]
|
||
},
|
||
)
|
||
cid = request.url.params.get("committee_id", "")
|
||
if path.endswith("/totals/"):
|
||
if cid == "C00765230":
|
||
return httpx.Response(200, json={"results": []})
|
||
return httpx.Response(200, json=totals)
|
||
if path.endswith("/schedule_b/"):
|
||
if cid == "C00765230" or "last_index" in request.url.params:
|
||
return httpx.Response(200, json={"results": [], "pagination": {}})
|
||
return httpx.Response(200, json=page1)
|
||
if path.startswith("/v1/committees"):
|
||
return httpx.Response(200, json={"results": []})
|
||
if path.endswith("/C00411553/"):
|
||
return httpx.Response(200, json=committee)
|
||
return httpx.Response(200, json={"results": []})
|
||
|
||
monkeypatch.setattr(
|
||
http,
|
||
"client",
|
||
lambda source, key=None: httpx.Client(
|
||
transport=httpx.MockTransport(handler),
|
||
headers={"X-Api-Key": key} if key else {},
|
||
),
|
||
)
|
||
res = runner.invoke(app, ["market", "fec", "--cycles", "2024", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "hoarded: C00765230 AMERICAN ANTHEM PAC" in res.output
|
||
rows = {r.committee_id: r for r in T.read_fec_committees(c)}
|
||
assert rows["C00765230"].basis == "hoarded" and rows["C00765230"].entity_key == ""
|
||
assert rows["C00411553"].basis == "curated"
|
||
assert (hoard / "fec" / "committee" / "C00765230.json").is_file()
|
||
from market.fec import contributions_by_cycle
|
||
|
||
assert contributions_by_cycle(c, "") == {}
|
||
assert contributions_by_cycle(c, "AMA") == {}
|
||
assert published == [True]
|
||
|
||
|
||
def test_fec_cli_refusal_prints_gap(fec, monkeypatch):
|
||
c, _, published, _, seen, state = fec
|
||
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
|
||
missing = runner.invoke(
|
||
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
|
||
)
|
||
assert missing.exit_code == 0, missing.output
|
||
assert "gap: fec:fec:" in missing.output
|
||
assert "FEC_API_KEY" in missing.output
|
||
assert seen == []
|
||
assert [g.source for g in T.read_gaps(c)] == ["fec:fec"]
|
||
assert published == [True]
|
||
monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET)
|
||
state["status"] = 403
|
||
res = runner.invoke(
|
||
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: fec:fec: fec: HTTP 403" in res.output
|
||
assert FEC_SECRET not in res.output
|
||
assert _count(c, "market.fec_committee") == 0
|
||
assert len(seen) == 1
|
||
assert all(FEC_SECRET not in str(r.url) for r in seen)
|
||
assert seen[0].headers["x-api-key"] == FEC_SECRET
|
||
|
||
|
||
def test_fec_cli_limit_keeps_the_gap_and_empty_registry(fec, monkeypatch):
|
||
c, _, _, _, seen, _ = fec
|
||
T.write_gap(c, "fec:fec", "old")
|
||
res = runner.invoke(
|
||
app,
|
||
["market", "fec", "--cycles", "2024,2022", "--limit", "1", "--write"],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stopped: request limit" in res.output
|
||
assert [g.note for g in T.read_gaps(c)] == ["old"]
|
||
assert _count(c, "market.fec_committee") == 1
|
||
assert _count(c, "market.fec_totals") == 0
|
||
assert len(seen) == 1
|
||
monkeypatch.setattr(mk, "_lda_tracked", lambda con: [])
|
||
empty = runner.invoke(app, ["market", "fec", "--cycles", "2024"])
|
||
assert empty.exit_code == 1 and "no tracked entities" in empty.output
|
||
blank = runner.invoke(app, ["market", "fec", "--cycles", ","])
|
||
assert blank.exit_code == 2 and "--cycles is empty" in _plain(blank.output)
|
||
|
||
|
||
# ── congress ─────────────────────────────────────────────────────────
|
||
|
||
CONGRESS_SECRET = "congress-cli-key-do-not-leak"
|
||
|
||
|
||
def _congress_states(c) -> None:
|
||
c.execute("CREATE SCHEMA IF NOT EXISTS reference_data")
|
||
c.execute(
|
||
"CREATE TABLE IF NOT EXISTS reference_data.ansi_fips_state ("
|
||
"ansi_fips_state_code VARCHAR, ansi_fips_state_abbreviation VARCHAR, "
|
||
"ansi_fips_state_name VARCHAR)"
|
||
)
|
||
c.execute(
|
||
"INSERT INTO reference_data.ansi_fips_state VALUES "
|
||
"('26', 'MI', 'Michigan'), ('02', 'AK', 'Alaska'), ('51', 'VA', 'Virginia')"
|
||
)
|
||
|
||
|
||
@pytest.fixture
|
||
def congress(env, monkeypatch):
|
||
from market import congress as congress_api
|
||
|
||
c, store, published, hoard = env
|
||
_congress_states(c)
|
||
T.ensure_tables(c)
|
||
seen: list[httpx.Request] = []
|
||
members = json.loads((FX / "congress_members_118.json").read_text())
|
||
bills = json.loads((FX / "congress_bills_page.json").read_text())
|
||
detail = json.loads((FX / "congress_bill_detail.json").read_text())
|
||
actions = json.loads((FX / "congress_bill_actions.json").read_text())
|
||
cosponsors = json.loads((FX / "congress_bill_cosponsors.json").read_text())
|
||
people = (FX / "legislators_snippet.yaml").read_bytes()
|
||
membership = (FX / "membership_snippet.yaml").read_bytes()
|
||
state = {"status": 200}
|
||
for name in ("member_pacer", "raw_pacer", "commit_pacer"):
|
||
monkeypatch.setattr(congress_api, name, lambda: _Pacer())
|
||
|
||
def handler(request: httpx.Request) -> httpx.Response:
|
||
seen.append(request)
|
||
if state["status"] != 200:
|
||
return httpx.Response(state["status"], text="no")
|
||
host = request.url.host
|
||
if host == "api.congress.gov":
|
||
path = request.url.path
|
||
if path.endswith("/actions"):
|
||
return httpx.Response(200, json=actions)
|
||
if path.endswith("/cosponsors"):
|
||
return httpx.Response(200, json=cosponsors)
|
||
parts = [part for part in path.split("/") if part]
|
||
if "bill" in parts and len(parts) >= 5:
|
||
return httpx.Response(200, json=detail)
|
||
if "bill" in parts:
|
||
return httpx.Response(200, json=bills)
|
||
return httpx.Response(200, json=members)
|
||
if host == "api.github.com":
|
||
return httpx.Response(200, json=[{"sha": "deadbeef"}])
|
||
if request.url.path.endswith("legislators-historical.yaml"):
|
||
return httpx.Response(200, content=b"[]\n")
|
||
if request.url.path.endswith("committee-membership-current.yaml"):
|
||
return httpx.Response(200, content=membership)
|
||
return httpx.Response(200, content=people)
|
||
|
||
def client(source, key=None):
|
||
headers = {"X-Api-Key": key} if key else {}
|
||
return httpx.Client(transport=httpx.MockTransport(handler), headers=headers)
|
||
|
||
monkeypatch.setattr(http, "client", client)
|
||
monkeypatch.setattr(http, "api_key", lambda source, env=None: CONGRESS_SECRET)
|
||
return c, store, published, hoard, seen, state
|
||
|
||
|
||
def test_members_cli_no_write_prints_counts(congress):
|
||
c, _, published, _, seen, _ = congress
|
||
res = runner.invoke(app, ["market", "members", "--from", "118", "--to", "118"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "2 members (2 house, 0 senate, 1 at-large)" in res.output
|
||
assert "congress 118: 2" in res.output
|
||
assert "dry run: pass --write" in res.output
|
||
assert _count(c, "market.congress_member") == 0
|
||
assert published == []
|
||
assert any(r.url.host == "api.congress.gov" for r in seen)
|
||
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
|
||
assert all("api_key" not in str(r.url) for r in seen)
|
||
congress_hits = [r for r in seen if r.url.host == "api.congress.gov"]
|
||
assert congress_hits[0].headers["x-api-key"] == CONGRESS_SECRET
|
||
github_hits = [r for r in seen if r.url.host != "api.congress.gov"]
|
||
assert github_hits
|
||
assert all(r.headers.get("x-api-key") in (None, "") for r in github_hits)
|
||
|
||
|
||
def test_members_cli_write(congress):
|
||
c, _, published, hoard, seen, _ = congress
|
||
T.write_gap(c, "congress:members", "old")
|
||
res = runner.invoke(
|
||
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stored: 2 members" in res.output
|
||
rows = {r.bioguide_id: r for r in T.read_members(c)}
|
||
assert rows["D000624"].district_id == "MI-12"
|
||
assert rows["D000624"].fec_ids == "H0XX00001"
|
||
assert rows["D000624"].source == "congress.gov"
|
||
assert rows["Y000033"].district_id == "AK-00"
|
||
assert rows["Y000033"].state == "AK"
|
||
assert T.read_gaps(c) == []
|
||
assert published == [True]
|
||
assert {
|
||
(t, n)
|
||
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||
} == {("market.congress_member", 2)}
|
||
manifest = (hoard / "MANIFEST.jsonl").read_text()
|
||
assert CONGRESS_SECRET not in manifest
|
||
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
|
||
kept = T.MemberRow(
|
||
117,
|
||
"Z000001",
|
||
"Other",
|
||
"Republican",
|
||
"MI",
|
||
"MI-01",
|
||
"house",
|
||
"",
|
||
"congress.gov",
|
||
)
|
||
T.write_members(c, [kept])
|
||
assert {r.congress for r in T.read_members(c)} == {117, 118}
|
||
|
||
|
||
def test_members_cli_missing_key_gap_only(congress, monkeypatch):
|
||
c, _, published, _, seen, _ = congress
|
||
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
|
||
res = runner.invoke(
|
||
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: congress:members:" in res.output
|
||
assert "CONGRESS_API_KEY" in res.output
|
||
assert "API_DATA_GOV_KEY" in res.output
|
||
assert seen == []
|
||
assert [g.source for g in T.read_gaps(c)] == ["congress:members"]
|
||
assert _count(c, "market.congress_member") == 0
|
||
assert published == [True]
|
||
|
||
|
||
def test_members_cli_refusal_prints_gap(congress):
|
||
c, _, published, _, seen, state = congress
|
||
state["status"] = 403
|
||
res = runner.invoke(
|
||
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: congress:members: congress: HTTP 403" in res.output
|
||
assert CONGRESS_SECRET not in res.output
|
||
assert _count(c, "market.congress_member") == 0
|
||
assert len(seen) == 1
|
||
assert seen[0].url.host == "api.congress.gov"
|
||
assert seen[0].headers["x-api-key"] == CONGRESS_SECRET
|
||
assert CONGRESS_SECRET not in str(seen[0].url)
|
||
|
||
|
||
def test_members_cli_from_after_to(congress):
|
||
res = runner.invoke(app, ["market", "members", "--from", "119", "--to", "110"])
|
||
assert res.exit_code == 2
|
||
assert "--from 119 is after --to 110" in res.output
|
||
|
||
|
||
def test_committees_cli_write(congress):
|
||
c, _, published, hoard, seen, _ = congress
|
||
T.write_gap(c, "legislators:committees", "old")
|
||
dry = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "119"])
|
||
assert dry.exit_code == 0, dry.output
|
||
assert "1 assignments" in dry.output
|
||
assert "dry run: pass --write" in dry.output
|
||
assert _count(c, "market.committee_assignment") == 0
|
||
assert published == []
|
||
res = runner.invoke(
|
||
app, ["market", "committees", "--from", "119", "--to", "119", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stored: 1 assignments" in res.output
|
||
(row,) = T.read_assignments(c)
|
||
assert row.committee_id == "HSWM02"
|
||
assert row.bioguide_id == "E000001"
|
||
assert row.snapshot == date(2025, 4, 1)
|
||
assert row.source_sha == "deadbeef"
|
||
assert row.congress == 119
|
||
assert T.read_gaps(c) == []
|
||
assert published == [True]
|
||
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
|
||
assert all(r.headers.get("x-api-key") in (None, "") for r in seen)
|
||
manifest = (hoard / "MANIFEST.jsonl").read_text()
|
||
assert CONGRESS_SECRET not in manifest
|
||
logged = c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||
assert ("market.committee_assignment", 1) in logged
|
||
|
||
|
||
def test_bills_cli_no_write_lists_matches(congress):
|
||
c, _, published, _, seen, _ = congress
|
||
res = runner.invoke(app, ["market", "bills", "--from", "118", "--to", "118"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "118 HR 1234 physician fee schedule" in res.output
|
||
assert "1 bills" in res.output
|
||
assert "congress 118: 1" in res.output
|
||
assert "Farm Bill" not in res.output
|
||
assert "dry run: pass --write" in res.output
|
||
assert _count(c, "market.bill") == 0
|
||
assert published == []
|
||
hits = [r for r in seen if r.url.host == "api.congress.gov"]
|
||
assert hits
|
||
assert all(CONGRESS_SECRET not in str(r.url) for r in hits)
|
||
assert all("api_key" not in str(r.url) for r in hits)
|
||
assert hits[0].headers["x-api-key"] == CONGRESS_SECRET
|
||
|
||
|
||
def test_bills_cli_offline_write(congress):
|
||
c, store, published, hoard, seen, _ = congress
|
||
T.write_gap(c, "congress:bills", "old")
|
||
res = runner.invoke(
|
||
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stored: 1 bills" in res.output
|
||
(bill,) = T.read_bills(c)
|
||
assert bill.bill_type == "HR"
|
||
assert bill.number == 1234
|
||
assert bill.matched_term == "physician fee schedule"
|
||
assert bill.policy_area == "Health"
|
||
assert bill.introduced == date(2023, 1, 9)
|
||
assert bill.item_key
|
||
assert len(T.read_bill_actions(c)) == 3
|
||
roles = sorted((m.role, m.bioguide_id) for m in T.read_bill_members(c))
|
||
assert roles == [
|
||
("cosponsor", "S000001"),
|
||
("cosponsor", "Y000033"),
|
||
("sponsor", "D000624"),
|
||
]
|
||
assert T.read_gaps(c) == []
|
||
assert published == [True]
|
||
assert any(
|
||
(i.url or "").endswith("/118th-congress/house-bill/1234")
|
||
for i in store.list_items()
|
||
)
|
||
assert ("market.bill", 1) in c.execute(
|
||
"SELECT table_name, rows FROM cms.ingest_log"
|
||
).fetchall()
|
||
manifest = (hoard / "MANIFEST.jsonl").read_text()
|
||
assert CONGRESS_SECRET not in manifest
|
||
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
|
||
kept = T.BillRow(
|
||
117,
|
||
"S",
|
||
1,
|
||
"Telehealth Modernization Act",
|
||
None,
|
||
"Health",
|
||
None,
|
||
"",
|
||
"telehealth",
|
||
"https://www.congress.gov/bill/117th-congress/senate-bill/1",
|
||
"",
|
||
)
|
||
T.write_bills(c, [kept])
|
||
assert {r.congress for r in T.read_bills(c)} == {117, 118}
|
||
|
||
|
||
def test_bills_cli_refusal_prints_gap(congress):
|
||
c, _, published, _, seen, state = congress
|
||
state["status"] = 403
|
||
res = runner.invoke(
|
||
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: congress:bills: congress: HTTP 403" in res.output
|
||
assert CONGRESS_SECRET not in res.output
|
||
assert _count(c, "market.bill") == 0
|
||
assert len(seen) == 1
|
||
assert seen[0].url.host == "api.congress.gov"
|
||
assert seen[0].headers["x-api-key"] == CONGRESS_SECRET
|
||
assert CONGRESS_SECRET not in str(seen[0].url)
|
||
assert published == [True]
|
||
|
||
|
||
def test_bills_cli_from_after_to(congress):
|
||
res = runner.invoke(app, ["market", "bills", "--from", "119", "--to", "110"])
|
||
assert res.exit_code == 2
|
||
assert "--from 119 is after --to 110" in res.output
|
||
|
||
|
||
def test_bills_cli_missing_key_gap_only(congress, monkeypatch):
|
||
c, _, published, _, seen, _ = congress
|
||
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
|
||
res = runner.invoke(
|
||
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: congress:bills:" in res.output
|
||
assert "CONGRESS_API_KEY" in res.output
|
||
assert seen == []
|
||
assert [g.source for g in T.read_gaps(c)] == ["congress:bills"]
|
||
assert _count(c, "market.bill") == 0
|
||
assert published == [True]
|
||
|
||
|
||
def test_bills_cli_limit_stores_nothing_and_keeps_the_gap(congress):
|
||
c, _, published, _, seen, _ = congress
|
||
T.write_gap(c, "congress:bills", "old")
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"bills",
|
||
"--from",
|
||
"118",
|
||
"--to",
|
||
"119",
|
||
"--limit",
|
||
"1",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stopped: request limit" in res.output
|
||
assert "stored:" not in res.output
|
||
assert [g.note for g in T.read_gaps(c)] == ["old"]
|
||
assert _count(c, "market.bill") == 0
|
||
assert published == []
|
||
assert [r.url.path for r in seen] == ["/v3/bill/118"]
|
||
|
||
|
||
def test_committees_cli_refusal_prints_gap(congress):
|
||
c, _, published, _, seen, state = congress
|
||
state["status"] = 403
|
||
res = runner.invoke(
|
||
app, ["market", "committees", "--from", "119", "--to", "119", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: legislators:committees: legislators: HTTP 403" in res.output
|
||
assert CONGRESS_SECRET not in res.output
|
||
assert _count(c, "market.committee_assignment") == 0
|
||
assert seen
|
||
assert published == [True]
|
||
|
||
|
||
COUNTY_REL = """\
|
||
GEOID_CD119_20|GEOID_COUNTY_20|AREALAND_COUNTY_20|AREALAND_PART
|
||
0101|01003|100|100
|
||
"""
|
||
HUD_SECRET = "hud-token-do-not-leak"
|
||
|
||
|
||
@pytest.fixture
|
||
def crosswalk(env, monkeypatch):
|
||
from market import districts as district_api
|
||
|
||
c, store, published, hoard = env
|
||
c.execute("CREATE SCHEMA IF NOT EXISTS reference_data")
|
||
c.execute(
|
||
"CREATE TABLE IF NOT EXISTS reference_data.ansi_fips_state ("
|
||
"ansi_fips_state_code VARCHAR, ansi_fips_state_abbreviation VARCHAR, "
|
||
"ansi_fips_state_name VARCHAR)"
|
||
)
|
||
c.execute(
|
||
"INSERT INTO reference_data.ansi_fips_state VALUES ('01', 'AL', 'Alabama')"
|
||
)
|
||
T.ensure_tables(c)
|
||
seen: list[httpx.Request] = []
|
||
state = {"status": 200}
|
||
zcta = (FX / "tab20_cd11920_zcta520_head.txt").read_bytes()
|
||
hud = (FX / "hud_zip_cd.json").read_bytes()
|
||
monkeypatch.setattr(district_api, "census_pacer", lambda: _Pacer())
|
||
monkeypatch.setattr(district_api, "hud_pacer", lambda: _Pacer())
|
||
|
||
def handler(request: httpx.Request) -> httpx.Response:
|
||
seen.append(request)
|
||
if state["status"] != 200:
|
||
return httpx.Response(state["status"], text="no")
|
||
if request.url.host == "www2.census.gov":
|
||
if "county20" in request.url.path:
|
||
return httpx.Response(200, content=COUNTY_REL.encode())
|
||
return httpx.Response(200, content=zcta)
|
||
if request.url.host == "www.huduser.gov":
|
||
return httpx.Response(200, content=hud)
|
||
return httpx.Response(404, text="no")
|
||
|
||
def client(source, key=None):
|
||
headers = {"Authorization": f"Bearer {key}"} if key else {}
|
||
return httpx.Client(transport=httpx.MockTransport(handler), headers=headers)
|
||
|
||
monkeypatch.setattr(http, "client", client)
|
||
monkeypatch.setattr(
|
||
http,
|
||
"api_key",
|
||
lambda source, env=None: HUD_SECRET if source == "hud" else None,
|
||
)
|
||
return c, store, published, hoard, seen, state
|
||
|
||
|
||
def test_districts_cli_no_write_prints_counts(crosswalk):
|
||
c, _, published, _, seen, _ = crosswalk
|
||
res = runner.invoke(app, ["market", "districts", "--congress", "119"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "2 zip rows, 1 county rows" in res.output
|
||
assert "congress 119: 2 zip rows, 1 county rows" in res.output
|
||
assert "dry run: pass --write" in res.output
|
||
assert _count(c, "market.zip_district") == 0
|
||
assert published == []
|
||
assert seen
|
||
assert all(r.url.host == "www2.census.gov" for r in seen)
|
||
|
||
|
||
def test_districts_cli_census_offline_write(crosswalk):
|
||
c, store, published, hoard, seen, _ = crosswalk
|
||
T.write_gap(c, "census:districts", "old")
|
||
res = runner.invoke(
|
||
app,
|
||
["market", "districts", "--congress", "118", "--congress", "119", "--write"],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stored: 4 zip rows, 2 county rows" in res.output
|
||
assert _count(c, "market.zip_district") == 4
|
||
assert _count(c, "market.county_district") == 2
|
||
assert T.read_gaps(c) == []
|
||
assert published == [True]
|
||
urls = [i.url or "" for i in store.list_items()]
|
||
assert any(u.endswith("tab20_cd11820_zcta520_natl.txt") for u in urls)
|
||
assert any(u.endswith("tab20_cd11920_county20_natl.txt") for u in urls)
|
||
assert len(urls) == 4
|
||
logged = c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||
assert ("market.zip_district", 4) in logged
|
||
assert ("market.county_district", 2) in logged
|
||
manifest = (hoard / "MANIFEST.jsonl").read_text()
|
||
assert "api_key" not in manifest
|
||
assert all("api_key" not in str(r.url) for r in seen)
|
||
|
||
|
||
def test_districts_cli_refusal_prints_gap(crosswalk):
|
||
c, _, published, _, seen, state = crosswalk
|
||
state["status"] = 403
|
||
res = runner.invoke(app, ["market", "districts", "--congress", "119", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: census:districts: census: HTTP 403" in res.output
|
||
assert _count(c, "market.zip_district") == 0
|
||
assert len(seen) == 1
|
||
assert published == [True]
|
||
|
||
|
||
def test_hud_cli_missing_token_writes_only_gap(crosswalk, monkeypatch):
|
||
c, _, published, _, seen, _ = crosswalk
|
||
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
|
||
kept = T.ZipDistrictRow(
|
||
119, "36010", "AL-01", 1.0, "census-zcta-land", "2020", "census"
|
||
)
|
||
T.write_zip_districts(c, [kept])
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"hud",
|
||
"--year",
|
||
"2025",
|
||
"--quarter",
|
||
"2",
|
||
"--congress",
|
||
"119",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: hud:hud:" in res.output
|
||
assert "HUD_API_TOKEN" in res.output
|
||
assert seen == []
|
||
assert [g.source for g in T.read_gaps(c)] == ["hud:hud"]
|
||
assert [(r.zip, r.method) for r in T.read_zip_districts(c)] == [
|
||
("36010", "census-zcta-land")
|
||
]
|
||
assert published == [True]
|
||
|
||
|
||
def test_hud_cli_offline_write(crosswalk):
|
||
c, _, published, hoard, seen, _ = crosswalk
|
||
kept = T.ZipDistrictRow(
|
||
119, "36010", "AL-01", 1.0, "census-zcta-land", "2020", "census"
|
||
)
|
||
T.write_zip_districts(c, [kept])
|
||
T.write_gap(c, "hud:hud", "old")
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"hud",
|
||
"--year",
|
||
"2025",
|
||
"--quarter",
|
||
"2",
|
||
"--congress",
|
||
"119",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stored: 2 zip rows" in res.output
|
||
methods = {(r.zip, r.method) for r in T.read_zip_districts(c)}
|
||
assert ("36010", "census-zcta-land") in methods
|
||
assert ("36009", "hud-res") in methods
|
||
assert ("36009", "hud-bus") in methods
|
||
assert T.read_gaps(c) == []
|
||
assert published == [True]
|
||
assert seen[0].headers["authorization"] == f"Bearer {HUD_SECRET}"
|
||
assert HUD_SECRET not in str(seen[0].url)
|
||
assert HUD_SECRET not in (hoard / "MANIFEST.jsonl").read_text()
|
||
assert seen[0].url.params["type"] == "5"
|
||
assert seen[0].url.params["query"] == "All"
|
||
|
||
|
||
def test_districts_cli_defaults_to_118_and_119(crosswalk):
|
||
res = runner.invoke(app, ["market", "districts"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "4 zip rows, 2 county rows" in res.output
|
||
assert "congress 118: 2 zip rows, 1 county rows" in res.output
|
||
assert "congress 119: 2 zip rows, 1 county rows" in res.output
|
||
assert "dry run: pass --write" in res.output
|
||
|
||
|
||
def test_hud_cli_no_write_prints_counts(crosswalk):
|
||
c, _, published, _, seen, _ = crosswalk
|
||
res = runner.invoke(
|
||
app, ["market", "hud", "--year", "2025", "--quarter", "2", "--congress", "119"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "2 zip rows" in res.output
|
||
assert "dry run: pass --write" in res.output
|
||
assert "stored:" not in res.output
|
||
assert _count(c, "market.zip_district") == 0
|
||
assert published == []
|
||
assert seen[0].headers["authorization"] == f"Bearer {HUD_SECRET}"
|
||
|
||
|
||
def test_hud_cli_refusal_prints_gap(crosswalk):
|
||
c, _, published, _, seen, state = crosswalk
|
||
state["status"] = 403
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"hud",
|
||
"--year",
|
||
"2025",
|
||
"--quarter",
|
||
"2",
|
||
"--congress",
|
||
"119",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: hud:hud: hud: HTTP 403" in res.output
|
||
assert HUD_SECRET not in res.output
|
||
assert _count(c, "market.zip_district") == 0
|
||
assert len(seen) == 1
|
||
assert published == [True]
|
||
|
||
|
||
def test_hud_cli_write_from_cache(crosswalk):
|
||
c, _, published, hoard, seen, _ = crosswalk
|
||
dest = hoard / "hud" / "zip-cd-2025-q2.json"
|
||
dest.parent.mkdir(parents=True)
|
||
dest.write_bytes((FX / "hud_zip_cd.json").read_bytes())
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"hud",
|
||
"--year",
|
||
"2025",
|
||
"--quarter",
|
||
"2",
|
||
"--congress",
|
||
"119",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stored: 2 zip rows" in res.output
|
||
assert seen == []
|
||
assert published == [True]
|
||
assert {r.method for r in T.read_zip_districts(c)} == {"hud-res", "hud-bus"}
|
||
|
||
|
||
def test_footprint_cli_missing_crosswalk_says_run_districts(env):
|
||
(
|
||
_,
|
||
_,
|
||
published,
|
||
_,
|
||
) = env
|
||
res = runner.invoke(
|
||
app, ["market", "footprint", "--family", "HOME-VISITS", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stack market districts" in res.output
|
||
assert published == []
|
||
|
||
|
||
def test_footprint_cli_entities_no_write(env, monkeypatch):
|
||
from market import footprint as footprint_api
|
||
|
||
c, _, published, _ = env
|
||
c.execute("CREATE SCHEMA IF NOT EXISTS cms")
|
||
c.execute(
|
||
"CREATE TABLE cms.home_health_agency_enrollments ("
|
||
"enrollment_id VARCHAR, organization_name VARCHAR, zip_code VARCHAR, "
|
||
"state VARCHAR)"
|
||
)
|
||
c.execute(
|
||
"INSERT INTO cms.home_health_agency_enrollments VALUES "
|
||
"('E1', 'Amedisys Home Health', '36009', 'AL'), "
|
||
"('E2', 'Amedisys Hospice', '48201', 'MI')"
|
||
)
|
||
T.write_entities(
|
||
c,
|
||
[T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")],
|
||
)
|
||
T.write_zip_districts(
|
||
c,
|
||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||
)
|
||
monkeypatch.setattr(
|
||
footprint_api,
|
||
"owner_files",
|
||
lambda: [FX / "hha_all_owners_head.csv"],
|
||
)
|
||
monkeypatch.setattr(footprint_api, "org_provider_rows", lambda con, entities: [])
|
||
res = runner.invoke(app, ["market", "footprint", "--entities", "--congress", "119"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "2 sites" in res.output
|
||
assert "dry run: pass --write" in res.output
|
||
assert _count(c, "market.entity_site") == 0
|
||
assert published == []
|
||
|
||
|
||
def test_footprint_cli_family_offline_write(env, monkeypatch):
|
||
from market import footprint as footprint_api
|
||
|
||
c, _, published, _ = env
|
||
T.write_zip_districts(
|
||
c,
|
||
[
|
||
T.ZipDistrictRow(
|
||
119, "36009", "AL-01", 0.75, "census-zcta-land", "2020", "c"
|
||
),
|
||
T.ZipDistrictRow(
|
||
119, "36009", "AL-02", 0.25, "census-zcta-land", "2020", "c"
|
||
),
|
||
],
|
||
)
|
||
monkeypatch.setattr(
|
||
footprint_api, "prov_svc_file", lambda: FX / "prov_svc_head.csv"
|
||
)
|
||
monkeypatch.setattr(footprint_api, "family_codes", lambda con, key: ["99343"])
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"footprint",
|
||
"--family",
|
||
"HOME-VISITS",
|
||
"--congress",
|
||
"119",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stored:" in res.output
|
||
rows = T.read_footprints(c)
|
||
assert {r.district_id for r in rows} >= {"AL-01", "AL-02", "??"}
|
||
assert published == [True]
|
||
|
||
|
||
def test_district_view_cli_prints(env):
|
||
c, _, _, _ = env
|
||
T.write_members(
|
||
c,
|
||
[
|
||
T.MemberRow(
|
||
119,
|
||
"T000481",
|
||
"Tlaib, Rashida",
|
||
"Democratic",
|
||
"MI",
|
||
"MI-12",
|
||
"house",
|
||
"",
|
||
"congress",
|
||
)
|
||
],
|
||
)
|
||
T.write_assignments(
|
||
c,
|
||
[
|
||
T.AssignmentRow(
|
||
119, date(2025, 4, 1), "HSWM02", "T000481", 1, "", "D", "sha"
|
||
)
|
||
],
|
||
)
|
||
T.write_footprints(
|
||
c,
|
||
[
|
||
T.FootprintRow(
|
||
"family",
|
||
"HOME-VISITS",
|
||
119,
|
||
"MI-12",
|
||
2024,
|
||
1,
|
||
0,
|
||
12,
|
||
75,
|
||
"census-zcta-land",
|
||
"prov-svc",
|
||
)
|
||
],
|
||
)
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"district-view",
|
||
"--family",
|
||
"HOME-VISITS",
|
||
"--congress",
|
||
"119",
|
||
"--top",
|
||
"10",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "MI-12" in res.output
|
||
assert "Tlaib, Rashida" in res.output
|
||
assert "HSWM02" in res.output
|
||
|
||
|
||
def test_footprint_cli_rejects_both_switches(env):
|
||
res = runner.invoke(
|
||
app, ["market", "footprint", "--entities", "--family", "HOME-VISITS"]
|
||
)
|
||
assert res.exit_code == 2
|
||
assert "pass exactly one of --entities or --family" in res.output
|
||
|
||
|
||
def test_footprint_cli_family_without_codes_writes_nothing(env, monkeypatch):
|
||
from market import footprint as footprint_api
|
||
|
||
c, _, published, _ = env
|
||
T.write_zip_districts(
|
||
c,
|
||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||
)
|
||
monkeypatch.setattr(footprint_api, "family_codes", lambda con, key: [])
|
||
res = runner.invoke(
|
||
app, ["market", "footprint", "--family", "HOME-VISITS", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "no codes for family HOME-VISITS" in res.output
|
||
assert published == []
|
||
|
||
|
||
def test_footprint_cli_entities_write(env, monkeypatch):
|
||
from market import footprint as footprint_api
|
||
|
||
c, _, published, _ = env
|
||
c.execute("CREATE SCHEMA IF NOT EXISTS cms")
|
||
c.execute(
|
||
"CREATE TABLE cms.home_health_agency_enrollments ("
|
||
"enrollment_id VARCHAR, organization_name VARCHAR, zip_code VARCHAR, "
|
||
"state VARCHAR)"
|
||
)
|
||
c.execute(
|
||
"INSERT INTO cms.home_health_agency_enrollments VALUES "
|
||
"('E1', 'Amedisys Home Health', '36009', 'AL'), "
|
||
"('E2', 'Amedisys Hospice', '48201', 'MI')"
|
||
)
|
||
T.write_entities(
|
||
c,
|
||
[T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")],
|
||
)
|
||
T.write_zip_districts(
|
||
c,
|
||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||
)
|
||
monkeypatch.setattr(
|
||
footprint_api, "owner_files", lambda: [FX / "hha_all_owners_head.csv"]
|
||
)
|
||
monkeypatch.setattr(footprint_api, "org_provider_rows", lambda con, entities: [])
|
||
res = runner.invoke(
|
||
app, ["market", "footprint", "--entities", "--congress", "119", "--write"]
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "stored:" in res.output
|
||
assert _count(c, "market.entity_site") == 2
|
||
assert {r.district_id for r in T.read_footprints(c)} >= {"AL-01", "??"}
|
||
assert published == [True]
|
||
|
||
|
||
def test_footprint_cli_counts_an_npi_site(env, monkeypatch):
|
||
from market import footprint as footprint_api
|
||
|
||
c, _, published, _ = env
|
||
T.write_entities(
|
||
c,
|
||
[T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")],
|
||
)
|
||
T.write_zip_districts(
|
||
c,
|
||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||
)
|
||
monkeypatch.setattr(footprint_api, "owner_files", lambda: [])
|
||
monkeypatch.setattr(
|
||
footprint_api,
|
||
"org_provider_rows",
|
||
lambda con, entities: [
|
||
{
|
||
"npi": "1234567890",
|
||
"entity_type_code": "2",
|
||
"provider_organization_name": "Clinic",
|
||
"parent_organization_name": "AMEDISYS HOLDING LLC",
|
||
"practice_zip_code": "36009",
|
||
"practice_state": "AL",
|
||
}
|
||
],
|
||
)
|
||
res = runner.invoke(app, ["market", "footprint", "--entities", "--congress", "119"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "1 sites" in res.output
|
||
assert published == []
|
||
|
||
|
||
def test_footprint_cli_empty_family_reports_zero_unmatched(env, monkeypatch):
|
||
from market import footprint as footprint_api
|
||
|
||
c, _, published, _ = env
|
||
T.write_zip_districts(
|
||
c,
|
||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||
)
|
||
monkeypatch.setattr(footprint_api, "family_codes", lambda con, key: ["99343"])
|
||
monkeypatch.setattr(footprint_api, "family_billers", lambda con, path, codes: [])
|
||
res = runner.invoke(app, ["market", "footprint", "--family", "HOME-VISITS"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "0 billers" in res.output
|
||
assert "unmatched ??: 0" in res.output
|
||
assert published == []
|
||
|
||
|
||
def test_district_view_cli_empty(env):
|
||
c, _, _, _ = env
|
||
T.write_footprints(c, [])
|
||
res = runner.invoke(app, ["market", "district-view", "--family", "HOME-VISITS"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "no footprint stored" in res.output
|
||
|
||
|
||
def _events_build():
|
||
from market.events import MarketEvent, Tenure
|
||
|
||
event = MarketEvent(
|
||
"rule-final:2022-23873",
|
||
"rule-final",
|
||
date(2022, 11, 2),
|
||
True,
|
||
"day",
|
||
"program",
|
||
"PFS",
|
||
"PETM225D",
|
||
0,
|
||
"2022-23873",
|
||
"2022-11-02T16:15:00.000-04:00",
|
||
"",
|
||
)
|
||
family = MarketEvent(
|
||
"rule-final:2022-23873",
|
||
"rule-final",
|
||
date(2022, 11, 2),
|
||
True,
|
||
"day",
|
||
"family",
|
||
"HOME-VISITS",
|
||
"PETM225D",
|
||
0,
|
||
"2022-23873",
|
||
"2022-11-02T16:15:00.000-04:00",
|
||
"",
|
||
)
|
||
tenure = Tenure(
|
||
"RUC",
|
||
"FAMILY PRACTICE",
|
||
("American Academy of Family Physicians",),
|
||
("permanent",),
|
||
2019,
|
||
2026,
|
||
True,
|
||
True,
|
||
)
|
||
return [event, family], 4, [tenure]
|
||
|
||
|
||
def test_events_cli_offline_write(env, monkeypatch):
|
||
from market import events as events_api
|
||
|
||
c, _, published, _ = env
|
||
monkeypatch.setattr(events_api, "build", lambda *a, **k: _events_build())
|
||
res = runner.invoke(app, ["market", "events", "--write"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "skipped rule: 4" in res.output
|
||
assert "stored:" in res.output
|
||
assert "FAMILY PRACTICE" in res.output
|
||
assert _count(c, "market.event") == 2
|
||
assert published == [True]
|
||
|
||
|
||
def test_events_cli_no_write_prints_counts(env, monkeypatch):
|
||
from market import events as events_api
|
||
|
||
c, _, published, _ = env
|
||
monkeypatch.setattr(events_api, "build", lambda *a, **k: _events_build())
|
||
res = runner.invoke(app, ["market", "events"])
|
||
assert res.exit_code == 0, res.output
|
||
assert "skipped rule: 4" in res.output
|
||
assert "2 events" in res.output
|
||
assert "HOME-VISITS" in res.output
|
||
assert "dry run: pass --write" in res.output
|
||
assert published == []
|
||
|
||
|
||
def test_events_cli_ama_refusal_gap(env, monkeypatch):
|
||
from market import events as events_api
|
||
|
||
c, _, published, _ = env
|
||
monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, []))
|
||
monkeypatch.setattr(
|
||
events_api,
|
||
"fetch_ama",
|
||
lambda *a, **k: (_ for _ in ()).throw(http.Refused("ama", 403, "denied")),
|
||
)
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"events",
|
||
"--ama-url",
|
||
"https://www.ama-assn.org/ruc-meetings",
|
||
"--body",
|
||
"RUC",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: ama:events:" in res.output
|
||
gaps = T.read_gaps(c)
|
||
assert any(g.source == "ama:events" for g in gaps)
|
||
assert published == [True]
|
||
|
||
|
||
def test_events_cli_ama_schedule_write(env, monkeypatch):
|
||
from market import events as events_api
|
||
|
||
c, _, published, _ = env
|
||
monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, []))
|
||
monkeypatch.setattr(
|
||
events_api,
|
||
"fetch_ama",
|
||
lambda *a, **k: "The RUC meets April 24–27, 2024.",
|
||
)
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"events",
|
||
"--ama-url",
|
||
"https://www.ama-assn.org/ruc-meetings",
|
||
"--body",
|
||
"RUC",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
rows = T.read_events(c)
|
||
assert any(r.event_id == "ruc-meeting:2024-04-24" for r in rows)
|
||
assert published == [True]
|
||
|
||
|
||
def test_events_cli_ama_calendar_stores_both_bodies(env, monkeypatch):
|
||
from market import events as events_api
|
||
|
||
c, _, published, _ = env
|
||
page = (
|
||
"CPT Editorial Panel meetings\nMay 6-8, 2027\n"
|
||
"AMA/Specialty Society RVS Update Committee meetings\n"
|
||
"Jan. 20-23, 2027\n"
|
||
)
|
||
monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, []))
|
||
monkeypatch.setattr(events_api, "fetch_ama", lambda *a, **k: page)
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"events",
|
||
"--ama-url",
|
||
"https://www.ama-assn.org/about/cpt-editorial-panel/cpt-editorial-panel-ruc-meetings-calendar",
|
||
"--body",
|
||
"RUC",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
ids = {row.event_id for row in T.read_events(c)}
|
||
assert ids == {"ruc-meeting:2027-01-20", "panel-meeting:2027-05-06"}
|
||
assert published == [True]
|
||
|
||
|
||
def test_events_cli_ama_needs_a_body(env):
|
||
res = runner.invoke(
|
||
app, ["market", "events", "--ama-url", "https://example.test/ruc"]
|
||
)
|
||
assert res.exit_code == 2
|
||
assert "pass --body RUC or CPT_PANEL" in res.output
|
||
|
||
|
||
def test_events_cli_ama_page_is_a_refusal(env, monkeypatch):
|
||
from market import events as events_api
|
||
|
||
c, _, published, _ = env
|
||
monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, []))
|
||
monkeypatch.setattr(events_api, "fetch_ama", lambda *a, **k: "Just a moment")
|
||
res = runner.invoke(
|
||
app,
|
||
[
|
||
"market",
|
||
"events",
|
||
"--ama-url",
|
||
"https://example.test/ruc",
|
||
"--body",
|
||
"RUC",
|
||
"--write",
|
||
],
|
||
)
|
||
assert res.exit_code == 0, res.output
|
||
assert "gap: ama:events: AMA page is a refusal" in res.output
|
||
assert any(g.source == "ama:events" for g in T.read_gaps(c))
|
||
assert published == [True]
|
||
|
||
|
||
def test_district_view_cli_needs_a_key(env):
|
||
res = runner.invoke(app, ["market", "district-view"])
|
||
assert res.exit_code == 2
|
||
assert "pass exactly one of --entity or --family" in res.output
|
||
|
||
|
||
def test_committees_cli_from_after_to(congress):
|
||
res = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "113"])
|
||
assert res.exit_code == 2
|
||
assert "--from 119 is after --to 113" in res.output
|