Files
stack/tests/cli/test_market_cli.py
kert fe52e513f9
Some checks failed
CI / lint (push) Successful in 41s
CI / notebooks-smoke (push) Successful in 2m13s
Deploy / changes (push) Successful in 19s
Infra CI / notebooks (push) Successful in 1m0s
Infra CI / zotero (push) Successful in 22s
CI / test (push) Successful in 5m3s
Infra CI / docs (push) Successful in 20s
Infra CI / api (push) Successful in 1m21s
Infra CI / bcda (push) Successful in 1m56s
Infra CI / llm (push) Successful in 1m4s
Infra CI / mc (push) Successful in 30s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m58s
Deploy / bcda (push) Successful in 1m56s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m4s
Deploy / report (push) Successful in 14s
Comment Pull / comment-pull (push) Failing after 29s
Notebooks Integration / notebooks-integration (push) Failing after 12m58s
feat(market): event windows — rule releases at public inspection (program/family/specialty), RUC/Panel meetings, seat tenures
2026-11-16 20:59:34 -05:00

3247 lines
113 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""stack market — entities, gaps."""
from __future__ import annotations
import json
import re
from datetime import date
from pathlib import Path
import duckdb
import httpx
import pytest
from typer.testing import CliRunner
import cli.market as mk
from bib.store import Store
from cli import app
from market import edgar, http
from market import tables as T
from market.seed import SEAT_LABEL_KEYS, load_seed
from pfs.codetables import SeatRow, write_seats
runner = CliRunner()
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
def _plain(text: str) -> str:
"""Strip ANSI. Rich highlights ``--option`` inside an error."""
return _ANSI_RE.sub("", text)
FX = Path(__file__).parent.parent / "market" / "fixtures"
class _Pacer:
def wait(self):
pass
def _seat(org, specialty, kind="specialty", year=2024):
return SeatRow("RUC", year, org, specialty, "permanent", "u", "K", "s", kind)
def _specialties() -> set[str]:
companies, orgs = load_seed()
out = {o.specialty for o in orgs if o.specialty}
out |= {
link.target
for e in (*companies, *orgs)
for link in e.links
if link.scope == "specialty"
}
return out - set(SEAT_LABEL_KEYS)
def _build_db() -> duckdb.DuckDBPyConnection:
c = duckdb.connect(":memory:")
c.execute("CREATE SCHEMA terminology")
c.execute(
"CREATE TABLE terminology.cms_acceptable_provider_specialty_codes "
"(specialty_description VARCHAR)"
)
c.executemany(
"INSERT INTO terminology.cms_acceptable_provider_specialty_codes VALUES (?)",
[(s.title(),) for s in sorted(_specialties())],
)
c.execute("CREATE SCHEMA pfs")
c.execute("CREATE TABLE pfs.cpt_code (code VARCHAR, descriptor VARCHAR)")
c.execute(
"INSERT INTO pfs.cpt_code VALUES ('99999', 'a licensed descriptor that no "
"filing quote should ever copy word for word')"
)
write_seats(
c,
"RUC",
[
_seat("Family Medicine", "FAMILY PRACTICE"),
_seat("Cardiology", "CARDIOLOGY"),
_seat("RUC Chair", "", "category"),
],
)
return c
class _Batch:
def __init__(self, c):
self.c = c
def __enter__(self):
return self.c
def __exit__(self, *a):
return False
@pytest.fixture
def env(monkeypatch, tmp_path):
c = _build_db()
monkeypatch.setattr(mk, "_batch", lambda: _Batch(c))
monkeypatch.setattr(mk, "_read", lambda: c.cursor())
store = Store(":memory:", storage_dir=tmp_path / "st")
monkeypatch.setattr(mk, "_store", lambda: store)
published: list[bool] = []
monkeypatch.setattr(mk, "_publish", lambda: published.append(True))
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
monkeypatch.setattr(edgar, "new_pacer", lambda: _Pacer())
try:
yield c, store, published, tmp_path / "market"
finally:
store.close()
c.close()
def _cache_offline(hoard: Path) -> None:
sec = hoard / "sec"
(sec / "submissions").mkdir(parents=True)
(sec / "company_tickers.json").write_bytes(
(FX / "company_tickers.json").read_bytes()
)
for name in (
"submissions_CIK0001564406.json",
"CIK0001564406-submissions-001.json",
):
target = name.replace("submissions_", "")
if name.startswith("submissions_"):
target = "CIK0001564406.json"
(sec / "submissions" / target).write_bytes((FX / name).read_bytes())
_cache_filings(hoard)
def _cache_filings(hoard: Path, companies=None) -> None:
"""A hoarded filing per cited accession, holding exactly the quotes the
seed cites to it."""
import html
companies = companies or load_seed()[0]
quotes: dict[tuple[str, str], list[str]] = {}
for _who, c, cite in edgar.filing_cites(companies):
if cite.adsh:
quotes.setdefault((c.cik, cite.adsh), []).append(cite.quote)
for (cik, adsh), qs in quotes.items():
p = hoard / "sec" / "filings" / cik / f"{adsh}.htm"
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(
"<html><body>"
+ "".join(f"<p>{html.escape(q)}</p>" for q in qs)
+ "</body></html>"
)
def _count(c, table):
return c.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
def test_entities_offline_writes_and_logs(env):
c, store, published, hoard = env
_cache_offline(hoard)
# a stale curated link the seed no longer carries
T.write_links(
c, [T.LinkRow("0001564406", "program", "QPP-MIPS", "curated", "x", "url", "u")]
)
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 0, res.output
companies, orgs = load_seed()
assert _count(c, "market.entity") == len(companies) + len(orgs)
osh = next(e for e in T.read_entities(c, "company") if e.key == "0001564406")
assert osh.status == "acquired" and osh.ended_on == date(2023, 5, 2)
assert osh.sic == "8000" and osh.item_key
assert "no submissions for AMED" in res.output
# the five fixture filings, and the seed's dated ticker
assert len(T.read_filings(c, "0001564406")) == 5
assert [s.source for s in T.read_securities(c, "0001564406")] == ["seed"]
links = T.read_links(c, basis="curated")
assert all(l.item_key for l in links)
assert ("program", "QPP-MIPS") not in {(l.scope, l.target) for l in links}
assert [l.target for l in T.read_links(c, basis="sic")] == ["8000"]
logs = c.execute(
"SELECT table_name, rows FROM cms.ingest_log WHERE module = 'market.edgar' "
"ORDER BY table_name"
).fetchall()
assert [t for t, _ in logs] == [
"market.entity",
"market.entity_link",
"market.filing",
"market.security",
]
assert published == [True]
assert "retired 1" in res.output
assert store.count() > 0
def test_entities_rerun_is_idempotent(env):
c, _, published, hoard = env
_cache_offline(hoard)
for _ in range(2):
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
companies, orgs = load_seed()
assert _count(c, "market.entity") == len(companies) + len(orgs)
assert _count(c, "market.filing") == 5
def test_entities_no_write_prints_counts(env, monkeypatch):
c, _, published, hoard = env
_cache_offline(hoard)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 0, res.output
companies, orgs = load_seed()
assert f"entities {len(companies) + len(orgs)}" in res.output
assert (
"company/acquired" in res.output
and "organization/specialty_society" in res.output
)
assert "dry run" in res.output and published == []
assert "warning" not in res.output # every seat specialty has a society
def test_entities_warns_about_uncovered_seat_specialty(env):
c, _, _, hoard = env
_cache_offline(hoard)
write_seats(c, "RUC", [_seat("Podiatry", "PODIATRY", year=2025)])
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 0
assert "seat specialties without a society: ['PODIATRY']" in res.output
def _handler(routes):
def handler(request: httpx.Request):
url = str(request.url)
for pattern, make in routes:
m = re.search(pattern, url)
if m:
return make(m)
return httpx.Response(404)
return handler
def _online(monkeypatch, routes):
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(_handler(routes))
),
)
def test_entities_refusal_prints_gap(env, monkeypatch):
c, _, published, hoard = env
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
res = runner.invoke(app, ["market", "entities", "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:entities:" in res.output and "403" in res.output
assert _count(c, "market.entity") == 0 and _count(c, "market.filing") == 0
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:entities"]
assert published == [True]
def test_entities_refusal_without_write_records_nothing(env, monkeypatch):
c, _, published, _ = env
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "entities"])
assert res.exit_code == 0 and "gap: sec-edgar:entities:" in res.output
assert published == []
def _sub_for(cik: str) -> dict:
return {
"cik": cik,
"name": f"Filer {cik}",
"sic": "8082",
"sicDescription": "Home Health",
"tickers": [],
"formerNames": [],
"filings": {
"recent": {
"accessionNumber": ["A-1", "A-2"],
"filingDate": ["2024-01-01", "2024-01-02"],
"reportDate": ["", ""],
"acceptanceDateTime": ["", ""],
"form": ["10-K", "4"],
"items": ["", ""],
"primaryDocument": ["a.htm", "b.xml"],
},
"files": [],
},
}
def test_sic_sweep_registers_filers_and_sic_links(env, monkeypatch):
c, _, published, hoard = env
_cache_filings(hoard)
atom = (FX / "sic_8082.atom").read_bytes().replace(b"0001468328", b"0000000042")
seen: list[str] = []
def sub(m):
seen.append(m.group(1))
cik = m.group(1)
if cik == "0001564406":
return httpx.Response(
200, content=(FX / "submissions_CIK0001564406.json").read_bytes()
)
if cik == "0000000042":
return httpx.Response(404) # no submissions file: skipped
return httpx.Response(200, json=_sub_for(cik))
def page(m):
return httpx.Response(
200, content=(FX / "CIK0001564406-submissions-001.json").read_bytes()
)
_online(
monkeypatch,
[
(
r"company_tickers\.json",
lambda m: httpx.Response(
200, content=(FX / "company_tickers.json").read_bytes()
),
),
(r"SIC=\d+.*start=0", lambda m: httpx.Response(200, content=atom)),
(r"submissions-001\.json", page),
(r"CIK(\d{10})\.json", sub),
],
)
res = runner.invoke(app, ["market", "entities", "--sic-sweep", "--write"])
assert res.exit_code == 0, res.output
assert res.output.count("SIC ") == len(edgar.HEALTH_SICS)
keys = {e.key for e in T.read_entities(c, "company")}
assert {"0001725255", "0001468328"} <= keys and "0000000042" not in keys
# a sweep filer keeps only the sweep forms; the seed's filings keep everything
assert [f.form for f in T.read_filings(c, "0001725255")] == ["10-K"]
assert len(T.read_filings(c, "0001468328")) == 2
assert len(T.read_filings(c, "0001564406")) == 5
sic = T.read_links(c, basis="sic")
assert {l.target for l in sic} == {"8082", "8000"}
assert any(l.entity_key == "0001725255" for l in sic)
def test_entities_seed_errors_abort(env, monkeypatch):
c, _, published, _ = env
companies, orgs = load_seed()
bad = companies[0].__class__(
**{
**companies[0].__dict__,
"links": (
companies[0].links[0].__class__("family", "NOPE", companies[0].cite),
),
}
)
monkeypatch.setattr(mk, "load_seed", lambda: ((bad, *companies[1:]), orgs))
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 1 and "seed error: OSH" in res.output
assert c.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
).fetchone() == (0,)
assert published == []
def test_entities_missing_vocabulary_tables_abort(env, monkeypatch):
bare = duckdb.connect(":memory:")
monkeypatch.setattr(mk, "_read", lambda: bare)
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 1 and "cannot validate the seed" in res.output
bare.close()
def test_entities_other_read_errors_propagate(env, monkeypatch):
def boom():
raise RuntimeError("disk on fire")
monkeypatch.setattr(mk, "_read", boom)
res = runner.invoke(app, ["market", "entities", "--offline"])
assert isinstance(res.exception, RuntimeError)
def test_require_key_missing_writes_only_gap(env, monkeypatch):
c, _, published, _ = env
monkeypatch.delenv("TIINGO_API_KEY", raising=False)
assert mk._require_key("tiingo", "tiingo:prices", write=True) is None
assert [(g.source, g.note) for g in T.read_gaps(c)] == [
("tiingo:prices", "TIINGO_API_KEY not in .env")
]
assert all(_count(c, f"market.{t}") == 0 for t in ("entity", "security", "filing"))
assert published == [True]
def test_require_key_present_returns_it(env, monkeypatch):
monkeypatch.setenv("TIINGO_API_KEY", "abc")
assert mk._require_key("tiingo", "tiingo:prices", write=True) == "abc"
def test_require_key_missing_prints_gap_without_write(env, monkeypatch, capsys):
monkeypatch.delenv("HUD_API_TOKEN", raising=False)
assert mk._require_key("hud", "hud:hud", write=False) is None
assert "gap: hud:hud: HUD_API_TOKEN not in .env" in capsys.readouterr().out
c, _, published, _ = env
assert published == []
def test_gap_cleared_after_success(env):
c, _, _, hoard = env
_cache_offline(hoard)
T.write_gap(c, "sec-edgar:entities", "403")
T.write_gap(c, "tiingo:prices", "no key")
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
assert {g.source for g in T.read_gaps(c)} == {
"tiingo:prices",
"web:renalmd.org",
"web:asco.org",
"web:ahip.org",
}
def test_gaps_command_lists_rows(env):
c, _, _, _ = env
T.write_gap(c, "tiingo:prices", "TIINGO_API_KEY not in .env")
res = runner.invoke(app, ["market", "gaps"])
assert res.exit_code == 0
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
def test_gaps_command_empty_and_missing_tables(env, monkeypatch):
c, _, _, _ = env
T.ensure_tables(c)
assert "no gaps recorded" in runner.invoke(app, ["market", "gaps"]).output
bare = duckdb.connect(":memory:")
monkeypatch.setattr(mk, "_read", lambda: bare)
assert "no market tables yet" in runner.invoke(app, ["market", "gaps"]).output
class _Broken:
def execute(self, *a, **k):
raise RuntimeError("x")
def close(self):
pass
monkeypatch.setattr(mk, "_read", lambda: _Broken())
assert isinstance(runner.invoke(app, ["market", "gaps"]).exception, RuntimeError)
bare.close()
def test_default_indirections_resolve_lazily(monkeypatch):
import conf.connect as cc
calls: list[str] = []
monkeypatch.setattr(cc, "duckdb_batch", lambda n: calls.append(f"batch {n}") or n)
monkeypatch.setattr(
cc, "duckdb", lambda n, read_only=True: calls.append(f"read {n}") or n
)
monkeypatch.setattr(cc, "bib", lambda: calls.append("bib") or "store")
monkeypatch.setattr(
cc, "publish_replica", lambda n: calls.append(f"pub {n}") or "p"
)
assert mk._batch() == "aco" and mk._read() == "aco" and mk._store() == "store"
mk._publish()
assert calls == ["batch aco", "read aco", "bib", "pub aco"]
def test_entities_aborts_when_a_quote_is_not_in_the_filing(env):
c, _, published, hoard = env
_cache_offline(hoard)
victim = next(hoard.glob("sec/filings/*/*.htm"))
victim.write_text("<p>nothing the seed quotes</p>")
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 1 and "quote error:" in res.output
assert published == []
assert c.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
).fetchone() == (0,)
def test_entities_aborts_when_a_cited_filing_is_not_hoarded(env):
c, _, published, hoard = env
_cache_offline(hoard)
next(hoard.glob("sec/filings/*/*.htm")).unlink()
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 1 and "not hoarded" in res.output
def test_entities_fetches_cited_filings_online(env, monkeypatch):
c, _, _, hoard = env
_cache_offline(hoard)
victim = next(hoard.glob("sec/filings/*/*.htm"))
cik, adsh = victim.parent.name, victim.stem
body = victim.read_bytes()
victim.unlink()
(hoard / "sec" / "submissions" / f"CIK{cik}.json").write_text(
json.dumps(
_sub_for(cik)
| {
"filings": {
"recent": {
**_sub_for(cik)["filings"]["recent"],
"accessionNumber": [adsh, "Z"],
"primaryDocument": ["doc.htm", "z.htm"],
},
"files": [],
}
}
)
)
seen = []
def serve(m):
seen.append(m.group(0))
return httpx.Response(200, content=body)
_online(monkeypatch, [(r"/Archives/edgar/data/\d+/\d+/doc\.htm", serve)])
res = runner.invoke(app, ["market", "entities"])
assert res.exit_code == 0, res.output
assert len(seen) == 1 and victim.exists()
def test_entities_manifest_and_ingest_log_sources(env):
c, _, _, hoard = env
_cache_offline(hoard)
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
manifest = hoard / "MANIFEST.jsonl"
assert manifest.exists() and manifest.read_text().count("\n") >= 6
rows = c.execute(
"SELECT table_name, source_file, sha256 FROM cms.ingest_log WHERE module = 'market.edgar'"
).fetchall()
assert len(rows) == 4 and all(sf and sha and len(sha) == 64 for _, sf, sha in rows)
def test_entities_records_unreachable_site_gaps_and_pseudo_orgs(env):
c, _, _, hoard = env
_cache_offline(hoard)
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
gaps = {g.source for g in T.read_gaps(c)}
assert {"web:renalmd.org", "web:asco.org", "web:ahip.org"} <= gaps
assert "sec-edgar:entities" not in gaps
pseudo = {e.key for e in T.read_entities(c, "organization") if e.pseudo}
assert pseudo == {"PRIMARY-CARE-SEAT", "ONCOLOGY-SEAT"}
def test_entities_retires_organizations_the_seed_dropped(env):
c, _, _, hoard = env
_cache_offline(hoard)
T.write_entities(c, [T.EntityRow("ASCO", "ASCO", "organization")])
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
assert "ASCO" not in {e.key for e in T.read_entities(c, "organization")}
def test_entities_aborts_when_a_quote_copies_a_cpt_descriptor(env):
c, _, published, hoard = env
_cache_offline(hoard)
victim = load_seed()[0][0].cite.quote
c.execute("DELETE FROM pfs.cpt_code")
c.execute("INSERT INTO pfs.cpt_code VALUES ('1', ?)", [victim])
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 1 and "ama error: OSH" in res.output
assert published == []
assert c.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
).fetchone() == (0,)
def test_entities_stores_ticker_history_and_link_dates(env):
c, _, _, hoard = env
_cache_offline(hoard)
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
assert [
(s.ticker, s.valid_to, s.priced) for s in T.read_securities(c, "0001156039")
] == [("ANTM", date(2022, 6, 27), False), ("ELV", None, True)]
assert [s.priced for s in T.read_securities(c, "0001739940")] == [False]
(clov,) = [
l
for l in T.read_links(c, basis="curated", target="APM")
if l.entity_key == "0001801170"
]
assert clov.valid_to == date(2023, 12, 31)
# ── stack market prices ──────────────────────────────────────────────
import io # noqa: E402
import zipfile # noqa: E402
from market import tiingo # noqa: E402
DAY = date(2026, 11, 12)
def _supported_zip(*rows: str) -> bytes:
head = "ticker,exchange,assetType,priceCurrency,startDate,endDate\n"
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as z:
z.writestr("supported_tickers.csv", head + "\n".join(rows) + "\n")
return buf.getvalue()
def _eod(day: str, close: float, adj: float | None = None) -> dict:
return {
"date": f"{day}T00:00:00.000Z",
"open": close,
"high": close,
"low": close,
"close": close,
"volume": 5,
"adjClose": close if adj is None else adj,
"divCash": 0.0,
"splitFactor": 1.0,
}
@pytest.fixture
def px(env, monkeypatch, tmp_path):
c, store, published, hoard = env
monkeypatch.setenv("TIINGO_API_KEY", "secret-key")
monkeypatch.setattr(tiingo, "state_dir", lambda: tmp_path / "state")
monkeypatch.setattr(tiingo, "new_pacer", lambda: _Pacer())
monkeypatch.setattr(tiingo, "today", lambda: DAY)
seen: list[httpx.Request] = []
series = {
"AAA": [
_eod("2024-01-02", 100),
_eod("2024-01-03", 110),
_eod("2024-01-04", 99),
],
"BBB": [_eod("2024-01-02", 10), _eod("2024-01-03", 11)],
"SPY": [_eod("2024-01-02", 400), _eod("2024-01-03", 404)],
"XLV": [_eod("2024-01-02", 100)],
"IHF": [_eod("2024-01-02", 50)],
}
z = _supported_zip(
"AAA,NYSE,Stock,USD,2000-01-01,2026-11-12",
"BBB,NYSE,Stock,USD,2000-01-01,2026-11-12",
"SPY,NYSE,ETF,USD,1993-01-29,2026-11-12",
"XLV,NYSE,ETF,USD,1998-12-22,2026-11-12",
"IHF,NYSE,ETF,USD,2006-05-05,2026-11-12",
"ZZZ,NYSE,Stock,USD,2000-01-01,2026-11-12",
)
def prices_for(m):
if m.group(1) not in series:
return httpx.Response(404, json={"detail": "Not found."})
return httpx.Response(200, json=series[m.group(1)])
routes = [
(r"supported_tickers\.zip", lambda m: httpx.Response(200, content=z)),
(r"/daily/(\w+)/prices", prices_for),
]
def handler(request):
seen.append(request)
return _handler(routes)(request)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(handler),
headers={"Authorization": f"Token {key}"} if key else {},
),
)
return c, store, published, hoard, seen, series, tmp_path / "state"
def _price_requests(seen):
return [r for r in seen if "/prices" in r.url.path]
def test_prices_cli_fetch_write(px):
c, _, published, hoard, seen, _, state = px
res = runner.invoke(
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
)
assert res.exit_code == 0, res.output
assert _count(c, "market.price_daily") == 5
assert [r.url.params["startDate"] for r in _price_requests(seen)] == [
"1993-01-29"
] * 2
assert all("secret-key" not in str(r.url) for r in seen)
keyed = [r for r in seen if "/prices" in r.url.path]
assert all(r.headers["authorization"] == "Token secret-key" for r in keyed)
free = [r for r in seen if "supported" in r.url.path]
assert free and "authorization" not in free[0].headers
assert {
(t, n)
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
} == {("market.price_daily", 5), ("market.series", 2)}
assert [s.value for s in T.read_series(c, ["AAA"])] == [pytest.approx(99.0)]
assert published == [True]
used = json.loads((state / "tiingo-2026-11.json").read_text())
assert used["symbols"] == ["AAA", "BBB"] and used["day_counts"] == {"2026-11-12": 2}
assert (hoard / "tiingo" / "eod" / "AAA" / "1993-01-29_2026-11-12.json").is_file()
assert "priced 2 symbols, 5 rows" in res.output and "2/450" in res.output
assert T.read_gaps(c) == []
def test_prices_incremental_pull_and_same_day_skip(px, monkeypatch):
c, _, _, _, seen, series, _ = px
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
assert runner.invoke(app, cmd).exit_code == 0
before = len(_price_requests(seen))
again = runner.invoke(app, cmd)
assert len(_price_requests(seen)) == before # priced today: nothing to do
assert "priced 0 symbols" in again.output
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
series["AAA"] = [_eod("2024-01-04", 99, 49.5), _eod("2024-01-05", 108.9, 54.45)]
assert runner.invoke(app, cmd).exit_code == 0
last = _price_requests(seen)[-1]
assert last.url.params["startDate"] == "2024-01-04"
rows = {r.day: r for r in T.read_prices(c, "AAA")}
assert len(rows) == 4 and rows[date(2024, 1, 4)].adj_close == 99
assert rows[date(2024, 1, 5)].ret == pytest.approx(0.1)
assert [m.value for m in T.read_series(c, ["AAA"])] == [pytest.approx(108.9)]
def test_prices_benchmarks_start_at_listing(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(app, ["market", "prices", "--benchmarks", "--write"])
assert res.exit_code == 0, res.output
starts = {
r.url.path.split("/")[3]: r.url.params["startDate"]
for r in _price_requests(seen)
}
assert starts == {"SPY": "1993-01-29", "XLV": "1998-12-22", "IHF": "2006-05-05"}
def test_prices_dry_run_reports_budget(px, monkeypatch):
c, _, published, _, seen, _, state = px
T.ensure_tables(c)
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
T.write_links( # revenue_exposure passes the strength gate for any SIC
c, [T.LinkRow("K1", "program", "PFS", "fts", "revenue_exposure", "adsh", "a")]
)
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
assert res.exit_code == 0, res.output
assert "1/450 symbols used, 449 remaining" in res.output
assert "1. SPY" in res.output and "used" in res.output
assert "AAA" in res.output and "K1" in res.output and "first pull" in res.output
assert _price_requests(seen) == [] and published == []
assert not (state / "tiingo-2026-11.json").read_text().count("AAA")
def test_prices_dry_run_without_market_tables_lists_benchmarks(px):
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
assert res.exit_code == 0, res.output
assert "3 symbols ranked" in res.output
def test_prices_dry_run_shows_stored_span(px):
c, *_ = px
T.ensure_tables(c)
T.write_prices(
c,
"AAA",
tiingo.to_prices(
"AAA", [_eod("2024-01-02", 1)], pulled_on=DAY, source_file="f", overlap=None
),
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--dry-run"])
assert "stored to 2024-01-02" in res.output
def test_prices_without_key_records_gap(px, monkeypatch):
c, _, published, _, seen, _, _ = px
monkeypatch.delenv("TIINGO_API_KEY")
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"]
assert _count(c, "market.price_daily") == 0 and seen == [] and published == [True]
def test_prices_refusal_prints_gap(px, monkeypatch):
c, _, published, _, _, _, _ = px
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(lambda r: httpx.Response(403))
),
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "gap: tiingo:prices:" in res.output and "403" in res.output
assert _count(c, "market.price_daily") == 0
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"] and published == [
True
]
def test_prices_refusal_on_prices_after_supported_ok(px, monkeypatch):
c, _, published, _, _, _, _ = px
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
routes = [
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
(r"/prices", lambda m: httpx.Response(401)),
]
_online(monkeypatch, routes)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA"])
assert (
res.exit_code == 0
and "gap: tiingo:prices:" in res.output
and "401" in res.output
)
assert published == []
def test_prices_nothing_selected_exits(px):
res = runner.invoke(app, ["market", "prices"])
assert res.exit_code == 2 and "nothing to price" in res.output
def test_prices_unknown_ticker_and_window_gap_become_gap_note(px):
c, _, _, _, _, series, _ = px
T.ensure_tables(c)
T.write_securities(
c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, date(2030, 1, 1), "s")]
)
res = runner.invoke(
app, ["market", "prices", "--ticker", "BBB", "--ticker", "ZZZ", "--write"]
)
assert res.exit_code == 0, res.output
assert "ZZZ: not found at Tiingo" in res.output
(gap,) = T.read_gaps(c)
assert (
"ZZZ: not found at Tiingo" in gap.note
and "BBB: Tiingo ends 2026-11-12" in gap.note
)
# BBB later prices cleanly: its entry goes, ZZZ's stays
T.write_securities(c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, None, "s")])
runner.invoke(app, ["market", "prices", "--ticker", "BBB", "--refresh", "--write"])
(gap,) = T.read_gaps(c)
assert gap.note == "ZZZ: not found at Tiingo"
def test_prices_clean_run_clears_gap_and_window_clips(px):
c, _, _, _, _, _, _ = px
T.ensure_tables(c)
T.write_gap(c, "tiingo:prices", "AAA: not found at Tiingo")
T.write_securities(
c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", date(2024, 1, 3), None, "s")]
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert [r.day for r in T.read_prices(c, "AAA")] == [
date(2024, 1, 3),
date(2024, 1, 4),
]
assert T.read_gaps(c) == []
def test_prices_limit_stops_downloads(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(
app, ["market", "prices", "--benchmarks", "--limit", "1", "--write"]
)
assert res.exit_code == 0 and len(_price_requests(seen)) == 1
assert [r.ticker for r in [*T.read_prices(c, "SPY")][:1]] == ["SPY"]
def test_prices_budget_exhausted_stops_and_writes_what_it_has(px):
c, _, published, _, seen, _, state = px
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps(
{
"month": "2026-11",
"symbols": [f"S{i}" for i in range(449)] + ["AAA"],
"day_counts": {},
}
)
)
res = runner.invoke(
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
)
assert res.exit_code == 0, res.output
assert "budget: stopped, BBB" in res.output
assert [r.ticker for r in T.read_prices(c, "AAA")][:1] == ["AAA"]
assert T.read_prices(c, "BBB") == [] and published == [True]
def test_prices_without_write_stores_nothing(px, monkeypatch):
c, _, published, _, _, _, _ = px
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "prices", "--ticker", "ZZZ"])
assert res.exit_code == 0, res.output
assert (
"dry run: pass --write" in res.output
and "gap: tiingo:prices: ZZZ" in res.output
)
assert published == []
def test_merge_gap_keeps_other_tickers_and_orders():
assert mk._merge_gap("", {}, set()) == ""
assert mk._merge_gap("A: x | B: y", {"C": "z"}, {"A"}) == "B: y | C: z"
def test_or_empty_reraises_other_errors():
def boom(_):
raise RuntimeError("disk")
with pytest.raises(RuntimeError):
mk._or_empty(boom, None, {})
def test_prices_ticker_refuses_unpriced_record_without_a_request(px):
c, _, _, _, seen, _, state = px
T.ensure_tables(c)
T.write_securities(
c, "K9", [T.SecurityRow("K9", "AAA", "NYSE", None, None, "s", False)]
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "skipped AAA: symbol record only" in res.output
assert _price_requests(seen) == [] and _count(c, "market.price_daily") == 0
assert not (state / "tiingo-2026-11.json").exists()
assert T.read_gaps(c) == []
def test_prices_unsupported_named_ticker_is_gap_not_request(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(app, ["market", "prices", "--ticker", "QQQQ", "--write"])
assert "skipped QQQQ: not on Tiingo's supported list" in res.output
assert _price_requests(seen) == []
(gap,) = T.read_gaps(c)
assert gap.note == "QQQQ: not on Tiingo's supported list"
def test_prices_universe_records_unsupported_seed_ticker_gap(px):
c, _, _, _, seen, _, _ = px
T.ensure_tables(c)
T.write_securities(
c, "0001800682", [T.SecurityRow("0001800682", "CANO", "NYSE", None, None, "s")]
)
res = runner.invoke(app, ["market", "prices", "--universe", "--write"])
assert res.exit_code == 0, res.output
assert "skipped CANO: not on Tiingo's supported list" in res.output
assert all("/CANO/" not in r.url.path for r in seen)
(gap,) = T.read_gaps(c)
assert gap.source == "tiingo:prices"
assert gap.note == "CANO: not on Tiingo's supported list"
# rerun with everything already priced today keeps the gap
again = runner.invoke(app, ["market", "prices", "--universe", "--write"])
assert again.exit_code == 0
assert [g.note for g in T.read_gaps(c)] == [gap.note]
def test_prices_missing_overlap_day_repulls_further_back(px, monkeypatch):
c, _, _, _, seen, _, _ = px
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
assert runner.invoke(app, cmd).exit_code == 0
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
def eod(m):
# the 2024-01-04 overlap day is absent from the narrow pull
start = next(r for r in seen[::-1] if "/prices" in r.url.path).url.params[
"startDate"
]
if start == "2024-01-04":
return httpx.Response(200, json=[_eod("2024-01-05", 54.45)])
return httpx.Response(
200, json=[_eod("2024-01-03", 55), _eod("2024-01-05", 60.5)]
)
handler = _handler(
[
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
(r"/daily/AAA/prices", eod),
]
)
def record(request):
seen.append(request)
return handler(request)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(record)),
)
res = runner.invoke(app, cmd)
assert res.exit_code == 0, res.output
assert "overlap day 2024-01-04 missing" in res.output
assert "re-pulling from 2023-12-25" in res.output
row = {r.day: r for r in T.read_prices(c, "AAA")}[date(2024, 1, 5)]
assert row.ret == pytest.approx(60.5 / 55 - 1)
# ── fts ──────────────────────────────────────────────────────────────
MPFS = '"Medicare Physician Fee Schedule"'
SNIPPET = (FX / "filing_snippet.htm").read_bytes()
LICENSED = "a licensed descriptor that no filing quote should ever copy word for word"
def _hit_json(cik, adsh, filename, sics, date_="2023-03-01"):
return {
"_id": f"{adsh}:{filename}",
"_source": {
"ciks": [cik],
"adsh": adsh,
"form": "10-K",
"file_date": date_,
"period_ending": "2022-12-31",
"sics": sics,
"display_names": [f"Filer {cik} (CIK {cik})"],
},
}
HITS = [
_hit_json("0000012345", "0000012345-23-000001", "a10k.htm", ["8071"]),
_hit_json("0000000002", "0000000002-23-000001", "reit.htm", ["6798"]),
_hit_json("0000012346", "0000012346-23-000001", "ex99.pdf", ["8082"]),
_hit_json("0000012347", "0000012347-23-000001", "gone.htm", ["8082"]),
]
def _fts_net(
monkeypatch, hits=HITS, docs=None, forbid_docs=False, status=200, broken=None
):
"""A mock EDGAR: the term "Medicare Physician Fee Schedule" has *hits* in
2023; every other query is empty."""
docs = docs if docs is not None else {"a10k.htm": SNIPPET}
requested: list[str] = []
def efts(m):
p = dict(httpx.URL(m.group(0)).params)
mine = p["q"] == MPFS and p["from"] == "0"
return httpx.Response(
status,
json={
"hits": {
"total": {"value": len(hits) if mine else 0},
"hits": hits if mine else [],
}
},
)
def doc(m):
requested.append(m.group(1))
if broken and m.group(1) in broken:
if broken[m.group(1)] == "timeout":
raise httpx.ReadTimeout("slow")
return httpx.Response(broken[m.group(1)])
body = docs.get(m.group(1))
if forbid_docs and body:
raise AssertionError("no document download expected")
return httpx.Response(200, content=body) if body else httpx.Response(404)
def sub(m):
return httpx.Response(200, json={**_sub_for(m.group(1)), "tickers": ["ZZZ"]})
_online(
monkeypatch,
[
(r"efts\.sec\.gov/.*", efts),
(r"/Archives/edgar/data/\d+/\d+/([^/?]+)$", doc),
(
r"company_tickers\.json",
lambda m: httpx.Response(
200, content=(FX / "company_tickers.json").read_bytes()
),
),
(r"CIK(\d{10})\.json", sub),
],
)
return requested
FTS = ["market", "fts", "--program", "PFS", "--from", "2023", "--to", "2023"]
def test_fts_cli_dry_run_lists_queries(env, monkeypatch):
c, _, published, hoard = env
monkeypatch.setattr(
http, "client", lambda *a, **k: (_ for _ in ()).throw(AssertionError("net"))
)
res = runner.invoke(app, [*FTS, "--dry-run"])
assert res.exit_code == 0, res.output
assert "program PFS: 5 terms, 5 queries" in res.output
assert "5 queries (0 already hoarded)" in res.output
assert published == []
full = runner.invoke(app, ["market", "fts", "--dry-run"])
assert "program MSSP: 1 terms" in full.output
assert "program CARE-MANAGEMENT: 2 terms" in full.output
assert f"{(5 + 2 + 1 + 1 + 1 + 2 + 2) * (dt_year() - 2000)} queries" in full.output
def dt_year():
import datetime
return datetime.date.today().year
def test_fts_cli_family_scope_and_bad_keys(env, monkeypatch):
monkeypatch.setattr("pfs.families.refresh_from", lambda con: 0)
res = runner.invoke(
app,
[
"market",
"fts",
"--family",
"HOME-VISITS",
"--dry-run",
"--from",
"2023",
"--to",
"2023",
],
)
assert res.exit_code == 0, res.output
assert "family HOME-VISITS: 8 terms, 8 queries" in res.output
assert (
runner.invoke(app, ["market", "fts", "--family", "NOPE", "--dry-run"]).exit_code
== 2
)
assert (
runner.invoke(
app, ["market", "fts", "--program", "NOPE", "--dry-run"]
).exit_code
== 2
)
def test_fts_cli_without_write_stores_nothing(env, monkeypatch):
c, _, published, hoard = env
_fts_net(monkeypatch)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, FTS)
assert res.exit_code == 0, res.output
assert "Medicare Physician Fee Schedule 10-K 2023: 4 hits" in res.output
assert "passages kept 1," in res.output and "new entities 1" in res.output
assert "fts links 1 (PFS 1)" in res.output and "dry run" in res.output
assert (
"gap: sec-edgar:fts: 0000012346-23-000001:ex99.pdf: format not readable"
in res.output
)
assert (
"gap: sec-edgar:fts: 0000012347-23-000001:gone.htm: document not found"
in res.output
)
assert published == []
# hoarded: the pages and the one downloaded readable document
assert (
hoard / "sec/fts/medicare-physician-fee-schedule/10-K/2023.0.json"
).is_file()
assert (hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm").is_file()
def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch):
c, store, published, hoard = env
curated = T.LinkRow(
"0000012345", "program", "APM", "curated", "curated", "adsh", "x", "", "K"
)
T.write_links(c, [curated])
_fts_net(monkeypatch)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
(ent,) = T.read_entities(c, "company")
assert ent.key == "0000012345" and ent.sic == "8082"
assert [s.ticker for s in T.read_securities(c, ent.key)] == ["ZZZ"]
assert [f.form for f in T.read_filings(c, ent.key)] == ["10-K"]
assert [l.target for l in T.read_links(c, basis="sic")] == ["8082"]
assert T.read_links(c, basis="curated") == [curated] # survives
(fts_link,) = T.read_links(c, basis="fts")
assert (fts_link.scope, fts_link.target, fts_link.strength) == (
"program",
"PFS",
"mention",
)
assert fts_link.item_key and fts_link.passage_id.startswith(
"0000012345-23-000001:a10k.htm:"
)
# the link's passage is verbatim in the hoarded filing, short, and cited
(p,) = [p for p in T.read_passages(c) if p.passage_id == fts_link.passage_id]
assert len(p.passage) <= 400 and p.item_key == fts_link.item_key
text = edgar.read_doc(hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm")
assert p.passage in text and p.label == "" and p.label_version == ""
assert len(T.read_hits(c)) == 4 and len(T.read_passages(c)) == 1
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
assert "2 piece(s) missing" in T.read_gaps(c)[0].note
logs = {
t
for (t,) in c.execute(
"SELECT table_name FROM cms.ingest_log WHERE module = 'market.edgar'"
).fetchall()
}
assert {"market.filing_hit", "market.filing_passage", "market.entity_link"} <= logs
assert published == [True] and store.count() == 1
# a rerun is idempotent and downloads nothing new
n_docs = _fts_net(monkeypatch, forbid_docs=True)
res2 = runner.invoke(app, [*FTS, "--write"])
assert res2.exit_code == 0
assert "passages kept 0," in res2.output # stored passages are not re-read
assert (
_count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1
)
assert n_docs == [] # the 404 is remembered, not asked again
assert "gone.htm: document not found" in res2.output
def test_fts_cli_refusal_prints_gap(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch, status=403)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
assert _count(c, "market.filing_hit") == 0 and _count(c, "market.entity") == 0
assert published == [True]
def test_fts_cli_withholds_ama_passages_and_drops_unsupported_links(env, monkeypatch):
c, _, _, _ = env
doc = f"<p>The Medicare Physician Fee Schedule: {LICENSED}.</p>".encode()
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": doc})
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "passages kept 0, withheld by the AMA guard 1" in res.output
assert "fts links 0 (none)" in res.output
assert T.read_links(c, basis="fts") == [] and T.read_passages(c) == []
assert len(T.read_hits(c)) == 1 # the hit stays
def test_fts_cli_unverifiable_quote_drops_link(env, monkeypatch):
c, _, _, hoard = env
_fts_net(monkeypatch, hits=HITS[:1])
real = mk._verified
def tamper(links, passages):
for p in passages.values():
p.__dict__["passage"] = "not in the filing"
return real(links, passages)
monkeypatch.setattr(mk, "_verified", tamper)
res = runner.invoke(app, [*FTS, "--write"])
assert "fts links 0" in res.output and "quote not found in the hoard" in res.output
assert T.read_links(c, basis="fts") == []
def test_verified_handles_missing_filing_and_unpassaged_links(env):
_, _, _, hoard = env
link = T.LinkRow(
"1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "A:b.htm:0"
)
p = T.PassageRow("A:b.htm:0", "1", "A", "b.htm", "t", 0, "x")
assert mk._verified([link], {("A:b.htm:0", "t"): p}) == ([], 1)
bare = T.LinkRow("1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "")
assert mk._verified([bare], {}) == ([], 1)
def test_fts_cli_limit_defers_downloads_and_writes_nothing(env, monkeypatch):
c, _, published, _ = env
hits = [
HITS[0],
_hit_json(
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
),
]
_fts_net(monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET, "b.htm": SNIPPET})
res = runner.invoke(app, [*FTS, "--limit", "1", "--write"])
assert res.exit_code == 0, res.output
assert "1 filing(s) not downloaded yet" in res.output
assert "incomplete (limit or time): nothing written" in res.output
assert published == []
# the newest filing was fetched first
assert "a10k.htm" in str(list(http.hoard("sec", "docs").rglob("*.htm")))
def test_fts_cli_deadline_stops_fetching(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch)
ticks = iter(range(0, 10_000, 100))
monkeypatch.setattr(mk.time, "monotonic", lambda: next(ticks))
res = runner.invoke(app, [*FTS, "--max-minutes", "1", "--write"])
assert res.exit_code == 0, res.output
assert "incomplete" in res.output and published == []
def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch):
c, _, _, _ = env
_fts_net(monkeypatch)
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
(p,) = [p for p in T.read_passages(c) if p.passage.startswith("Substantially")]
c.execute(
"UPDATE market.filing_passage SET label = 'risk_factor', label_version = 'v' "
"WHERE passage_id = ?",
[p.passage_id],
)
_fts_net(monkeypatch) # a refresh reads the documents again
res = runner.invoke(app, [*FTS, "--refresh", "--write"])
assert res.exit_code == 0 and "new entities 0" in res.output
got = {q.passage_id: q for q in T.read_passages(c)}
assert (
got[p.passage_id].label == "risk_factor"
and got[p.passage_id].label_version == "v"
)
(link,) = T.read_links(c, basis="fts")
assert link.strength == "risk_factor" and link.passage_id == p.passage_id
def test_fts_cli_missing_submissions_is_a_gap(env, monkeypatch):
c, _, _, _ = env
_fts_net(monkeypatch, hits=HITS[:1])
_online(
monkeypatch,
[
(
r"efts\.sec\.gov/.*",
lambda m: (
httpx.Response(
200, json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
)
if "Medicare+Physician" in m.group(0)
or "Medicare%20Physician" in m.group(0)
else httpx.Response(
200, json={"hits": {"total": {"value": 0}, "hits": []}}
)
),
),
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
(
r"company_tickers\.json",
lambda m: httpx.Response(
200, content=(FX / "company_tickers.json").read_bytes()
),
),
],
)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "CIK0000012345: no submissions, no entity" in res.output
assert T.read_entities(c) == []
# classification: the local model, resumable through the ledger
class _Ask:
model = "qwen-test"
def __init__(self, answers):
self.answers = answers
self.calls = 0
def __call__(self, system, user, schema):
self.calls += 1
return self.answers.pop(0)
TWO = [
HITS[0],
_hit_json("0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"),
]
TWO_DOCS = {"a10k.htm": SNIPPET, "b.htm": SNIPPET}
def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path):
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / f"{task}-{fam}.jsonl"
)
# a passage of a term outside this run's scope is never sent to the model
T.write_passages(
c, [T.PassageRow("X:y.htm:1", "9", "X", "y.htm", "other term", 1, "z")]
)
ask = _Ask(
[
{"label": "revenue_exposure", "reason": "r"},
None,
{"label": "incidental", "reason": "i"},
]
)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0, res.output
assert "classified in 1 round(s): {'done': 1" in res.output and ask.calls == 1
labelled = [p for p in T.read_passages(c) if p.label]
assert len(labelled) == 1 and labelled[0].label == "revenue_exposure"
assert labelled[0].label_version == "model-reading exposure-v1 qwen-test"
# resume: the done unit is skipped; the next call fails (None) and stays unlabelled
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0 and ask.calls == 2
assert "'failed': 1" in res.output
# a rerun without --classify keeps what is stored
res = runner.invoke(app, [*FTS, "--write"])
assert len([p for p in T.read_passages(c) if p.label]) == 1
got = sorted(l.strength for l in T.read_links(c, basis="fts"))
assert got == ["mention", "revenue_exposure"]
def test_fts_cli_incidental_label_makes_no_link(env, monkeypatch, tmp_path):
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
)
ask = _Ask([{"label": "incidental", "reason": "i"}] * 2)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
res = runner.invoke(app, [*FTS, "--classify", "--write"])
assert res.exit_code == 0, res.output
assert T.read_links(c, basis="fts") == [] and len(T.read_passages(c)) == 2
def test_asker_builds_json_extractor(monkeypatch):
sentinel = object()
seen = {}
monkeypatch.setattr("llm.config.load", lambda: "cfg")
monkeypatch.setattr(
"llm.pool.HostPool.from_config", classmethod(lambda cls, cfg: ("pool", cfg))
)
def make(cfg, pool):
seen["args"] = (cfg, pool)
return sentinel
monkeypatch.setattr("llm.structured.json_extractor", make)
assert mk._asker() is sentinel
assert seen["args"] == ("cfg", ("pool", "cfg"))
def test_gap_note_truncates():
note = mk._gap_note([f"i{n}" for n in range(20)])
assert note.startswith("20 piece(s) missing: i0; i1") and "... 5 more" in note
def test_fts_cli_submissions_refusal_is_a_gap(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch, hits=HITS[:1])
_online(
monkeypatch,
[
(
r"efts\.sec\.gov/.*",
lambda m: httpx.Response(
200,
json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
if "Medicare%20Physician" in m.group(0)
or "Medicare+Physician" in m.group(0)
else {"hits": {"total": {"value": 0}, "hits": []}},
),
),
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
(r"CIK\d{10}\.json", lambda m: httpx.Response(403)),
(r"company_tickers\.json", lambda m: httpx.Response(403)),
],
)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
assert _count(c, "market.filing_hit") == 0 and published == [True]
def test_fts_cli_long_gap_list_is_abridged(env, monkeypatch):
c, _, _, _ = env
hits = [
_hit_json("0000012346", f"0000012346-23-{n:06d}", "x.pdf", ["8082"])
for n in range(22)
]
_fts_net(monkeypatch, hits=hits)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts: ... 2 more" in res.output
assert "... 7 more" in T.read_gaps(c)[0].note
def test_fts_cli_flaky_document_is_skipped(env, monkeypatch):
hits = [
HITS[0],
_hit_json(
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
),
]
_fts_net(
monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET}, broken={"b.htm": "timeout"}
)
monkeypatch.setattr("time.sleep", lambda s: None)
res = runner.invoke(app, FTS)
assert res.exit_code == 0, res.output
assert "b.htm: sec-edgar: HTTP 0 ReadTimeout" in res.output
assert "passages kept 1," in res.output
def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch, hits=HITS[:1], broken={"a10k.htm": 403})
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts: sec-edgar: HTTP 403" in res.output
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [
True
]
TWICE = (
b"<p>First, we are paid under the Medicare Physician Fee Schedule rates.</p>"
b"<p>Second, our risk comes from the Medicare Physician Fee Schedule cuts.</p>"
)
def test_fts_cli_classify_falls_back_to_the_next_passage(env, monkeypatch, tmp_path):
"""An incidental first passage retires it; the link moves to the next
passage, which the next round reads."""
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
)
ask = _Ask(
[
{"label": "incidental", "reason": "i"},
{"label": "risk_factor", "reason": "r"},
]
)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE})
res = runner.invoke(app, [*FTS, "--classify", "--write"])
assert res.exit_code == 0, res.output
assert "classified in 2 round(s)" in res.output and ask.calls == 2
(link,) = T.read_links(c, basis="fts")
assert link.strength == "risk_factor"
labels = {p.passage_id: p.label for p in T.read_passages(c)}
assert sorted(labels.values()) == ["incidental", "risk_factor"]
text = {p.passage_id: p.passage for p in T.read_passages(c)}[link.passage_id]
assert text.startswith("Second")
def test_fts_cli_classify_limit_stops_between_rounds(env, monkeypatch, tmp_path):
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
)
ask = _Ask([{"label": "incidental", "reason": "i"}] * 3)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE})
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0, res.output
assert ask.calls == 1 and "classified in 1 round(s)" in res.output
# the link now rests on the second, still unlabelled passage
(link,) = T.read_links(c, basis="fts")
assert link.strength == "mention"
# ── lda ──────────────────────────────────────────────────────────────
def _lda_rec(**over):
rec = json.loads((FX / "lda_aafp_2024_q1.json").read_text())
rec.update(over)
return rec
def _lda_entities(c, *extra):
T.write_entities(
c,
[
T.EntityRow(
"AAFP",
"American Academy of Family Physicians",
"organization",
aliases="AMERICAN ACADEMY OF FAMILY PHYSICIANS",
),
T.EntityRow(
"PRIMARY-CARE-SEAT",
"Primary care rotating-seat societies",
"organization",
aliases="PRIMARY CARE ROTATING SEAT",
pseudo=True,
),
T.EntityRow(
"0000000001",
"SWEEP ONLY CO",
"company",
cik="0000000001",
aliases="SWEEP ONLY",
),
*extra,
],
)
@pytest.fixture
def lda(env, monkeypatch):
from market import lda as L
c, store, published, hoard = env
_lda_entities(c)
seen: list[httpx.Request] = []
monkeypatch.setattr(L, "new_pacer", lambda key=None: _Pacer())
monkeypatch.setattr(mk, "_lda_now", lambda: date(2026, 11, 13))
state = {"results": [_lda_rec()], "status": 200, "text": None}
def handler(request):
seen.append(request)
if state["status"] != 200:
return httpx.Response(state["status"], text="no")
if state["text"] is not None:
return httpx.Response(200, text=state["text"])
q = request.url.params
name = q.get("client_name") or ""
results = state["results"]
if name:
results = [
r for r in results if name.upper() in r["client"]["name"].upper()
]
year = q.get("filing_year")
if year:
results = [r for r in results if str(r["filing_year"]) == year]
return httpx.Response(
200,
json={
"count": len(results),
"next": None,
"previous": None,
"results": results,
},
)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(handler)),
)
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
return c, store, published, hoard, seen, state
def test_lda_cli_entity_write(lda):
c, store, published, hoard, seen, state = lda
foundation = _lda_rec(
filing_uuid="f0000000-0000-0000-0000-000000000000",
client={
**_lda_rec()["client"],
"name": "AMERICAN ACADEMY OF FAMILY PHYSICIANS FOUNDATION",
},
)
state["results"] = [_lda_rec(), foundation]
T.write_gap(c, "lda:lda", "old")
res = runner.invoke(
app,
[
"market",
"lda",
"--entity",
"AAFP",
"--from",
"2024",
"--to",
"2024",
"--write",
],
)
assert res.exit_code == 0, res.output
rows = T.read_lda_filings(c)
assert [(r.filing_uuid[:8], r.entity_key, r.matched_by) for r in rows] == [
("df5f17c4", "AAFP", "client")
]
assert rows[0].amount == 825016.0 and rows[0].source_file.endswith(".1.json")
assert _count(c, "market.lda_issue") == 5
assert T.read_gaps(c) == []
assert published == [True]
assert {
(t, n)
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
} == {("market.lda_filing", 1), ("market.lda_issue", 5)}
assert seen and all(r.url.host == "lda.gov" for r in seen)
assert seen[0].url.params["client_name"] == "American Academy of Family Physicians"
assert seen[0].url.params["filing_year"] == "2024"
assert "authorization" not in seen[0].headers
files = list((hoard / "lda" / "2024").glob("*.1.json"))
assert files and all(
"american-academy-of-family-physicians" in f.name for f in files
)
assert "1 filings" in res.output and "1 clients" in res.output
assert "HCR 1" in res.output and "MMM 1" in res.output
assert [i.title for i in store.list_items() if "Lobbying" in i.title] == [
"Lobbying Disclosure Act filings, 2024"
]
def test_lda_cli_without_write_stores_nothing(lda):
c, _, published, _, seen, _ = lda
res = runner.invoke(app, ["market", "lda", "--from", "2024", "--to", "2024"])
assert res.exit_code == 0, res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.lda_filing") == 0
assert published == []
names = {r.url.params["client_name"] for r in seen}
assert "American Academy of Family Physicians" in names
assert not any("PRIMARY CARE" in n.upper() for n in names) # pseudo skipped
assert not any("SWEEP" in n.upper() for n in names) # not a seed company
def test_lda_cli_terms_offline(lda):
c, _, _, hoard, seen, state = lda
other = _lda_rec(
filing_uuid="a0000000-0000-0000-0000-000000000000",
client={**_lda_rec()["client"], "id": 5, "name": "SOME UNTRACKED CLIENT"},
)
state["results"] = [_lda_rec(), other]
cmd = [
"market",
"lda",
"--entity",
"AAFP",
"--terms",
"--from",
"2024",
"--to",
"2024",
]
res = runner.invoke(app, [*cmd, "--write"])
assert res.exit_code == 0, res.output
terms = [r for r in seen if "filing_specific_lobbying_issues" in r.url.params]
assert [r.url.params["filing_specific_lobbying_issues"] for r in terms] == list(
__import__("market.lda", fromlist=["x"]).LDA_TERMS
)
got = {
r.filing_uuid[:2]: (r.entity_key, r.matched_by) for r in T.read_lda_filings(c)
}
assert got == {"df": ("AAFP", "client"), "a0": ("", "")}
# a second run reads only the hoard: not one request leaves
n = len(seen)
again = runner.invoke(app, cmd)
assert again.exit_code == 0 and len(seen) == n
assert "2 filings" in again.output and "0 requests" in again.output
assert "terms 5" in again.output
def test_lda_cli_range_uses_all_dir_and_filters_years(lda):
c, _, _, hoard, seen, state = lda
old = _lda_rec(filing_uuid="b0000000-0000-0000-0000-000000000000", filing_year=2015)
state["results"] = [_lda_rec(), old]
res = runner.invoke(
app,
[
"market",
"lda",
"--entity",
"AAFP",
"--from",
"2020",
"--to",
"2024",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "filing_year" not in seen[0].url.params
assert list((hoard / "lda" / "all").glob("*.1.json"))
assert [r.filing_year for r in T.read_lda_filings(c)] == [2024]
def test_lda_cli_refusal_prints_gap(lda):
c, _, published, _, seen, state = lda
state["status"] = 403
res = runner.invoke(
app,
[
"market",
"lda",
"--entity",
"AAFP",
"--from",
"2024",
"--to",
"2024",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "gap: lda:lda: lda: HTTP 403" in res.output
assert [(g.source) for g in T.read_gaps(c)] == ["lda:lda"]
assert _count(c, "market.lda_filing") == 0
assert len(seen) == 1 # no workaround, no retry on a 403
state["status"], state["text"] = 200, "<html>Just a moment</html>"
res = runner.invoke(
app, ["market", "lda", "--entity", "AAFP", "--from", "2024", "--to", "2024"]
)
assert "gap: lda:lda:" in res.output and "not JSON" in res.output
assert len(T.read_gaps(c)) == 1 # no --write: not recorded again
def test_lda_cli_limit_and_minutes_stop_early_and_keep_the_gap(lda, monkeypatch):
c, _, _, _, seen, _ = lda
_lda_entities(
c,
T.EntityRow(
"ACP",
"American College of Physicians",
"organization",
aliases="AMERICAN COLLEGE OF PHYSICIANS",
),
)
T.write_gap(c, "lda:lda", "old")
res = runner.invoke(
app,
["market", "lda", "--from", "2024", "--to", "2024", "--limit", "1", "--write"],
)
assert res.exit_code == 0, res.output
assert len(seen) == 1 and "stopped: request limit" in res.output
assert [g.note for g in T.read_gaps(c)] == ["old"]
assert _count(c, "market.lda_filing") == 1
ticks = iter([0.0])
monkeypatch.setattr(mk.time, "monotonic", lambda: next(ticks, 10_000.0))
res = runner.invoke(
app,
[
"market",
"lda",
"--from",
"2024",
"--to",
"2024",
"--max-minutes",
"1",
"--refresh",
],
)
assert res.exit_code == 0, res.output
assert "stopped: time limit" in res.output
def test_lda_cli_bad_entity_and_empty_registry(lda, monkeypatch):
c, *_ = lda
res = runner.invoke(app, ["market", "lda", "--entity", "NOPE"])
assert res.exit_code == 2 and "unknown entity: NOPE" in res.output
res = runner.invoke(app, ["market", "lda", "--entity", "PRIMARY-CARE-SEAT"])
assert res.exit_code == 2
bare = duckdb.connect(":memory:")
monkeypatch.setattr(mk, "_read", lambda: bare)
res = runner.invoke(app, ["market", "lda"])
assert res.exit_code == 1 and "stack market entities --write" in res.output
def test_lda_cli_resupersedes_across_runs(lda):
c, _, _, _, _, state = lda
cmd = [
"market",
"lda",
"--entity",
"AAFP",
"--from",
"2024",
"--to",
"2024",
"--write",
]
assert runner.invoke(app, cmd).exit_code == 0
amend = _lda_rec(
filing_uuid="c0000000-0000-0000-0000-000000000000",
filing_type="1A",
dt_posted="2024-05-01T11:45:16-04:00",
expenses="900000.00",
)
state["results"] = [amend]
assert runner.invoke(app, [*cmd, "--refresh"]).exit_code == 0
got = {r.filing_uuid[:2]: r.superseded for r in T.read_lda_filings(c)}
assert got == {"df": True, "c0": False}
def test_lda_cli_year_order_and_seed_company_tracked(lda, monkeypatch):
c, _, _, _, seen, state = lda
monkeypatch.undo() # the real clock, only to read it
assert mk._lda_now() == date.today()
res = runner.invoke(app, ["market", "lda", "--from", "2025", "--to", "2024"])
assert res.exit_code == 2 and "is after --to" in res.output
def test_lda_cli_tracks_seed_companies_only(lda):
c, _, _, _, seen, state = lda
seed = load_seed()[0][0]
T.write_entities(
c,
[
T.EntityRow(
seed.cik,
seed.name,
"company",
cik=seed.cik,
aliases="|".join(seed.aliases),
)
],
)
res = runner.invoke(
app, ["market", "lda", "--entity", seed.cik, "--from", "2024", "--to", "2024"]
)
assert res.exit_code == 0, res.output
assert {r.url.params["client_name"] for r in seen} >= {seed.name}
assert "0 filings" in res.output
# ── fec ──────────────────────────────────────────────────────────────
FEC_SECRET = "fec-cli-key-do-not-leak"
def _fec_entities(c):
T.write_entities(
c,
[
T.EntityRow(
"AAFP",
"American Academy of Family Physicians",
"organization",
aliases="AMERICAN ACADEMY OF FAMILY PHYSICIANS",
),
T.EntityRow("AMA", "AMA", "organization", aliases="AMA"),
T.EntityRow("LOOKALIKE", "LOOKALIKE", "organization", aliases="LOOKALIKE"),
T.EntityRow(
"PRIMARY-CARE-SEAT",
"Primary care rotating-seat societies",
"organization",
aliases="PRIMARY CARE ROTATING SEAT",
pseudo=True,
),
],
)
@pytest.fixture
def fec(env, monkeypatch):
from market import fec as fec_api
c, store, published, hoard = env
_fec_entities(c)
seen: list[httpx.Request] = []
page1 = json.loads((FX / "fec_schedule_b_page1.json").read_text())
totals = json.loads((FX / "fec_totals_C00411553_2024.json").read_text())
committee = json.loads((FX / "fec_committees_aafp.json").read_text())
monkeypatch.setattr(fec_api, "new_pacer", lambda: _Pacer())
state = {"status": 200}
def handler(request):
seen.append(request)
if state["status"] != 200:
return httpx.Response(state["status"], text="no")
path = request.url.path
if path.endswith("/totals/"):
return httpx.Response(200, json=totals)
if path.endswith("/schedule_b/"):
if "last_index" in request.url.params:
return httpx.Response(200, json={"results": [], "pagination": {}})
return httpx.Response(200, json=page1)
if path.startswith("/v1/committees"):
return httpx.Response(
200,
json={
"results": [
{
"committee_id": "C00999999",
"name": "AMALGAMATED TRANSIT UNION PAC",
"committee_type": "Q",
"designation": "U",
}
]
},
)
return httpx.Response(200, json=committee)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(handler),
headers={"X-Api-Key": key} if key else {},
),
)
monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET)
return c, store, published, hoard, seen, state
def test_fec_cli_write_from_cache(fec):
c, _, published, hoard, seen, state = fec
T.write_gap(c, "fec:fec", "old")
dry = runner.invoke(app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024"])
assert dry.exit_code == 0, dry.output
assert "dry run: pass --write" in dry.output
assert "1 committees (1 curated, 0 exact-name, 0 hoarded)" in dry.output
assert _count(c, "market.fec_committee") == 0
assert published == []
assert seen and all("/committees" not in r.url.path for r in seen)
assert all(FEC_SECRET not in str(r.url) for r in seen)
n = len(seen)
state["status"] = 500
res = runner.invoke(
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
)
assert res.exit_code == 0, res.output
assert len(seen) == n # the hoard answered; the 500 was never requested
assert "stored: 1 committees, 1 totals, 3 schedule B rows" in res.output
(committee,) = T.read_fec_committees(c)
assert committee.basis == "curated" and committee.committee_id == "C00411553"
assert committee.source_file.endswith("fec/committee/C00411553.json")
(totals,) = T.read_fec_totals(c)
assert totals.receipts == 513798.01
assert totals.fed_candidate_committee_contributions == 683300.0
names = {r.recipient_name for r in T.read_fec_contributions(c)}
assert "JANE Q PUBLIC" in names and "MEMO OF THE 100" in names
from market.fec import contributions_by_cycle
assert contributions_by_cycle(c, "AAFP") == {2024: 125.0}
assert T.read_gaps(c) == []
assert published == [True]
assert {
(t, n)
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
} == {
("market.fec_committee", 1),
("market.fec_totals", 1),
("market.fec_contribution", 3),
}
manifest = (hoard / "MANIFEST.jsonl").read_text()
assert FEC_SECRET not in manifest
assert (hoard / "fec" / "schedule_b" / "C00411553" / "2024" / "1.json").is_file()
def test_fec_cli_no_write_prints_candidates(fec):
c, _, published, _, seen, _ = fec
res = runner.invoke(
app, ["market", "fec", "--entity", "LOOKALIKE", "--cycles", "2024"]
)
assert res.exit_code == 0, res.output
assert "candidate: LOOKALIKE C00999999 AMALGAMATED TRANSIT UNION PAC" in res.output
assert "0 committees" in res.output and "1 candidates" in res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.fec_committee") == 0
assert _count(c, "market.fec_contribution") == 0
assert published == []
assert seen[0].url.params["q"] == "LOOKALIKE"
assert all(
"/committee/" not in r.url.path or r.url.path.startswith("/v1/committees")
for r in seen
)
default = runner.invoke(app, ["market", "fec", "--entity", "LOOKALIKE"])
assert default.exit_code == 0, default.output
assert "1 candidates" in default.output
bad = runner.invoke(app, ["market", "fec", "--entity", "NOPE", "--cycles", "2024"])
assert bad.exit_code == 2 and "unknown entity: NOPE" in bad.output
cycles = runner.invoke(app, ["market", "fec", "--cycles", "nope"])
assert cycles.exit_code == 2 and "not a list of years" in cycles.output
def test_fec_cli_full_pull_keeps_hoard_ids(fec, monkeypatch):
from market import fec as fec_api
c, _, published, hoard, seen, _ = fec
monkeypatch.setattr(fec_api, "hoard_ids", lambda: ("C00765230",))
committee = json.loads((FX / "fec_committees_aafp.json").read_text())
totals = json.loads((FX / "fec_totals_C00411553_2024.json").read_text())
page1 = json.loads((FX / "fec_schedule_b_page1.json").read_text())
def handler(request):
seen.append(request)
path = request.url.path
if path == "/v1/committee/C00765230/":
return httpx.Response(
200,
json={
"results": [
{
"committee_id": "C00765230",
"name": "AMERICAN ANTHEM PAC",
"committee_type": "V",
"designation": "U",
}
]
},
)
cid = request.url.params.get("committee_id", "")
if path.endswith("/totals/"):
if cid == "C00765230":
return httpx.Response(200, json={"results": []})
return httpx.Response(200, json=totals)
if path.endswith("/schedule_b/"):
if cid == "C00765230" or "last_index" in request.url.params:
return httpx.Response(200, json={"results": [], "pagination": {}})
return httpx.Response(200, json=page1)
if path.startswith("/v1/committees"):
return httpx.Response(200, json={"results": []})
if path.endswith("/C00411553/"):
return httpx.Response(200, json=committee)
return httpx.Response(200, json={"results": []})
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(handler),
headers={"X-Api-Key": key} if key else {},
),
)
res = runner.invoke(app, ["market", "fec", "--cycles", "2024", "--write"])
assert res.exit_code == 0, res.output
assert "hoarded: C00765230 AMERICAN ANTHEM PAC" in res.output
rows = {r.committee_id: r for r in T.read_fec_committees(c)}
assert rows["C00765230"].basis == "hoarded" and rows["C00765230"].entity_key == ""
assert rows["C00411553"].basis == "curated"
assert (hoard / "fec" / "committee" / "C00765230.json").is_file()
from market.fec import contributions_by_cycle
assert contributions_by_cycle(c, "") == {}
assert contributions_by_cycle(c, "AMA") == {}
assert published == [True]
def test_fec_cli_refusal_prints_gap(fec, monkeypatch):
c, _, published, _, seen, state = fec
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
missing = runner.invoke(
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
)
assert missing.exit_code == 0, missing.output
assert "gap: fec:fec:" in missing.output
assert "FEC_API_KEY" in missing.output
assert seen == []
assert [g.source for g in T.read_gaps(c)] == ["fec:fec"]
assert published == [True]
monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET)
state["status"] = 403
res = runner.invoke(
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: fec:fec: fec: HTTP 403" in res.output
assert FEC_SECRET not in res.output
assert _count(c, "market.fec_committee") == 0
assert len(seen) == 1
assert all(FEC_SECRET not in str(r.url) for r in seen)
assert seen[0].headers["x-api-key"] == FEC_SECRET
def test_fec_cli_limit_keeps_the_gap_and_empty_registry(fec, monkeypatch):
c, _, _, _, seen, _ = fec
T.write_gap(c, "fec:fec", "old")
res = runner.invoke(
app,
["market", "fec", "--cycles", "2024,2022", "--limit", "1", "--write"],
)
assert res.exit_code == 0, res.output
assert "stopped: request limit" in res.output
assert [g.note for g in T.read_gaps(c)] == ["old"]
assert _count(c, "market.fec_committee") == 1
assert _count(c, "market.fec_totals") == 0
assert len(seen) == 1
monkeypatch.setattr(mk, "_lda_tracked", lambda con: [])
empty = runner.invoke(app, ["market", "fec", "--cycles", "2024"])
assert empty.exit_code == 1 and "no tracked entities" in empty.output
blank = runner.invoke(app, ["market", "fec", "--cycles", ","])
assert blank.exit_code == 2 and "--cycles is empty" in _plain(blank.output)
# ── congress ─────────────────────────────────────────────────────────
CONGRESS_SECRET = "congress-cli-key-do-not-leak"
def _congress_states(c) -> None:
c.execute("CREATE SCHEMA IF NOT EXISTS reference_data")
c.execute(
"CREATE TABLE IF NOT EXISTS reference_data.ansi_fips_state ("
"ansi_fips_state_code VARCHAR, ansi_fips_state_abbreviation VARCHAR, "
"ansi_fips_state_name VARCHAR)"
)
c.execute(
"INSERT INTO reference_data.ansi_fips_state VALUES "
"('26', 'MI', 'Michigan'), ('02', 'AK', 'Alaska'), ('51', 'VA', 'Virginia')"
)
@pytest.fixture
def congress(env, monkeypatch):
from market import congress as congress_api
c, store, published, hoard = env
_congress_states(c)
T.ensure_tables(c)
seen: list[httpx.Request] = []
members = json.loads((FX / "congress_members_118.json").read_text())
bills = json.loads((FX / "congress_bills_page.json").read_text())
detail = json.loads((FX / "congress_bill_detail.json").read_text())
actions = json.loads((FX / "congress_bill_actions.json").read_text())
cosponsors = json.loads((FX / "congress_bill_cosponsors.json").read_text())
people = (FX / "legislators_snippet.yaml").read_bytes()
membership = (FX / "membership_snippet.yaml").read_bytes()
state = {"status": 200}
for name in ("member_pacer", "raw_pacer", "commit_pacer"):
monkeypatch.setattr(congress_api, name, lambda: _Pacer())
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request)
if state["status"] != 200:
return httpx.Response(state["status"], text="no")
host = request.url.host
if host == "api.congress.gov":
path = request.url.path
if path.endswith("/actions"):
return httpx.Response(200, json=actions)
if path.endswith("/cosponsors"):
return httpx.Response(200, json=cosponsors)
parts = [part for part in path.split("/") if part]
if "bill" in parts and len(parts) >= 5:
return httpx.Response(200, json=detail)
if "bill" in parts:
return httpx.Response(200, json=bills)
return httpx.Response(200, json=members)
if host == "api.github.com":
return httpx.Response(200, json=[{"sha": "deadbeef"}])
if request.url.path.endswith("legislators-historical.yaml"):
return httpx.Response(200, content=b"[]\n")
if request.url.path.endswith("committee-membership-current.yaml"):
return httpx.Response(200, content=membership)
return httpx.Response(200, content=people)
def client(source, key=None):
headers = {"X-Api-Key": key} if key else {}
return httpx.Client(transport=httpx.MockTransport(handler), headers=headers)
monkeypatch.setattr(http, "client", client)
monkeypatch.setattr(http, "api_key", lambda source, env=None: CONGRESS_SECRET)
return c, store, published, hoard, seen, state
def test_members_cli_no_write_prints_counts(congress):
c, _, published, _, seen, _ = congress
res = runner.invoke(app, ["market", "members", "--from", "118", "--to", "118"])
assert res.exit_code == 0, res.output
assert "2 members (2 house, 0 senate, 1 at-large)" in res.output
assert "congress 118: 2" in res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.congress_member") == 0
assert published == []
assert any(r.url.host == "api.congress.gov" for r in seen)
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
assert all("api_key" not in str(r.url) for r in seen)
congress_hits = [r for r in seen if r.url.host == "api.congress.gov"]
assert congress_hits[0].headers["x-api-key"] == CONGRESS_SECRET
github_hits = [r for r in seen if r.url.host != "api.congress.gov"]
assert github_hits
assert all(r.headers.get("x-api-key") in (None, "") for r in github_hits)
def test_members_cli_write(congress):
c, _, published, hoard, seen, _ = congress
T.write_gap(c, "congress:members", "old")
res = runner.invoke(
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "stored: 2 members" in res.output
rows = {r.bioguide_id: r for r in T.read_members(c)}
assert rows["D000624"].district_id == "MI-12"
assert rows["D000624"].fec_ids == "H0XX00001"
assert rows["D000624"].source == "congress.gov"
assert rows["Y000033"].district_id == "AK-00"
assert rows["Y000033"].state == "AK"
assert T.read_gaps(c) == []
assert published == [True]
assert {
(t, n)
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
} == {("market.congress_member", 2)}
manifest = (hoard / "MANIFEST.jsonl").read_text()
assert CONGRESS_SECRET not in manifest
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
kept = T.MemberRow(
117,
"Z000001",
"Other",
"Republican",
"MI",
"MI-01",
"house",
"",
"congress.gov",
)
T.write_members(c, [kept])
assert {r.congress for r in T.read_members(c)} == {117, 118}
def test_members_cli_missing_key_gap_only(congress, monkeypatch):
c, _, published, _, seen, _ = congress
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
res = runner.invoke(
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: congress:members:" in res.output
assert "CONGRESS_API_KEY" in res.output
assert "API_DATA_GOV_KEY" in res.output
assert seen == []
assert [g.source for g in T.read_gaps(c)] == ["congress:members"]
assert _count(c, "market.congress_member") == 0
assert published == [True]
def test_members_cli_refusal_prints_gap(congress):
c, _, published, _, seen, state = congress
state["status"] = 403
res = runner.invoke(
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: congress:members: congress: HTTP 403" in res.output
assert CONGRESS_SECRET not in res.output
assert _count(c, "market.congress_member") == 0
assert len(seen) == 1
assert seen[0].url.host == "api.congress.gov"
assert seen[0].headers["x-api-key"] == CONGRESS_SECRET
assert CONGRESS_SECRET not in str(seen[0].url)
def test_members_cli_from_after_to(congress):
res = runner.invoke(app, ["market", "members", "--from", "119", "--to", "110"])
assert res.exit_code == 2
assert "--from 119 is after --to 110" in res.output
def test_committees_cli_write(congress):
c, _, published, hoard, seen, _ = congress
T.write_gap(c, "legislators:committees", "old")
dry = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "119"])
assert dry.exit_code == 0, dry.output
assert "1 assignments" in dry.output
assert "dry run: pass --write" in dry.output
assert _count(c, "market.committee_assignment") == 0
assert published == []
res = runner.invoke(
app, ["market", "committees", "--from", "119", "--to", "119", "--write"]
)
assert res.exit_code == 0, res.output
assert "stored: 1 assignments" in res.output
(row,) = T.read_assignments(c)
assert row.committee_id == "HSWM02"
assert row.bioguide_id == "E000001"
assert row.snapshot == date(2025, 4, 1)
assert row.source_sha == "deadbeef"
assert row.congress == 119
assert T.read_gaps(c) == []
assert published == [True]
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
assert all(r.headers.get("x-api-key") in (None, "") for r in seen)
manifest = (hoard / "MANIFEST.jsonl").read_text()
assert CONGRESS_SECRET not in manifest
logged = c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
assert ("market.committee_assignment", 1) in logged
def test_bills_cli_no_write_lists_matches(congress):
c, _, published, _, seen, _ = congress
res = runner.invoke(app, ["market", "bills", "--from", "118", "--to", "118"])
assert res.exit_code == 0, res.output
assert "118 HR 1234 physician fee schedule" in res.output
assert "1 bills" in res.output
assert "congress 118: 1" in res.output
assert "Farm Bill" not in res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.bill") == 0
assert published == []
hits = [r for r in seen if r.url.host == "api.congress.gov"]
assert hits
assert all(CONGRESS_SECRET not in str(r.url) for r in hits)
assert all("api_key" not in str(r.url) for r in hits)
assert hits[0].headers["x-api-key"] == CONGRESS_SECRET
def test_bills_cli_offline_write(congress):
c, store, published, hoard, seen, _ = congress
T.write_gap(c, "congress:bills", "old")
res = runner.invoke(
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "stored: 1 bills" in res.output
(bill,) = T.read_bills(c)
assert bill.bill_type == "HR"
assert bill.number == 1234
assert bill.matched_term == "physician fee schedule"
assert bill.policy_area == "Health"
assert bill.introduced == date(2023, 1, 9)
assert bill.item_key
assert len(T.read_bill_actions(c)) == 3
roles = sorted((m.role, m.bioguide_id) for m in T.read_bill_members(c))
assert roles == [
("cosponsor", "S000001"),
("cosponsor", "Y000033"),
("sponsor", "D000624"),
]
assert T.read_gaps(c) == []
assert published == [True]
assert any(
(i.url or "").endswith("/118th-congress/house-bill/1234")
for i in store.list_items()
)
assert ("market.bill", 1) in c.execute(
"SELECT table_name, rows FROM cms.ingest_log"
).fetchall()
manifest = (hoard / "MANIFEST.jsonl").read_text()
assert CONGRESS_SECRET not in manifest
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
kept = T.BillRow(
117,
"S",
1,
"Telehealth Modernization Act",
None,
"Health",
None,
"",
"telehealth",
"https://www.congress.gov/bill/117th-congress/senate-bill/1",
"",
)
T.write_bills(c, [kept])
assert {r.congress for r in T.read_bills(c)} == {117, 118}
def test_bills_cli_refusal_prints_gap(congress):
c, _, published, _, seen, state = congress
state["status"] = 403
res = runner.invoke(
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: congress:bills: congress: HTTP 403" in res.output
assert CONGRESS_SECRET not in res.output
assert _count(c, "market.bill") == 0
assert len(seen) == 1
assert seen[0].url.host == "api.congress.gov"
assert seen[0].headers["x-api-key"] == CONGRESS_SECRET
assert CONGRESS_SECRET not in str(seen[0].url)
assert published == [True]
def test_bills_cli_from_after_to(congress):
res = runner.invoke(app, ["market", "bills", "--from", "119", "--to", "110"])
assert res.exit_code == 2
assert "--from 119 is after --to 110" in res.output
def test_bills_cli_missing_key_gap_only(congress, monkeypatch):
c, _, published, _, seen, _ = congress
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
res = runner.invoke(
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: congress:bills:" in res.output
assert "CONGRESS_API_KEY" in res.output
assert seen == []
assert [g.source for g in T.read_gaps(c)] == ["congress:bills"]
assert _count(c, "market.bill") == 0
assert published == [True]
def test_bills_cli_limit_stores_nothing_and_keeps_the_gap(congress):
c, _, published, _, seen, _ = congress
T.write_gap(c, "congress:bills", "old")
res = runner.invoke(
app,
[
"market",
"bills",
"--from",
"118",
"--to",
"119",
"--limit",
"1",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "stopped: request limit" in res.output
assert "stored:" not in res.output
assert [g.note for g in T.read_gaps(c)] == ["old"]
assert _count(c, "market.bill") == 0
assert published == []
assert [r.url.path for r in seen] == ["/v3/bill/118"]
def test_committees_cli_refusal_prints_gap(congress):
c, _, published, _, seen, state = congress
state["status"] = 403
res = runner.invoke(
app, ["market", "committees", "--from", "119", "--to", "119", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: legislators:committees: legislators: HTTP 403" in res.output
assert CONGRESS_SECRET not in res.output
assert _count(c, "market.committee_assignment") == 0
assert seen
assert published == [True]
COUNTY_REL = """\
GEOID_CD119_20|GEOID_COUNTY_20|AREALAND_COUNTY_20|AREALAND_PART
0101|01003|100|100
"""
HUD_SECRET = "hud-token-do-not-leak"
@pytest.fixture
def crosswalk(env, monkeypatch):
from market import districts as district_api
c, store, published, hoard = env
c.execute("CREATE SCHEMA IF NOT EXISTS reference_data")
c.execute(
"CREATE TABLE IF NOT EXISTS reference_data.ansi_fips_state ("
"ansi_fips_state_code VARCHAR, ansi_fips_state_abbreviation VARCHAR, "
"ansi_fips_state_name VARCHAR)"
)
c.execute(
"INSERT INTO reference_data.ansi_fips_state VALUES ('01', 'AL', 'Alabama')"
)
T.ensure_tables(c)
seen: list[httpx.Request] = []
state = {"status": 200}
zcta = (FX / "tab20_cd11920_zcta520_head.txt").read_bytes()
hud = (FX / "hud_zip_cd.json").read_bytes()
monkeypatch.setattr(district_api, "census_pacer", lambda: _Pacer())
monkeypatch.setattr(district_api, "hud_pacer", lambda: _Pacer())
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request)
if state["status"] != 200:
return httpx.Response(state["status"], text="no")
if request.url.host == "www2.census.gov":
if "county20" in request.url.path:
return httpx.Response(200, content=COUNTY_REL.encode())
return httpx.Response(200, content=zcta)
if request.url.host == "www.huduser.gov":
return httpx.Response(200, content=hud)
return httpx.Response(404, text="no")
def client(source, key=None):
headers = {"Authorization": f"Bearer {key}"} if key else {}
return httpx.Client(transport=httpx.MockTransport(handler), headers=headers)
monkeypatch.setattr(http, "client", client)
monkeypatch.setattr(
http,
"api_key",
lambda source, env=None: HUD_SECRET if source == "hud" else None,
)
return c, store, published, hoard, seen, state
def test_districts_cli_no_write_prints_counts(crosswalk):
c, _, published, _, seen, _ = crosswalk
res = runner.invoke(app, ["market", "districts", "--congress", "119"])
assert res.exit_code == 0, res.output
assert "2 zip rows, 1 county rows" in res.output
assert "congress 119: 2 zip rows, 1 county rows" in res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.zip_district") == 0
assert published == []
assert seen
assert all(r.url.host == "www2.census.gov" for r in seen)
def test_districts_cli_census_offline_write(crosswalk):
c, store, published, hoard, seen, _ = crosswalk
T.write_gap(c, "census:districts", "old")
res = runner.invoke(
app,
["market", "districts", "--congress", "118", "--congress", "119", "--write"],
)
assert res.exit_code == 0, res.output
assert "stored: 4 zip rows, 2 county rows" in res.output
assert _count(c, "market.zip_district") == 4
assert _count(c, "market.county_district") == 2
assert T.read_gaps(c) == []
assert published == [True]
urls = [i.url or "" for i in store.list_items()]
assert any(u.endswith("tab20_cd11820_zcta520_natl.txt") for u in urls)
assert any(u.endswith("tab20_cd11920_county20_natl.txt") for u in urls)
assert len(urls) == 4
logged = c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
assert ("market.zip_district", 4) in logged
assert ("market.county_district", 2) in logged
manifest = (hoard / "MANIFEST.jsonl").read_text()
assert "api_key" not in manifest
assert all("api_key" not in str(r.url) for r in seen)
def test_districts_cli_refusal_prints_gap(crosswalk):
c, _, published, _, seen, state = crosswalk
state["status"] = 403
res = runner.invoke(app, ["market", "districts", "--congress", "119", "--write"])
assert res.exit_code == 0, res.output
assert "gap: census:districts: census: HTTP 403" in res.output
assert _count(c, "market.zip_district") == 0
assert len(seen) == 1
assert published == [True]
def test_hud_cli_missing_token_writes_only_gap(crosswalk, monkeypatch):
c, _, published, _, seen, _ = crosswalk
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
kept = T.ZipDistrictRow(
119, "36010", "AL-01", 1.0, "census-zcta-land", "2020", "census"
)
T.write_zip_districts(c, [kept])
res = runner.invoke(
app,
[
"market",
"hud",
"--year",
"2025",
"--quarter",
"2",
"--congress",
"119",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "gap: hud:hud:" in res.output
assert "HUD_API_TOKEN" in res.output
assert seen == []
assert [g.source for g in T.read_gaps(c)] == ["hud:hud"]
assert [(r.zip, r.method) for r in T.read_zip_districts(c)] == [
("36010", "census-zcta-land")
]
assert published == [True]
def test_hud_cli_offline_write(crosswalk):
c, _, published, hoard, seen, _ = crosswalk
kept = T.ZipDistrictRow(
119, "36010", "AL-01", 1.0, "census-zcta-land", "2020", "census"
)
T.write_zip_districts(c, [kept])
T.write_gap(c, "hud:hud", "old")
res = runner.invoke(
app,
[
"market",
"hud",
"--year",
"2025",
"--quarter",
"2",
"--congress",
"119",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "stored: 2 zip rows" in res.output
methods = {(r.zip, r.method) for r in T.read_zip_districts(c)}
assert ("36010", "census-zcta-land") in methods
assert ("36009", "hud-res") in methods
assert ("36009", "hud-bus") in methods
assert T.read_gaps(c) == []
assert published == [True]
assert seen[0].headers["authorization"] == f"Bearer {HUD_SECRET}"
assert HUD_SECRET not in str(seen[0].url)
assert HUD_SECRET not in (hoard / "MANIFEST.jsonl").read_text()
assert seen[0].url.params["type"] == "5"
assert seen[0].url.params["query"] == "All"
def test_districts_cli_defaults_to_118_and_119(crosswalk):
res = runner.invoke(app, ["market", "districts"])
assert res.exit_code == 0, res.output
assert "4 zip rows, 2 county rows" in res.output
assert "congress 118: 2 zip rows, 1 county rows" in res.output
assert "congress 119: 2 zip rows, 1 county rows" in res.output
assert "dry run: pass --write" in res.output
def test_hud_cli_no_write_prints_counts(crosswalk):
c, _, published, _, seen, _ = crosswalk
res = runner.invoke(
app, ["market", "hud", "--year", "2025", "--quarter", "2", "--congress", "119"]
)
assert res.exit_code == 0, res.output
assert "2 zip rows" in res.output
assert "dry run: pass --write" in res.output
assert "stored:" not in res.output
assert _count(c, "market.zip_district") == 0
assert published == []
assert seen[0].headers["authorization"] == f"Bearer {HUD_SECRET}"
def test_hud_cli_refusal_prints_gap(crosswalk):
c, _, published, _, seen, state = crosswalk
state["status"] = 403
res = runner.invoke(
app,
[
"market",
"hud",
"--year",
"2025",
"--quarter",
"2",
"--congress",
"119",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "gap: hud:hud: hud: HTTP 403" in res.output
assert HUD_SECRET not in res.output
assert _count(c, "market.zip_district") == 0
assert len(seen) == 1
assert published == [True]
def test_hud_cli_write_from_cache(crosswalk):
c, _, published, hoard, seen, _ = crosswalk
dest = hoard / "hud" / "zip-cd-2025-q2.json"
dest.parent.mkdir(parents=True)
dest.write_bytes((FX / "hud_zip_cd.json").read_bytes())
res = runner.invoke(
app,
[
"market",
"hud",
"--year",
"2025",
"--quarter",
"2",
"--congress",
"119",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "stored: 2 zip rows" in res.output
assert seen == []
assert published == [True]
assert {r.method for r in T.read_zip_districts(c)} == {"hud-res", "hud-bus"}
def test_footprint_cli_missing_crosswalk_says_run_districts(env):
(
_,
_,
published,
_,
) = env
res = runner.invoke(
app, ["market", "footprint", "--family", "HOME-VISITS", "--write"]
)
assert res.exit_code == 0, res.output
assert "stack market districts" in res.output
assert published == []
def test_footprint_cli_entities_no_write(env, monkeypatch):
from market import footprint as footprint_api
c, _, published, _ = env
c.execute("CREATE SCHEMA IF NOT EXISTS cms")
c.execute(
"CREATE TABLE cms.home_health_agency_enrollments ("
"enrollment_id VARCHAR, organization_name VARCHAR, zip_code VARCHAR, "
"state VARCHAR)"
)
c.execute(
"INSERT INTO cms.home_health_agency_enrollments VALUES "
"('E1', 'Amedisys Home Health', '36009', 'AL'), "
"('E2', 'Amedisys Hospice', '48201', 'MI')"
)
T.write_entities(
c,
[T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")],
)
T.write_zip_districts(
c,
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
)
monkeypatch.setattr(
footprint_api,
"owner_files",
lambda: [FX / "hha_all_owners_head.csv"],
)
monkeypatch.setattr(footprint_api, "org_provider_rows", lambda con, entities: [])
res = runner.invoke(app, ["market", "footprint", "--entities", "--congress", "119"])
assert res.exit_code == 0, res.output
assert "2 sites" in res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.entity_site") == 0
assert published == []
def test_footprint_cli_family_offline_write(env, monkeypatch):
from market import footprint as footprint_api
c, _, published, _ = env
T.write_zip_districts(
c,
[
T.ZipDistrictRow(
119, "36009", "AL-01", 0.75, "census-zcta-land", "2020", "c"
),
T.ZipDistrictRow(
119, "36009", "AL-02", 0.25, "census-zcta-land", "2020", "c"
),
],
)
monkeypatch.setattr(
footprint_api, "prov_svc_file", lambda: FX / "prov_svc_head.csv"
)
monkeypatch.setattr(footprint_api, "family_codes", lambda con, key: ["99343"])
res = runner.invoke(
app,
[
"market",
"footprint",
"--family",
"HOME-VISITS",
"--congress",
"119",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "stored:" in res.output
rows = T.read_footprints(c)
assert {r.district_id for r in rows} >= {"AL-01", "AL-02", "??"}
assert published == [True]
def test_district_view_cli_prints(env):
c, _, _, _ = env
T.write_members(
c,
[
T.MemberRow(
119,
"T000481",
"Tlaib, Rashida",
"Democratic",
"MI",
"MI-12",
"house",
"",
"congress",
)
],
)
T.write_assignments(
c,
[
T.AssignmentRow(
119, date(2025, 4, 1), "HSWM02", "T000481", 1, "", "D", "sha"
)
],
)
T.write_footprints(
c,
[
T.FootprintRow(
"family",
"HOME-VISITS",
119,
"MI-12",
2024,
1,
0,
12,
75,
"census-zcta-land",
"prov-svc",
)
],
)
res = runner.invoke(
app,
[
"market",
"district-view",
"--family",
"HOME-VISITS",
"--congress",
"119",
"--top",
"10",
],
)
assert res.exit_code == 0, res.output
assert "MI-12" in res.output
assert "Tlaib, Rashida" in res.output
assert "HSWM02" in res.output
def test_footprint_cli_rejects_both_switches(env):
res = runner.invoke(
app, ["market", "footprint", "--entities", "--family", "HOME-VISITS"]
)
assert res.exit_code == 2
assert "pass exactly one of --entities or --family" in res.output
def test_footprint_cli_family_without_codes_writes_nothing(env, monkeypatch):
from market import footprint as footprint_api
c, _, published, _ = env
T.write_zip_districts(
c,
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
)
monkeypatch.setattr(footprint_api, "family_codes", lambda con, key: [])
res = runner.invoke(
app, ["market", "footprint", "--family", "HOME-VISITS", "--write"]
)
assert res.exit_code == 0, res.output
assert "no codes for family HOME-VISITS" in res.output
assert published == []
def test_footprint_cli_entities_write(env, monkeypatch):
from market import footprint as footprint_api
c, _, published, _ = env
c.execute("CREATE SCHEMA IF NOT EXISTS cms")
c.execute(
"CREATE TABLE cms.home_health_agency_enrollments ("
"enrollment_id VARCHAR, organization_name VARCHAR, zip_code VARCHAR, "
"state VARCHAR)"
)
c.execute(
"INSERT INTO cms.home_health_agency_enrollments VALUES "
"('E1', 'Amedisys Home Health', '36009', 'AL'), "
"('E2', 'Amedisys Hospice', '48201', 'MI')"
)
T.write_entities(
c,
[T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")],
)
T.write_zip_districts(
c,
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
)
monkeypatch.setattr(
footprint_api, "owner_files", lambda: [FX / "hha_all_owners_head.csv"]
)
monkeypatch.setattr(footprint_api, "org_provider_rows", lambda con, entities: [])
res = runner.invoke(
app, ["market", "footprint", "--entities", "--congress", "119", "--write"]
)
assert res.exit_code == 0, res.output
assert "stored:" in res.output
assert _count(c, "market.entity_site") == 2
assert {r.district_id for r in T.read_footprints(c)} >= {"AL-01", "??"}
assert published == [True]
def test_footprint_cli_counts_an_npi_site(env, monkeypatch):
from market import footprint as footprint_api
c, _, published, _ = env
T.write_entities(
c,
[T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")],
)
T.write_zip_districts(
c,
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
)
monkeypatch.setattr(footprint_api, "owner_files", lambda: [])
monkeypatch.setattr(
footprint_api,
"org_provider_rows",
lambda con, entities: [
{
"npi": "1234567890",
"entity_type_code": "2",
"provider_organization_name": "Clinic",
"parent_organization_name": "AMEDISYS HOLDING LLC",
"practice_zip_code": "36009",
"practice_state": "AL",
}
],
)
res = runner.invoke(app, ["market", "footprint", "--entities", "--congress", "119"])
assert res.exit_code == 0, res.output
assert "1 sites" in res.output
assert published == []
def test_footprint_cli_empty_family_reports_zero_unmatched(env, monkeypatch):
from market import footprint as footprint_api
c, _, published, _ = env
T.write_zip_districts(
c,
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
)
monkeypatch.setattr(footprint_api, "family_codes", lambda con, key: ["99343"])
monkeypatch.setattr(footprint_api, "family_billers", lambda con, path, codes: [])
res = runner.invoke(app, ["market", "footprint", "--family", "HOME-VISITS"])
assert res.exit_code == 0, res.output
assert "0 billers" in res.output
assert "unmatched ??: 0" in res.output
assert published == []
def test_district_view_cli_empty(env):
c, _, _, _ = env
T.write_footprints(c, [])
res = runner.invoke(app, ["market", "district-view", "--family", "HOME-VISITS"])
assert res.exit_code == 0, res.output
assert "no footprint stored" in res.output
def _events_build():
from market.events import MarketEvent, Tenure
event = MarketEvent(
"rule-final:2022-23873",
"rule-final",
date(2022, 11, 2),
True,
"day",
"program",
"PFS",
"PETM225D",
0,
"2022-23873",
"2022-11-02T16:15:00.000-04:00",
"",
)
family = MarketEvent(
"rule-final:2022-23873",
"rule-final",
date(2022, 11, 2),
True,
"day",
"family",
"HOME-VISITS",
"PETM225D",
0,
"2022-23873",
"2022-11-02T16:15:00.000-04:00",
"",
)
tenure = Tenure(
"RUC",
"FAMILY PRACTICE",
("American Academy of Family Physicians",),
("permanent",),
2019,
2026,
True,
True,
)
return [event, family], 4, [tenure]
def test_events_cli_offline_write(env, monkeypatch):
from market import events as events_api
c, _, published, _ = env
monkeypatch.setattr(events_api, "build", lambda *a, **k: _events_build())
res = runner.invoke(app, ["market", "events", "--write"])
assert res.exit_code == 0, res.output
assert "skipped rule: 4" in res.output
assert "stored:" in res.output
assert "FAMILY PRACTICE" in res.output
assert _count(c, "market.event") == 2
assert published == [True]
def test_events_cli_no_write_prints_counts(env, monkeypatch):
from market import events as events_api
c, _, published, _ = env
monkeypatch.setattr(events_api, "build", lambda *a, **k: _events_build())
res = runner.invoke(app, ["market", "events"])
assert res.exit_code == 0, res.output
assert "skipped rule: 4" in res.output
assert "2 events" in res.output
assert "HOME-VISITS" in res.output
assert "dry run: pass --write" in res.output
assert published == []
def test_events_cli_ama_refusal_gap(env, monkeypatch):
from market import events as events_api
c, _, published, _ = env
monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, []))
monkeypatch.setattr(
events_api,
"fetch_ama",
lambda *a, **k: (_ for _ in ()).throw(http.Refused("ama", 403, "denied")),
)
res = runner.invoke(
app,
[
"market",
"events",
"--ama-url",
"https://www.ama-assn.org/ruc-meetings",
"--body",
"RUC",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "gap: ama:events:" in res.output
gaps = T.read_gaps(c)
assert any(g.source == "ama:events" for g in gaps)
assert published == [True]
def test_events_cli_ama_schedule_write(env, monkeypatch):
from market import events as events_api
c, _, published, _ = env
monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, []))
monkeypatch.setattr(
events_api,
"fetch_ama",
lambda *a, **k: "The RUC meets April 24–27, 2024.",
)
res = runner.invoke(
app,
[
"market",
"events",
"--ama-url",
"https://www.ama-assn.org/ruc-meetings",
"--body",
"RUC",
"--write",
],
)
assert res.exit_code == 0, res.output
rows = T.read_events(c)
assert any(r.event_id == "ruc-meeting:2024-04-24" for r in rows)
assert published == [True]
def test_events_cli_ama_calendar_stores_both_bodies(env, monkeypatch):
from market import events as events_api
c, _, published, _ = env
page = (
"CPT Editorial Panel meetings\nMay 6-8, 2027\n"
"AMA/Specialty Society RVS Update Committee meetings\n"
"Jan. 20-23, 2027\n"
)
monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, []))
monkeypatch.setattr(events_api, "fetch_ama", lambda *a, **k: page)
res = runner.invoke(
app,
[
"market",
"events",
"--ama-url",
"https://www.ama-assn.org/about/cpt-editorial-panel/cpt-editorial-panel-ruc-meetings-calendar",
"--body",
"RUC",
"--write",
],
)
assert res.exit_code == 0, res.output
ids = {row.event_id for row in T.read_events(c)}
assert ids == {"ruc-meeting:2027-01-20", "panel-meeting:2027-05-06"}
assert published == [True]
def test_events_cli_ama_needs_a_body(env):
res = runner.invoke(
app, ["market", "events", "--ama-url", "https://example.test/ruc"]
)
assert res.exit_code == 2
assert "pass --body RUC or CPT_PANEL" in res.output
def test_events_cli_ama_page_is_a_refusal(env, monkeypatch):
from market import events as events_api
c, _, published, _ = env
monkeypatch.setattr(events_api, "build", lambda *a, **k: ([], 0, []))
monkeypatch.setattr(events_api, "fetch_ama", lambda *a, **k: "Just a moment")
res = runner.invoke(
app,
[
"market",
"events",
"--ama-url",
"https://example.test/ruc",
"--body",
"RUC",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "gap: ama:events: AMA page is a refusal" in res.output
assert any(g.source == "ama:events" for g in T.read_gaps(c))
assert published == [True]
def test_district_view_cli_needs_a_key(env):
res = runner.invoke(app, ["market", "district-view"])
assert res.exit_code == 2
assert "pass exactly one of --entity or --family" in res.output
def test_committees_cli_from_after_to(congress):
res = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "113"])
assert res.exit_code == 2
assert "--from 119 is after --to 113" in res.output