Files
stack/tests/cli/test_market_cli.py

1007 lines
35 KiB
Python

"""stack market — entities, gaps."""
from __future__ import annotations
import json
import re
from datetime import date
from pathlib import Path
import duckdb
import httpx
import pytest
from typer.testing import CliRunner
import cli.market as mk
from bib.store import Store
from cli import app
from market import edgar, http
from market import tables as T
from market.seed import SEAT_LABEL_KEYS, load_seed
from pfs.codetables import SeatRow, write_seats
runner = CliRunner()
FX = Path(__file__).parent.parent / "market" / "fixtures"
class _Pacer:
def wait(self):
pass
def _seat(org, specialty, kind="specialty", year=2024):
return SeatRow("RUC", year, org, specialty, "permanent", "u", "K", "s", kind)
def _specialties() -> set[str]:
companies, orgs = load_seed()
out = {o.specialty for o in orgs if o.specialty}
out |= {
link.target
for e in (*companies, *orgs)
for link in e.links
if link.scope == "specialty"
}
return out - set(SEAT_LABEL_KEYS)
def _build_db() -> duckdb.DuckDBPyConnection:
c = duckdb.connect(":memory:")
c.execute("CREATE SCHEMA terminology")
c.execute(
"CREATE TABLE terminology.cms_acceptable_provider_specialty_codes "
"(specialty_description VARCHAR)"
)
c.executemany(
"INSERT INTO terminology.cms_acceptable_provider_specialty_codes VALUES (?)",
[(s.title(),) for s in sorted(_specialties())],
)
c.execute("CREATE SCHEMA pfs")
c.execute("CREATE TABLE pfs.cpt_code (code VARCHAR, descriptor VARCHAR)")
c.execute(
"INSERT INTO pfs.cpt_code VALUES ('99999', 'a licensed descriptor that no "
"filing quote should ever copy word for word')"
)
write_seats(
c,
"RUC",
[
_seat("Family Medicine", "FAMILY PRACTICE"),
_seat("Cardiology", "CARDIOLOGY"),
_seat("RUC Chair", "", "category"),
],
)
return c
class _Batch:
def __init__(self, c):
self.c = c
def __enter__(self):
return self.c
def __exit__(self, *a):
return False
@pytest.fixture
def env(monkeypatch, tmp_path):
c = _build_db()
monkeypatch.setattr(mk, "_batch", lambda: _Batch(c))
monkeypatch.setattr(mk, "_read", lambda: c.cursor())
store = Store(":memory:", storage_dir=tmp_path / "st")
monkeypatch.setattr(mk, "_store", lambda: store)
published: list[bool] = []
monkeypatch.setattr(mk, "_publish", lambda: published.append(True))
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
monkeypatch.setattr(edgar, "new_pacer", lambda: _Pacer())
try:
yield c, store, published, tmp_path / "market"
finally:
store.close()
c.close()
def _cache_offline(hoard: Path) -> None:
sec = hoard / "sec"
(sec / "submissions").mkdir(parents=True)
(sec / "company_tickers.json").write_bytes(
(FX / "company_tickers.json").read_bytes()
)
for name in (
"submissions_CIK0001564406.json",
"CIK0001564406-submissions-001.json",
):
target = name.replace("submissions_", "")
if name.startswith("submissions_"):
target = "CIK0001564406.json"
(sec / "submissions" / target).write_bytes((FX / name).read_bytes())
_cache_filings(hoard)
def _cache_filings(hoard: Path, companies=None) -> None:
"""A hoarded filing per cited accession, holding exactly the quotes the
seed cites to it."""
import html
companies = companies or load_seed()[0]
quotes: dict[tuple[str, str], list[str]] = {}
for _who, c, cite in edgar.filing_cites(companies):
if cite.adsh:
quotes.setdefault((c.cik, cite.adsh), []).append(cite.quote)
for (cik, adsh), qs in quotes.items():
p = hoard / "sec" / "filings" / cik / f"{adsh}.htm"
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(
"<html><body>"
+ "".join(f"<p>{html.escape(q)}</p>" for q in qs)
+ "</body></html>"
)
def _count(c, table):
return c.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
def test_entities_offline_writes_and_logs(env):
c, store, published, hoard = env
_cache_offline(hoard)
# a stale curated link the seed no longer carries
T.write_links(
c, [T.LinkRow("0001564406", "program", "QPP-MIPS", "curated", "x", "url", "u")]
)
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 0, res.output
companies, orgs = load_seed()
assert _count(c, "market.entity") == len(companies) + len(orgs)
osh = next(e for e in T.read_entities(c, "company") if e.key == "0001564406")
assert osh.status == "acquired" and osh.ended_on == date(2023, 5, 2)
assert osh.sic == "8000" and osh.item_key
assert "no submissions for AMED" in res.output
# the five fixture filings, and the seed's dated ticker
assert len(T.read_filings(c, "0001564406")) == 5
assert [s.source for s in T.read_securities(c, "0001564406")] == ["seed"]
links = T.read_links(c, basis="curated")
assert all(l.item_key for l in links)
assert ("program", "QPP-MIPS") not in {(l.scope, l.target) for l in links}
assert [l.target for l in T.read_links(c, basis="sic")] == ["8000"]
logs = c.execute(
"SELECT table_name, rows FROM cms.ingest_log WHERE module = 'market.edgar' "
"ORDER BY table_name"
).fetchall()
assert [t for t, _ in logs] == [
"market.entity",
"market.entity_link",
"market.filing",
"market.security",
]
assert published == [True]
assert "retired 1" in res.output
assert store.count() > 0
def test_entities_rerun_is_idempotent(env):
c, _, published, hoard = env
_cache_offline(hoard)
for _ in range(2):
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
companies, orgs = load_seed()
assert _count(c, "market.entity") == len(companies) + len(orgs)
assert _count(c, "market.filing") == 5
def test_entities_no_write_prints_counts(env, monkeypatch):
c, _, published, hoard = env
_cache_offline(hoard)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 0, res.output
companies, orgs = load_seed()
assert f"entities {len(companies) + len(orgs)}" in res.output
assert (
"company/acquired" in res.output
and "organization/specialty_society" in res.output
)
assert "dry run" in res.output and published == []
assert "warning" not in res.output # every seat specialty has a society
def test_entities_warns_about_uncovered_seat_specialty(env):
c, _, _, hoard = env
_cache_offline(hoard)
write_seats(c, "RUC", [_seat("Podiatry", "PODIATRY", year=2025)])
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 0
assert "seat specialties without a society: ['PODIATRY']" in res.output
def _handler(routes):
def handler(request: httpx.Request):
url = str(request.url)
for pattern, make in routes:
m = re.search(pattern, url)
if m:
return make(m)
return httpx.Response(404)
return handler
def _online(monkeypatch, routes):
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(_handler(routes))
),
)
def test_entities_refusal_prints_gap(env, monkeypatch):
c, _, published, hoard = env
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
res = runner.invoke(app, ["market", "entities", "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:entities:" in res.output and "403" in res.output
assert _count(c, "market.entity") == 0 and _count(c, "market.filing") == 0
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:entities"]
assert published == [True]
def test_entities_refusal_without_write_records_nothing(env, monkeypatch):
c, _, published, _ = env
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "entities"])
assert res.exit_code == 0 and "gap: sec-edgar:entities:" in res.output
assert published == []
def _sub_for(cik: str) -> dict:
return {
"cik": cik,
"name": f"Filer {cik}",
"sic": "8082",
"sicDescription": "Home Health",
"tickers": [],
"formerNames": [],
"filings": {
"recent": {
"accessionNumber": ["A-1", "A-2"],
"filingDate": ["2024-01-01", "2024-01-02"],
"reportDate": ["", ""],
"acceptanceDateTime": ["", ""],
"form": ["10-K", "4"],
"items": ["", ""],
"primaryDocument": ["a.htm", "b.xml"],
},
"files": [],
},
}
def test_sic_sweep_registers_filers_and_sic_links(env, monkeypatch):
c, _, published, hoard = env
_cache_filings(hoard)
atom = (FX / "sic_8082.atom").read_bytes().replace(b"0001468328", b"0000000042")
seen: list[str] = []
def sub(m):
seen.append(m.group(1))
cik = m.group(1)
if cik == "0001564406":
return httpx.Response(
200, content=(FX / "submissions_CIK0001564406.json").read_bytes()
)
if cik == "0000000042":
return httpx.Response(404) # no submissions file: skipped
return httpx.Response(200, json=_sub_for(cik))
def page(m):
return httpx.Response(
200, content=(FX / "CIK0001564406-submissions-001.json").read_bytes()
)
_online(
monkeypatch,
[
(
r"company_tickers\.json",
lambda m: httpx.Response(
200, content=(FX / "company_tickers.json").read_bytes()
),
),
(r"SIC=\d+.*start=0", lambda m: httpx.Response(200, content=atom)),
(r"submissions-001\.json", page),
(r"CIK(\d{10})\.json", sub),
],
)
res = runner.invoke(app, ["market", "entities", "--sic-sweep", "--write"])
assert res.exit_code == 0, res.output
assert res.output.count("SIC ") == len(edgar.HEALTH_SICS)
keys = {e.key for e in T.read_entities(c, "company")}
assert {"0001725255", "0001468328"} <= keys and "0000000042" not in keys
# a sweep filer keeps only the sweep forms; the seed's filings keep everything
assert [f.form for f in T.read_filings(c, "0001725255")] == ["10-K"]
assert len(T.read_filings(c, "0001468328")) == 2
assert len(T.read_filings(c, "0001564406")) == 5
sic = T.read_links(c, basis="sic")
assert {l.target for l in sic} == {"8082", "8000"}
assert any(l.entity_key == "0001725255" for l in sic)
def test_entities_seed_errors_abort(env, monkeypatch):
c, _, published, _ = env
companies, orgs = load_seed()
bad = companies[0].__class__(
**{
**companies[0].__dict__,
"links": (
companies[0].links[0].__class__("family", "NOPE", companies[0].cite),
),
}
)
monkeypatch.setattr(mk, "load_seed", lambda: ((bad, *companies[1:]), orgs))
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 1 and "seed error: OSH" in res.output
assert c.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
).fetchone() == (0,)
assert published == []
def test_entities_missing_vocabulary_tables_abort(env, monkeypatch):
bare = duckdb.connect(":memory:")
monkeypatch.setattr(mk, "_read", lambda: bare)
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 1 and "cannot validate the seed" in res.output
bare.close()
def test_entities_other_read_errors_propagate(env, monkeypatch):
def boom():
raise RuntimeError("disk on fire")
monkeypatch.setattr(mk, "_read", boom)
res = runner.invoke(app, ["market", "entities", "--offline"])
assert isinstance(res.exception, RuntimeError)
def test_require_key_missing_writes_only_gap(env, monkeypatch):
c, _, published, _ = env
monkeypatch.delenv("TIINGO_API_KEY", raising=False)
assert mk._require_key("tiingo", "tiingo:prices", write=True) is None
assert [(g.source, g.note) for g in T.read_gaps(c)] == [
("tiingo:prices", "TIINGO_API_KEY not in .env")
]
assert all(_count(c, f"market.{t}") == 0 for t in ("entity", "security", "filing"))
assert published == [True]
def test_require_key_present_returns_it(env, monkeypatch):
monkeypatch.setenv("TIINGO_API_KEY", "abc")
assert mk._require_key("tiingo", "tiingo:prices", write=True) == "abc"
def test_require_key_missing_prints_gap_without_write(env, monkeypatch, capsys):
monkeypatch.delenv("HUD_API_TOKEN", raising=False)
assert mk._require_key("hud", "hud:hud", write=False) is None
assert "gap: hud:hud: HUD_API_TOKEN not in .env" in capsys.readouterr().out
c, _, published, _ = env
assert published == []
def test_gap_cleared_after_success(env):
c, _, _, hoard = env
_cache_offline(hoard)
T.write_gap(c, "sec-edgar:entities", "403")
T.write_gap(c, "tiingo:prices", "no key")
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
assert {g.source for g in T.read_gaps(c)} == {
"tiingo:prices",
"web:renalmd.org",
"web:asco.org",
"web:ahip.org",
}
def test_gaps_command_lists_rows(env):
c, _, _, _ = env
T.write_gap(c, "tiingo:prices", "TIINGO_API_KEY not in .env")
res = runner.invoke(app, ["market", "gaps"])
assert res.exit_code == 0
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
def test_gaps_command_empty_and_missing_tables(env, monkeypatch):
c, _, _, _ = env
T.ensure_tables(c)
assert "no gaps recorded" in runner.invoke(app, ["market", "gaps"]).output
bare = duckdb.connect(":memory:")
monkeypatch.setattr(mk, "_read", lambda: bare)
assert "no market tables yet" in runner.invoke(app, ["market", "gaps"]).output
class _Broken:
def execute(self, *a, **k):
raise RuntimeError("x")
def close(self):
pass
monkeypatch.setattr(mk, "_read", lambda: _Broken())
assert isinstance(runner.invoke(app, ["market", "gaps"]).exception, RuntimeError)
bare.close()
def test_default_indirections_resolve_lazily(monkeypatch):
import conf.connect as cc
calls: list[str] = []
monkeypatch.setattr(cc, "duckdb_batch", lambda n: calls.append(f"batch {n}") or n)
monkeypatch.setattr(
cc, "duckdb", lambda n, read_only=True: calls.append(f"read {n}") or n
)
monkeypatch.setattr(cc, "bib", lambda: calls.append("bib") or "store")
monkeypatch.setattr(
cc, "publish_replica", lambda n: calls.append(f"pub {n}") or "p"
)
assert mk._batch() == "aco" and mk._read() == "aco" and mk._store() == "store"
mk._publish()
assert calls == ["batch aco", "read aco", "bib", "pub aco"]
def test_entities_aborts_when_a_quote_is_not_in_the_filing(env):
c, _, published, hoard = env
_cache_offline(hoard)
victim = next(hoard.glob("sec/filings/*/*.htm"))
victim.write_text("<p>nothing the seed quotes</p>")
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 1 and "quote error:" in res.output
assert published == []
assert c.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
).fetchone() == (0,)
def test_entities_aborts_when_a_cited_filing_is_not_hoarded(env):
c, _, published, hoard = env
_cache_offline(hoard)
next(hoard.glob("sec/filings/*/*.htm")).unlink()
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 1 and "not hoarded" in res.output
def test_entities_fetches_cited_filings_online(env, monkeypatch):
c, _, _, hoard = env
_cache_offline(hoard)
victim = next(hoard.glob("sec/filings/*/*.htm"))
cik, adsh = victim.parent.name, victim.stem
body = victim.read_bytes()
victim.unlink()
(hoard / "sec" / "submissions" / f"CIK{cik}.json").write_text(
json.dumps(
_sub_for(cik)
| {
"filings": {
"recent": {
**_sub_for(cik)["filings"]["recent"],
"accessionNumber": [adsh, "Z"],
"primaryDocument": ["doc.htm", "z.htm"],
},
"files": [],
}
}
)
)
seen = []
def serve(m):
seen.append(m.group(0))
return httpx.Response(200, content=body)
_online(monkeypatch, [(r"/Archives/edgar/data/\d+/\d+/doc\.htm", serve)])
res = runner.invoke(app, ["market", "entities"])
assert res.exit_code == 0, res.output
assert len(seen) == 1 and victim.exists()
def test_entities_manifest_and_ingest_log_sources(env):
c, _, _, hoard = env
_cache_offline(hoard)
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
manifest = hoard / "MANIFEST.jsonl"
assert manifest.exists() and manifest.read_text().count("\n") >= 6
rows = c.execute(
"SELECT table_name, source_file, sha256 FROM cms.ingest_log WHERE module = 'market.edgar'"
).fetchall()
assert len(rows) == 4 and all(sf and sha and len(sha) == 64 for _, sf, sha in rows)
def test_entities_records_unreachable_site_gaps_and_pseudo_orgs(env):
c, _, _, hoard = env
_cache_offline(hoard)
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
gaps = {g.source for g in T.read_gaps(c)}
assert {"web:renalmd.org", "web:asco.org", "web:ahip.org"} <= gaps
assert "sec-edgar:entities" not in gaps
pseudo = {e.key for e in T.read_entities(c, "organization") if e.pseudo}
assert pseudo == {"PRIMARY-CARE-SEAT", "ONCOLOGY-SEAT"}
def test_entities_retires_organizations_the_seed_dropped(env):
c, _, _, hoard = env
_cache_offline(hoard)
T.write_entities(c, [T.EntityRow("ASCO", "ASCO", "organization")])
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
assert "ASCO" not in {e.key for e in T.read_entities(c, "organization")}
def test_entities_aborts_when_a_quote_copies_a_cpt_descriptor(env):
c, _, published, hoard = env
_cache_offline(hoard)
victim = load_seed()[0][0].cite.quote
c.execute("DELETE FROM pfs.cpt_code")
c.execute("INSERT INTO pfs.cpt_code VALUES ('1', ?)", [victim])
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 1 and "ama error: OSH" in res.output
assert published == []
assert c.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
).fetchone() == (0,)
def test_entities_stores_ticker_history_and_link_dates(env):
c, _, _, hoard = env
_cache_offline(hoard)
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
assert [
(s.ticker, s.valid_to, s.priced) for s in T.read_securities(c, "0001156039")
] == [("ANTM", date(2022, 6, 27), False), ("ELV", None, True)]
assert [s.priced for s in T.read_securities(c, "0001739940")] == [False]
(clov,) = [
l
for l in T.read_links(c, basis="curated", target="APM")
if l.entity_key == "0001801170"
]
assert clov.valid_to == date(2023, 12, 31)
# ── stack market prices ──────────────────────────────────────────────
import io # noqa: E402
import zipfile # noqa: E402
from market import tiingo # noqa: E402
DAY = date(2026, 11, 12)
def _supported_zip(*rows: str) -> bytes:
head = "ticker,exchange,assetType,priceCurrency,startDate,endDate\n"
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as z:
z.writestr("supported_tickers.csv", head + "\n".join(rows) + "\n")
return buf.getvalue()
def _eod(day: str, close: float, adj: float | None = None) -> dict:
return {
"date": f"{day}T00:00:00.000Z",
"open": close,
"high": close,
"low": close,
"close": close,
"volume": 5,
"adjClose": close if adj is None else adj,
"divCash": 0.0,
"splitFactor": 1.0,
}
@pytest.fixture
def px(env, monkeypatch, tmp_path):
c, store, published, hoard = env
monkeypatch.setenv("TIINGO_API_KEY", "secret-key")
monkeypatch.setattr(tiingo, "state_dir", lambda: tmp_path / "state")
monkeypatch.setattr(tiingo, "new_pacer", lambda: _Pacer())
monkeypatch.setattr(tiingo, "today", lambda: DAY)
seen: list[httpx.Request] = []
series = {
"AAA": [
_eod("2024-01-02", 100),
_eod("2024-01-03", 110),
_eod("2024-01-04", 99),
],
"BBB": [_eod("2024-01-02", 10), _eod("2024-01-03", 11)],
"SPY": [_eod("2024-01-02", 400), _eod("2024-01-03", 404)],
"XLV": [_eod("2024-01-02", 100)],
"IHF": [_eod("2024-01-02", 50)],
}
z = _supported_zip(
"AAA,NYSE,Stock,USD,2000-01-01,2026-11-12",
"BBB,NYSE,Stock,USD,2000-01-01,2026-11-12",
"SPY,NYSE,ETF,USD,1993-01-29,2026-11-12",
"XLV,NYSE,ETF,USD,1998-12-22,2026-11-12",
"IHF,NYSE,ETF,USD,2006-05-05,2026-11-12",
"ZZZ,NYSE,Stock,USD,2000-01-01,2026-11-12",
)
def prices_for(m):
if m.group(1) not in series:
return httpx.Response(404, json={"detail": "Not found."})
return httpx.Response(200, json=series[m.group(1)])
routes = [
(r"supported_tickers\.zip", lambda m: httpx.Response(200, content=z)),
(r"/daily/(\w+)/prices", prices_for),
]
def handler(request):
seen.append(request)
return _handler(routes)(request)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(handler),
headers={"Authorization": f"Token {key}"} if key else {},
),
)
return c, store, published, hoard, seen, series, tmp_path / "state"
def _price_requests(seen):
return [r for r in seen if "/prices" in r.url.path]
def test_prices_cli_fetch_write(px):
c, _, published, hoard, seen, _, state = px
res = runner.invoke(
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
)
assert res.exit_code == 0, res.output
assert _count(c, "market.price_daily") == 5
assert [r.url.params["startDate"] for r in _price_requests(seen)] == [
"1993-01-29"
] * 2
assert all("secret-key" not in str(r.url) for r in seen)
keyed = [r for r in seen if "/prices" in r.url.path]
assert all(r.headers["authorization"] == "Token secret-key" for r in keyed)
free = [r for r in seen if "supported" in r.url.path]
assert free and "authorization" not in free[0].headers
assert {
(t, n)
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
} == {("market.price_daily", 5), ("market.series", 2)}
assert [s.value for s in T.read_series(c, ["AAA"])] == [pytest.approx(99.0)]
assert published == [True]
used = json.loads((state / "tiingo-2026-11.json").read_text())
assert used["symbols"] == ["AAA", "BBB"] and used["day_counts"] == {"2026-11-12": 2}
assert (hoard / "tiingo" / "eod" / "AAA" / "1993-01-29_2026-11-12.json").is_file()
assert "priced 2 symbols, 5 rows" in res.output and "2/450" in res.output
assert T.read_gaps(c) == []
def test_prices_incremental_pull_and_same_day_skip(px, monkeypatch):
c, _, _, _, seen, series, _ = px
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
assert runner.invoke(app, cmd).exit_code == 0
before = len(_price_requests(seen))
again = runner.invoke(app, cmd)
assert len(_price_requests(seen)) == before # priced today: nothing to do
assert "priced 0 symbols" in again.output
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
series["AAA"] = [_eod("2024-01-04", 99, 49.5), _eod("2024-01-05", 108.9, 54.45)]
assert runner.invoke(app, cmd).exit_code == 0
last = _price_requests(seen)[-1]
assert last.url.params["startDate"] == "2024-01-04"
rows = {r.day: r for r in T.read_prices(c, "AAA")}
assert len(rows) == 4 and rows[date(2024, 1, 4)].adj_close == 99
assert rows[date(2024, 1, 5)].ret == pytest.approx(0.1)
assert [m.value for m in T.read_series(c, ["AAA"])] == [pytest.approx(108.9)]
def test_prices_benchmarks_start_at_listing(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(app, ["market", "prices", "--benchmarks", "--write"])
assert res.exit_code == 0, res.output
starts = {
r.url.path.split("/")[3]: r.url.params["startDate"]
for r in _price_requests(seen)
}
assert starts == {"SPY": "1993-01-29", "XLV": "1998-12-22", "IHF": "2006-05-05"}
def test_prices_dry_run_reports_budget(px, monkeypatch):
c, _, published, _, seen, _, state = px
T.ensure_tables(c)
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
T.write_links(c, [T.LinkRow("K1", "program", "PFS", "fts", "mention", "adsh", "a")])
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
assert res.exit_code == 0, res.output
assert "1/450 symbols used, 449 remaining" in res.output
assert "1. SPY" in res.output and "used" in res.output
assert "AAA" in res.output and "K1" in res.output and "first pull" in res.output
assert _price_requests(seen) == [] and published == []
assert not (state / "tiingo-2026-11.json").read_text().count("AAA")
def test_prices_dry_run_without_market_tables_lists_benchmarks(px):
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
assert res.exit_code == 0, res.output
assert "3 symbols ranked" in res.output
def test_prices_dry_run_shows_stored_span(px):
c, *_ = px
T.ensure_tables(c)
T.write_prices(
c,
"AAA",
tiingo.to_prices(
"AAA", [_eod("2024-01-02", 1)], pulled_on=DAY, source_file="f", overlap=None
),
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--dry-run"])
assert "stored to 2024-01-02" in res.output
def test_prices_without_key_records_gap(px, monkeypatch):
c, _, published, _, seen, _, _ = px
monkeypatch.delenv("TIINGO_API_KEY")
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"]
assert _count(c, "market.price_daily") == 0 and seen == [] and published == [True]
def test_prices_refusal_prints_gap(px, monkeypatch):
c, _, published, _, _, _, _ = px
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(lambda r: httpx.Response(403))
),
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "gap: tiingo:prices:" in res.output and "403" in res.output
assert _count(c, "market.price_daily") == 0
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"] and published == [
True
]
def test_prices_refusal_on_prices_after_supported_ok(px, monkeypatch):
c, _, published, _, _, _, _ = px
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
routes = [
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
(r"/prices", lambda m: httpx.Response(401)),
]
_online(monkeypatch, routes)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA"])
assert (
res.exit_code == 0
and "gap: tiingo:prices:" in res.output
and "401" in res.output
)
assert published == []
def test_prices_nothing_selected_exits(px):
res = runner.invoke(app, ["market", "prices"])
assert res.exit_code == 2 and "nothing to price" in res.output
def test_prices_unknown_ticker_and_window_gap_become_gap_note(px):
c, _, _, _, _, series, _ = px
T.ensure_tables(c)
T.write_securities(
c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, date(2030, 1, 1), "s")]
)
res = runner.invoke(
app, ["market", "prices", "--ticker", "BBB", "--ticker", "ZZZ", "--write"]
)
assert res.exit_code == 0, res.output
assert "ZZZ: not found at Tiingo" in res.output
(gap,) = T.read_gaps(c)
assert (
"ZZZ: not found at Tiingo" in gap.note
and "BBB: Tiingo ends 2026-11-12" in gap.note
)
# BBB later prices cleanly: its entry goes, ZZZ's stays
T.write_securities(c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, None, "s")])
runner.invoke(app, ["market", "prices", "--ticker", "BBB", "--refresh", "--write"])
(gap,) = T.read_gaps(c)
assert gap.note == "ZZZ: not found at Tiingo"
def test_prices_clean_run_clears_gap_and_window_clips(px):
c, _, _, _, _, _, _ = px
T.ensure_tables(c)
T.write_gap(c, "tiingo:prices", "AAA: not found at Tiingo")
T.write_securities(
c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", date(2024, 1, 3), None, "s")]
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert [r.day for r in T.read_prices(c, "AAA")] == [
date(2024, 1, 3),
date(2024, 1, 4),
]
assert T.read_gaps(c) == []
def test_prices_limit_stops_downloads(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(
app, ["market", "prices", "--benchmarks", "--limit", "1", "--write"]
)
assert res.exit_code == 0 and len(_price_requests(seen)) == 1
assert [r.ticker for r in [*T.read_prices(c, "SPY")][:1]] == ["SPY"]
def test_prices_budget_exhausted_stops_and_writes_what_it_has(px):
c, _, published, _, seen, _, state = px
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps(
{
"month": "2026-11",
"symbols": [f"S{i}" for i in range(449)] + ["AAA"],
"day_counts": {},
}
)
)
res = runner.invoke(
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
)
assert res.exit_code == 0, res.output
assert "budget: stopped, BBB" in res.output
assert [r.ticker for r in T.read_prices(c, "AAA")][:1] == ["AAA"]
assert T.read_prices(c, "BBB") == [] and published == [True]
def test_prices_without_write_stores_nothing(px, monkeypatch):
c, _, published, _, _, _, _ = px
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "prices", "--ticker", "ZZZ"])
assert res.exit_code == 0, res.output
assert (
"dry run: pass --write" in res.output
and "gap: tiingo:prices: ZZZ" in res.output
)
assert published == []
def test_merge_gap_keeps_other_tickers_and_orders():
assert mk._merge_gap("", {}, set()) == ""
assert mk._merge_gap("A: x | B: y", {"C": "z"}, {"A"}) == "B: y | C: z"
def test_or_empty_reraises_other_errors():
def boom(_):
raise RuntimeError("disk")
with pytest.raises(RuntimeError):
mk._or_empty(boom, None, {})
def test_prices_ticker_refuses_unpriced_record_without_a_request(px):
c, _, _, _, seen, _, state = px
T.ensure_tables(c)
T.write_securities(
c, "K9", [T.SecurityRow("K9", "AAA", "NYSE", None, None, "s", False)]
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "skipped AAA: symbol record only" in res.output
assert _price_requests(seen) == [] and _count(c, "market.price_daily") == 0
assert not (state / "tiingo-2026-11.json").exists()
assert T.read_gaps(c) == []
def test_prices_unsupported_named_ticker_is_gap_not_request(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(app, ["market", "prices", "--ticker", "QQQQ", "--write"])
assert "skipped QQQQ: not on Tiingo's supported list" in res.output
assert _price_requests(seen) == []
(gap,) = T.read_gaps(c)
assert gap.note == "QQQQ: not on Tiingo's supported list"
def test_prices_universe_records_unsupported_seed_ticker_gap(px):
c, _, _, _, seen, _, _ = px
T.ensure_tables(c)
T.write_securities(
c, "0001800682", [T.SecurityRow("0001800682", "CANO", "NYSE", None, None, "s")]
)
res = runner.invoke(app, ["market", "prices", "--universe", "--write"])
assert res.exit_code == 0, res.output
assert "skipped CANO: not on Tiingo's supported list" in res.output
assert all("/CANO/" not in r.url.path for r in seen)
(gap,) = T.read_gaps(c)
assert gap.source == "tiingo:prices"
assert gap.note == "CANO: not on Tiingo's supported list"
# rerun with everything already priced today keeps the gap
again = runner.invoke(app, ["market", "prices", "--universe", "--write"])
assert again.exit_code == 0
assert [g.note for g in T.read_gaps(c)] == [gap.note]
def test_prices_missing_overlap_day_repulls_further_back(px, monkeypatch):
c, _, _, _, seen, _, _ = px
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
assert runner.invoke(app, cmd).exit_code == 0
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
def eod(m):
# the 2024-01-04 overlap day is absent from the narrow pull
start = next(r for r in seen[::-1] if "/prices" in r.url.path).url.params[
"startDate"
]
if start == "2024-01-04":
return httpx.Response(200, json=[_eod("2024-01-05", 54.45)])
return httpx.Response(
200, json=[_eod("2024-01-03", 55), _eod("2024-01-05", 60.5)]
)
handler = _handler(
[
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
(r"/daily/AAA/prices", eod),
]
)
def record(request):
seen.append(request)
return handler(request)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(record)),
)
res = runner.invoke(app, cmd)
assert res.exit_code == 0, res.output
assert "overlap day 2024-01-04 missing" in res.output
assert "re-pulling from 2023-12-25" in res.output
row = {r.day: r for r in T.read_prices(c, "AAA")}[date(2024, 1, 5)]
assert row.ret == pytest.approx(60.5 / 55 - 1)