Some checks failed
CI / lint (push) Successful in 57s
CI / notebooks-smoke (push) Successful in 2m45s
Deploy / changes (push) Successful in 18s
Infra CI / notebooks (push) Successful in 1m14s
CI / test (push) Failing after 5m26s
Infra CI / zotero (push) Successful in 21s
Infra CI / docs (push) Successful in 1m35s
Infra CI / api (push) Successful in 1m22s
Infra CI / llm (push) Successful in 1m14s
Infra CI / mc (push) Successful in 36s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
1513 lines
54 KiB
Python
1513 lines
54 KiB
Python
"""stack market — entities, gaps."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
import duckdb
|
|
import httpx
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
import cli.market as mk
|
|
from bib.store import Store
|
|
from cli import app
|
|
from market import edgar, http
|
|
from market import tables as T
|
|
from market.seed import SEAT_LABEL_KEYS, load_seed
|
|
from pfs.codetables import SeatRow, write_seats
|
|
|
|
runner = CliRunner()
|
|
FX = Path(__file__).parent.parent / "market" / "fixtures"
|
|
|
|
|
|
class _Pacer:
|
|
def wait(self):
|
|
pass
|
|
|
|
|
|
def _seat(org, specialty, kind="specialty", year=2024):
|
|
return SeatRow("RUC", year, org, specialty, "permanent", "u", "K", "s", kind)
|
|
|
|
|
|
def _specialties() -> set[str]:
|
|
companies, orgs = load_seed()
|
|
out = {o.specialty for o in orgs if o.specialty}
|
|
out |= {
|
|
link.target
|
|
for e in (*companies, *orgs)
|
|
for link in e.links
|
|
if link.scope == "specialty"
|
|
}
|
|
return out - set(SEAT_LABEL_KEYS)
|
|
|
|
|
|
def _build_db() -> duckdb.DuckDBPyConnection:
|
|
c = duckdb.connect(":memory:")
|
|
c.execute("CREATE SCHEMA terminology")
|
|
c.execute(
|
|
"CREATE TABLE terminology.cms_acceptable_provider_specialty_codes "
|
|
"(specialty_description VARCHAR)"
|
|
)
|
|
c.executemany(
|
|
"INSERT INTO terminology.cms_acceptable_provider_specialty_codes VALUES (?)",
|
|
[(s.title(),) for s in sorted(_specialties())],
|
|
)
|
|
c.execute("CREATE SCHEMA pfs")
|
|
c.execute("CREATE TABLE pfs.cpt_code (code VARCHAR, descriptor VARCHAR)")
|
|
c.execute(
|
|
"INSERT INTO pfs.cpt_code VALUES ('99999', 'a licensed descriptor that no "
|
|
"filing quote should ever copy word for word')"
|
|
)
|
|
write_seats(
|
|
c,
|
|
"RUC",
|
|
[
|
|
_seat("Family Medicine", "FAMILY PRACTICE"),
|
|
_seat("Cardiology", "CARDIOLOGY"),
|
|
_seat("RUC Chair", "", "category"),
|
|
],
|
|
)
|
|
return c
|
|
|
|
|
|
class _Batch:
|
|
def __init__(self, c):
|
|
self.c = c
|
|
|
|
def __enter__(self):
|
|
return self.c
|
|
|
|
def __exit__(self, *a):
|
|
return False
|
|
|
|
|
|
@pytest.fixture
|
|
def env(monkeypatch, tmp_path):
|
|
c = _build_db()
|
|
monkeypatch.setattr(mk, "_batch", lambda: _Batch(c))
|
|
monkeypatch.setattr(mk, "_read", lambda: c.cursor())
|
|
store = Store(":memory:", storage_dir=tmp_path / "st")
|
|
monkeypatch.setattr(mk, "_store", lambda: store)
|
|
published: list[bool] = []
|
|
monkeypatch.setattr(mk, "_publish", lambda: published.append(True))
|
|
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
|
|
monkeypatch.setattr(edgar, "new_pacer", lambda: _Pacer())
|
|
try:
|
|
yield c, store, published, tmp_path / "market"
|
|
finally:
|
|
store.close()
|
|
c.close()
|
|
|
|
|
|
def _cache_offline(hoard: Path) -> None:
|
|
sec = hoard / "sec"
|
|
(sec / "submissions").mkdir(parents=True)
|
|
(sec / "company_tickers.json").write_bytes(
|
|
(FX / "company_tickers.json").read_bytes()
|
|
)
|
|
for name in (
|
|
"submissions_CIK0001564406.json",
|
|
"CIK0001564406-submissions-001.json",
|
|
):
|
|
target = name.replace("submissions_", "")
|
|
if name.startswith("submissions_"):
|
|
target = "CIK0001564406.json"
|
|
(sec / "submissions" / target).write_bytes((FX / name).read_bytes())
|
|
_cache_filings(hoard)
|
|
|
|
|
|
def _cache_filings(hoard: Path, companies=None) -> None:
|
|
"""A hoarded filing per cited accession, holding exactly the quotes the
|
|
seed cites to it."""
|
|
import html
|
|
|
|
companies = companies or load_seed()[0]
|
|
quotes: dict[tuple[str, str], list[str]] = {}
|
|
for _who, c, cite in edgar.filing_cites(companies):
|
|
if cite.adsh:
|
|
quotes.setdefault((c.cik, cite.adsh), []).append(cite.quote)
|
|
for (cik, adsh), qs in quotes.items():
|
|
p = hoard / "sec" / "filings" / cik / f"{adsh}.htm"
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_text(
|
|
"<html><body>"
|
|
+ "".join(f"<p>{html.escape(q)}</p>" for q in qs)
|
|
+ "</body></html>"
|
|
)
|
|
|
|
|
|
def _count(c, table):
|
|
return c.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
|
|
|
|
|
def test_entities_offline_writes_and_logs(env):
|
|
c, store, published, hoard = env
|
|
_cache_offline(hoard)
|
|
# a stale curated link the seed no longer carries
|
|
T.write_links(
|
|
c, [T.LinkRow("0001564406", "program", "QPP-MIPS", "curated", "x", "url", "u")]
|
|
)
|
|
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
companies, orgs = load_seed()
|
|
assert _count(c, "market.entity") == len(companies) + len(orgs)
|
|
osh = next(e for e in T.read_entities(c, "company") if e.key == "0001564406")
|
|
assert osh.status == "acquired" and osh.ended_on == date(2023, 5, 2)
|
|
assert osh.sic == "8000" and osh.item_key
|
|
assert "no submissions for AMED" in res.output
|
|
# the five fixture filings, and the seed's dated ticker
|
|
assert len(T.read_filings(c, "0001564406")) == 5
|
|
assert [s.source for s in T.read_securities(c, "0001564406")] == ["seed"]
|
|
links = T.read_links(c, basis="curated")
|
|
assert all(l.item_key for l in links)
|
|
assert ("program", "QPP-MIPS") not in {(l.scope, l.target) for l in links}
|
|
assert [l.target for l in T.read_links(c, basis="sic")] == ["8000"]
|
|
logs = c.execute(
|
|
"SELECT table_name, rows FROM cms.ingest_log WHERE module = 'market.edgar' "
|
|
"ORDER BY table_name"
|
|
).fetchall()
|
|
assert [t for t, _ in logs] == [
|
|
"market.entity",
|
|
"market.entity_link",
|
|
"market.filing",
|
|
"market.security",
|
|
]
|
|
assert published == [True]
|
|
assert "retired 1" in res.output
|
|
assert store.count() > 0
|
|
|
|
|
|
def test_entities_rerun_is_idempotent(env):
|
|
c, _, published, hoard = env
|
|
_cache_offline(hoard)
|
|
for _ in range(2):
|
|
assert (
|
|
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
|
== 0
|
|
)
|
|
companies, orgs = load_seed()
|
|
assert _count(c, "market.entity") == len(companies) + len(orgs)
|
|
assert _count(c, "market.filing") == 5
|
|
|
|
|
|
def test_entities_no_write_prints_counts(env, monkeypatch):
|
|
c, _, published, hoard = env
|
|
_cache_offline(hoard)
|
|
monkeypatch.setattr(
|
|
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
|
)
|
|
res = runner.invoke(app, ["market", "entities", "--offline"])
|
|
assert res.exit_code == 0, res.output
|
|
companies, orgs = load_seed()
|
|
assert f"entities {len(companies) + len(orgs)}" in res.output
|
|
assert (
|
|
"company/acquired" in res.output
|
|
and "organization/specialty_society" in res.output
|
|
)
|
|
assert "dry run" in res.output and published == []
|
|
assert "warning" not in res.output # every seat specialty has a society
|
|
|
|
|
|
def test_entities_warns_about_uncovered_seat_specialty(env):
|
|
c, _, _, hoard = env
|
|
_cache_offline(hoard)
|
|
write_seats(c, "RUC", [_seat("Podiatry", "PODIATRY", year=2025)])
|
|
res = runner.invoke(app, ["market", "entities", "--offline"])
|
|
assert res.exit_code == 0
|
|
assert "seat specialties without a society: ['PODIATRY']" in res.output
|
|
|
|
|
|
def _handler(routes):
|
|
def handler(request: httpx.Request):
|
|
url = str(request.url)
|
|
for pattern, make in routes:
|
|
m = re.search(pattern, url)
|
|
if m:
|
|
return make(m)
|
|
return httpx.Response(404)
|
|
|
|
return handler
|
|
|
|
|
|
def _online(monkeypatch, routes):
|
|
monkeypatch.setattr(
|
|
http,
|
|
"client",
|
|
lambda source, key=None: httpx.Client(
|
|
transport=httpx.MockTransport(_handler(routes))
|
|
),
|
|
)
|
|
|
|
|
|
def test_entities_refusal_prints_gap(env, monkeypatch):
|
|
c, _, published, hoard = env
|
|
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
|
|
res = runner.invoke(app, ["market", "entities", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "gap: sec-edgar:entities:" in res.output and "403" in res.output
|
|
assert _count(c, "market.entity") == 0 and _count(c, "market.filing") == 0
|
|
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:entities"]
|
|
assert published == [True]
|
|
|
|
|
|
def test_entities_refusal_without_write_records_nothing(env, monkeypatch):
|
|
c, _, published, _ = env
|
|
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
|
|
monkeypatch.setattr(
|
|
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
|
)
|
|
res = runner.invoke(app, ["market", "entities"])
|
|
assert res.exit_code == 0 and "gap: sec-edgar:entities:" in res.output
|
|
assert published == []
|
|
|
|
|
|
def _sub_for(cik: str) -> dict:
|
|
return {
|
|
"cik": cik,
|
|
"name": f"Filer {cik}",
|
|
"sic": "8082",
|
|
"sicDescription": "Home Health",
|
|
"tickers": [],
|
|
"formerNames": [],
|
|
"filings": {
|
|
"recent": {
|
|
"accessionNumber": ["A-1", "A-2"],
|
|
"filingDate": ["2024-01-01", "2024-01-02"],
|
|
"reportDate": ["", ""],
|
|
"acceptanceDateTime": ["", ""],
|
|
"form": ["10-K", "4"],
|
|
"items": ["", ""],
|
|
"primaryDocument": ["a.htm", "b.xml"],
|
|
},
|
|
"files": [],
|
|
},
|
|
}
|
|
|
|
|
|
def test_sic_sweep_registers_filers_and_sic_links(env, monkeypatch):
|
|
c, _, published, hoard = env
|
|
_cache_filings(hoard)
|
|
atom = (FX / "sic_8082.atom").read_bytes().replace(b"0001468328", b"0000000042")
|
|
seen: list[str] = []
|
|
|
|
def sub(m):
|
|
seen.append(m.group(1))
|
|
cik = m.group(1)
|
|
if cik == "0001564406":
|
|
return httpx.Response(
|
|
200, content=(FX / "submissions_CIK0001564406.json").read_bytes()
|
|
)
|
|
if cik == "0000000042":
|
|
return httpx.Response(404) # no submissions file: skipped
|
|
return httpx.Response(200, json=_sub_for(cik))
|
|
|
|
def page(m):
|
|
return httpx.Response(
|
|
200, content=(FX / "CIK0001564406-submissions-001.json").read_bytes()
|
|
)
|
|
|
|
_online(
|
|
monkeypatch,
|
|
[
|
|
(
|
|
r"company_tickers\.json",
|
|
lambda m: httpx.Response(
|
|
200, content=(FX / "company_tickers.json").read_bytes()
|
|
),
|
|
),
|
|
(r"SIC=\d+.*start=0", lambda m: httpx.Response(200, content=atom)),
|
|
(r"submissions-001\.json", page),
|
|
(r"CIK(\d{10})\.json", sub),
|
|
],
|
|
)
|
|
res = runner.invoke(app, ["market", "entities", "--sic-sweep", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert res.output.count("SIC ") == len(edgar.HEALTH_SICS)
|
|
keys = {e.key for e in T.read_entities(c, "company")}
|
|
assert {"0001725255", "0001468328"} <= keys and "0000000042" not in keys
|
|
# a sweep filer keeps only the sweep forms; the seed's filings keep everything
|
|
assert [f.form for f in T.read_filings(c, "0001725255")] == ["10-K"]
|
|
assert len(T.read_filings(c, "0001468328")) == 2
|
|
assert len(T.read_filings(c, "0001564406")) == 5
|
|
sic = T.read_links(c, basis="sic")
|
|
assert {l.target for l in sic} == {"8082", "8000"}
|
|
assert any(l.entity_key == "0001725255" for l in sic)
|
|
|
|
|
|
def test_entities_seed_errors_abort(env, monkeypatch):
|
|
c, _, published, _ = env
|
|
companies, orgs = load_seed()
|
|
bad = companies[0].__class__(
|
|
**{
|
|
**companies[0].__dict__,
|
|
"links": (
|
|
companies[0].links[0].__class__("family", "NOPE", companies[0].cite),
|
|
),
|
|
}
|
|
)
|
|
monkeypatch.setattr(mk, "load_seed", lambda: ((bad, *companies[1:]), orgs))
|
|
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
|
|
assert res.exit_code == 1 and "seed error: OSH" in res.output
|
|
assert c.execute(
|
|
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
|
|
).fetchone() == (0,)
|
|
assert published == []
|
|
|
|
|
|
def test_entities_missing_vocabulary_tables_abort(env, monkeypatch):
|
|
bare = duckdb.connect(":memory:")
|
|
monkeypatch.setattr(mk, "_read", lambda: bare)
|
|
res = runner.invoke(app, ["market", "entities", "--offline"])
|
|
assert res.exit_code == 1 and "cannot validate the seed" in res.output
|
|
bare.close()
|
|
|
|
|
|
def test_entities_other_read_errors_propagate(env, monkeypatch):
|
|
def boom():
|
|
raise RuntimeError("disk on fire")
|
|
|
|
monkeypatch.setattr(mk, "_read", boom)
|
|
res = runner.invoke(app, ["market", "entities", "--offline"])
|
|
assert isinstance(res.exception, RuntimeError)
|
|
|
|
|
|
def test_require_key_missing_writes_only_gap(env, monkeypatch):
|
|
c, _, published, _ = env
|
|
monkeypatch.delenv("TIINGO_API_KEY", raising=False)
|
|
assert mk._require_key("tiingo", "tiingo:prices", write=True) is None
|
|
assert [(g.source, g.note) for g in T.read_gaps(c)] == [
|
|
("tiingo:prices", "TIINGO_API_KEY not in .env")
|
|
]
|
|
assert all(_count(c, f"market.{t}") == 0 for t in ("entity", "security", "filing"))
|
|
assert published == [True]
|
|
|
|
|
|
def test_require_key_present_returns_it(env, monkeypatch):
|
|
monkeypatch.setenv("TIINGO_API_KEY", "abc")
|
|
assert mk._require_key("tiingo", "tiingo:prices", write=True) == "abc"
|
|
|
|
|
|
def test_require_key_missing_prints_gap_without_write(env, monkeypatch, capsys):
|
|
monkeypatch.delenv("HUD_API_TOKEN", raising=False)
|
|
assert mk._require_key("hud", "hud:hud", write=False) is None
|
|
assert "gap: hud:hud: HUD_API_TOKEN not in .env" in capsys.readouterr().out
|
|
c, _, published, _ = env
|
|
assert published == []
|
|
|
|
|
|
def test_gap_cleared_after_success(env):
|
|
c, _, _, hoard = env
|
|
_cache_offline(hoard)
|
|
T.write_gap(c, "sec-edgar:entities", "403")
|
|
T.write_gap(c, "tiingo:prices", "no key")
|
|
assert (
|
|
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
|
== 0
|
|
)
|
|
assert {g.source for g in T.read_gaps(c)} == {
|
|
"tiingo:prices",
|
|
"web:renalmd.org",
|
|
"web:asco.org",
|
|
"web:ahip.org",
|
|
}
|
|
|
|
|
|
def test_gaps_command_lists_rows(env):
|
|
c, _, _, _ = env
|
|
T.write_gap(c, "tiingo:prices", "TIINGO_API_KEY not in .env")
|
|
res = runner.invoke(app, ["market", "gaps"])
|
|
assert res.exit_code == 0
|
|
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
|
|
|
|
|
|
def test_gaps_command_empty_and_missing_tables(env, monkeypatch):
|
|
c, _, _, _ = env
|
|
T.ensure_tables(c)
|
|
assert "no gaps recorded" in runner.invoke(app, ["market", "gaps"]).output
|
|
bare = duckdb.connect(":memory:")
|
|
monkeypatch.setattr(mk, "_read", lambda: bare)
|
|
assert "no market tables yet" in runner.invoke(app, ["market", "gaps"]).output
|
|
|
|
class _Broken:
|
|
def execute(self, *a, **k):
|
|
raise RuntimeError("x")
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
monkeypatch.setattr(mk, "_read", lambda: _Broken())
|
|
assert isinstance(runner.invoke(app, ["market", "gaps"]).exception, RuntimeError)
|
|
bare.close()
|
|
|
|
|
|
def test_default_indirections_resolve_lazily(monkeypatch):
|
|
import conf.connect as cc
|
|
|
|
calls: list[str] = []
|
|
monkeypatch.setattr(cc, "duckdb_batch", lambda n: calls.append(f"batch {n}") or n)
|
|
monkeypatch.setattr(
|
|
cc, "duckdb", lambda n, read_only=True: calls.append(f"read {n}") or n
|
|
)
|
|
monkeypatch.setattr(cc, "bib", lambda: calls.append("bib") or "store")
|
|
monkeypatch.setattr(
|
|
cc, "publish_replica", lambda n: calls.append(f"pub {n}") or "p"
|
|
)
|
|
assert mk._batch() == "aco" and mk._read() == "aco" and mk._store() == "store"
|
|
mk._publish()
|
|
assert calls == ["batch aco", "read aco", "bib", "pub aco"]
|
|
|
|
|
|
def test_entities_aborts_when_a_quote_is_not_in_the_filing(env):
|
|
c, _, published, hoard = env
|
|
_cache_offline(hoard)
|
|
victim = next(hoard.glob("sec/filings/*/*.htm"))
|
|
victim.write_text("<p>nothing the seed quotes</p>")
|
|
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
|
|
assert res.exit_code == 1 and "quote error:" in res.output
|
|
assert published == []
|
|
assert c.execute(
|
|
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
|
|
).fetchone() == (0,)
|
|
|
|
|
|
def test_entities_aborts_when_a_cited_filing_is_not_hoarded(env):
|
|
c, _, published, hoard = env
|
|
_cache_offline(hoard)
|
|
next(hoard.glob("sec/filings/*/*.htm")).unlink()
|
|
res = runner.invoke(app, ["market", "entities", "--offline"])
|
|
assert res.exit_code == 1 and "not hoarded" in res.output
|
|
|
|
|
|
def test_entities_fetches_cited_filings_online(env, monkeypatch):
|
|
c, _, _, hoard = env
|
|
_cache_offline(hoard)
|
|
victim = next(hoard.glob("sec/filings/*/*.htm"))
|
|
cik, adsh = victim.parent.name, victim.stem
|
|
body = victim.read_bytes()
|
|
victim.unlink()
|
|
(hoard / "sec" / "submissions" / f"CIK{cik}.json").write_text(
|
|
json.dumps(
|
|
_sub_for(cik)
|
|
| {
|
|
"filings": {
|
|
"recent": {
|
|
**_sub_for(cik)["filings"]["recent"],
|
|
"accessionNumber": [adsh, "Z"],
|
|
"primaryDocument": ["doc.htm", "z.htm"],
|
|
},
|
|
"files": [],
|
|
}
|
|
}
|
|
)
|
|
)
|
|
seen = []
|
|
|
|
def serve(m):
|
|
seen.append(m.group(0))
|
|
return httpx.Response(200, content=body)
|
|
|
|
_online(monkeypatch, [(r"/Archives/edgar/data/\d+/\d+/doc\.htm", serve)])
|
|
res = runner.invoke(app, ["market", "entities"])
|
|
assert res.exit_code == 0, res.output
|
|
assert len(seen) == 1 and victim.exists()
|
|
|
|
|
|
def test_entities_manifest_and_ingest_log_sources(env):
|
|
c, _, _, hoard = env
|
|
_cache_offline(hoard)
|
|
assert (
|
|
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
|
== 0
|
|
)
|
|
manifest = hoard / "MANIFEST.jsonl"
|
|
assert manifest.exists() and manifest.read_text().count("\n") >= 6
|
|
rows = c.execute(
|
|
"SELECT table_name, source_file, sha256 FROM cms.ingest_log WHERE module = 'market.edgar'"
|
|
).fetchall()
|
|
assert len(rows) == 4 and all(sf and sha and len(sha) == 64 for _, sf, sha in rows)
|
|
|
|
|
|
def test_entities_records_unreachable_site_gaps_and_pseudo_orgs(env):
|
|
c, _, _, hoard = env
|
|
_cache_offline(hoard)
|
|
assert (
|
|
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
|
== 0
|
|
)
|
|
gaps = {g.source for g in T.read_gaps(c)}
|
|
assert {"web:renalmd.org", "web:asco.org", "web:ahip.org"} <= gaps
|
|
assert "sec-edgar:entities" not in gaps
|
|
pseudo = {e.key for e in T.read_entities(c, "organization") if e.pseudo}
|
|
assert pseudo == {"PRIMARY-CARE-SEAT", "ONCOLOGY-SEAT"}
|
|
|
|
|
|
def test_entities_retires_organizations_the_seed_dropped(env):
|
|
c, _, _, hoard = env
|
|
_cache_offline(hoard)
|
|
T.write_entities(c, [T.EntityRow("ASCO", "ASCO", "organization")])
|
|
assert (
|
|
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
|
== 0
|
|
)
|
|
assert "ASCO" not in {e.key for e in T.read_entities(c, "organization")}
|
|
|
|
|
|
def test_entities_aborts_when_a_quote_copies_a_cpt_descriptor(env):
|
|
c, _, published, hoard = env
|
|
_cache_offline(hoard)
|
|
victim = load_seed()[0][0].cite.quote
|
|
c.execute("DELETE FROM pfs.cpt_code")
|
|
c.execute("INSERT INTO pfs.cpt_code VALUES ('1', ?)", [victim])
|
|
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
|
|
assert res.exit_code == 1 and "ama error: OSH" in res.output
|
|
assert published == []
|
|
assert c.execute(
|
|
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
|
|
).fetchone() == (0,)
|
|
|
|
|
|
def test_entities_stores_ticker_history_and_link_dates(env):
|
|
c, _, _, hoard = env
|
|
_cache_offline(hoard)
|
|
assert (
|
|
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
|
|
== 0
|
|
)
|
|
assert [
|
|
(s.ticker, s.valid_to, s.priced) for s in T.read_securities(c, "0001156039")
|
|
] == [("ANTM", date(2022, 6, 27), False), ("ELV", None, True)]
|
|
assert [s.priced for s in T.read_securities(c, "0001739940")] == [False]
|
|
(clov,) = [
|
|
l
|
|
for l in T.read_links(c, basis="curated", target="APM")
|
|
if l.entity_key == "0001801170"
|
|
]
|
|
assert clov.valid_to == date(2023, 12, 31)
|
|
|
|
|
|
# ── stack market prices ──────────────────────────────────────────────
|
|
import io # noqa: E402
|
|
import zipfile # noqa: E402
|
|
|
|
from market import tiingo # noqa: E402
|
|
|
|
DAY = date(2026, 11, 12)
|
|
|
|
|
|
def _supported_zip(*rows: str) -> bytes:
|
|
head = "ticker,exchange,assetType,priceCurrency,startDate,endDate\n"
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w") as z:
|
|
z.writestr("supported_tickers.csv", head + "\n".join(rows) + "\n")
|
|
return buf.getvalue()
|
|
|
|
|
|
def _eod(day: str, close: float, adj: float | None = None) -> dict:
|
|
return {
|
|
"date": f"{day}T00:00:00.000Z",
|
|
"open": close,
|
|
"high": close,
|
|
"low": close,
|
|
"close": close,
|
|
"volume": 5,
|
|
"adjClose": close if adj is None else adj,
|
|
"divCash": 0.0,
|
|
"splitFactor": 1.0,
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def px(env, monkeypatch, tmp_path):
|
|
c, store, published, hoard = env
|
|
monkeypatch.setenv("TIINGO_API_KEY", "secret-key")
|
|
monkeypatch.setattr(tiingo, "state_dir", lambda: tmp_path / "state")
|
|
monkeypatch.setattr(tiingo, "new_pacer", lambda: _Pacer())
|
|
monkeypatch.setattr(tiingo, "today", lambda: DAY)
|
|
seen: list[httpx.Request] = []
|
|
series = {
|
|
"AAA": [
|
|
_eod("2024-01-02", 100),
|
|
_eod("2024-01-03", 110),
|
|
_eod("2024-01-04", 99),
|
|
],
|
|
"BBB": [_eod("2024-01-02", 10), _eod("2024-01-03", 11)],
|
|
"SPY": [_eod("2024-01-02", 400), _eod("2024-01-03", 404)],
|
|
"XLV": [_eod("2024-01-02", 100)],
|
|
"IHF": [_eod("2024-01-02", 50)],
|
|
}
|
|
z = _supported_zip(
|
|
"AAA,NYSE,Stock,USD,2000-01-01,2026-11-12",
|
|
"BBB,NYSE,Stock,USD,2000-01-01,2026-11-12",
|
|
"SPY,NYSE,ETF,USD,1993-01-29,2026-11-12",
|
|
"XLV,NYSE,ETF,USD,1998-12-22,2026-11-12",
|
|
"IHF,NYSE,ETF,USD,2006-05-05,2026-11-12",
|
|
"ZZZ,NYSE,Stock,USD,2000-01-01,2026-11-12",
|
|
)
|
|
|
|
def prices_for(m):
|
|
if m.group(1) not in series:
|
|
return httpx.Response(404, json={"detail": "Not found."})
|
|
return httpx.Response(200, json=series[m.group(1)])
|
|
|
|
routes = [
|
|
(r"supported_tickers\.zip", lambda m: httpx.Response(200, content=z)),
|
|
(r"/daily/(\w+)/prices", prices_for),
|
|
]
|
|
|
|
def handler(request):
|
|
seen.append(request)
|
|
return _handler(routes)(request)
|
|
|
|
monkeypatch.setattr(
|
|
http,
|
|
"client",
|
|
lambda source, key=None: httpx.Client(
|
|
transport=httpx.MockTransport(handler),
|
|
headers={"Authorization": f"Token {key}"} if key else {},
|
|
),
|
|
)
|
|
return c, store, published, hoard, seen, series, tmp_path / "state"
|
|
|
|
|
|
def _price_requests(seen):
|
|
return [r for r in seen if "/prices" in r.url.path]
|
|
|
|
|
|
def test_prices_cli_fetch_write(px):
|
|
c, _, published, hoard, seen, _, state = px
|
|
res = runner.invoke(
|
|
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
|
|
)
|
|
assert res.exit_code == 0, res.output
|
|
assert _count(c, "market.price_daily") == 5
|
|
assert [r.url.params["startDate"] for r in _price_requests(seen)] == [
|
|
"1993-01-29"
|
|
] * 2
|
|
assert all("secret-key" not in str(r.url) for r in seen)
|
|
keyed = [r for r in seen if "/prices" in r.url.path]
|
|
assert all(r.headers["authorization"] == "Token secret-key" for r in keyed)
|
|
free = [r for r in seen if "supported" in r.url.path]
|
|
assert free and "authorization" not in free[0].headers
|
|
assert {
|
|
(t, n)
|
|
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
|
} == {("market.price_daily", 5), ("market.series", 2)}
|
|
assert [s.value for s in T.read_series(c, ["AAA"])] == [pytest.approx(99.0)]
|
|
assert published == [True]
|
|
used = json.loads((state / "tiingo-2026-11.json").read_text())
|
|
assert used["symbols"] == ["AAA", "BBB"] and used["day_counts"] == {"2026-11-12": 2}
|
|
assert (hoard / "tiingo" / "eod" / "AAA" / "1993-01-29_2026-11-12.json").is_file()
|
|
assert "priced 2 symbols, 5 rows" in res.output and "2/450" in res.output
|
|
assert T.read_gaps(c) == []
|
|
|
|
|
|
def test_prices_incremental_pull_and_same_day_skip(px, monkeypatch):
|
|
c, _, _, _, seen, series, _ = px
|
|
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
|
|
assert runner.invoke(app, cmd).exit_code == 0
|
|
before = len(_price_requests(seen))
|
|
again = runner.invoke(app, cmd)
|
|
assert len(_price_requests(seen)) == before # priced today: nothing to do
|
|
assert "priced 0 symbols" in again.output
|
|
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
|
|
series["AAA"] = [_eod("2024-01-04", 99, 49.5), _eod("2024-01-05", 108.9, 54.45)]
|
|
assert runner.invoke(app, cmd).exit_code == 0
|
|
last = _price_requests(seen)[-1]
|
|
assert last.url.params["startDate"] == "2024-01-04"
|
|
rows = {r.day: r for r in T.read_prices(c, "AAA")}
|
|
assert len(rows) == 4 and rows[date(2024, 1, 4)].adj_close == 99
|
|
assert rows[date(2024, 1, 5)].ret == pytest.approx(0.1)
|
|
assert [m.value for m in T.read_series(c, ["AAA"])] == [pytest.approx(108.9)]
|
|
|
|
|
|
def test_prices_benchmarks_start_at_listing(px):
|
|
c, _, _, _, seen, _, _ = px
|
|
res = runner.invoke(app, ["market", "prices", "--benchmarks", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
starts = {
|
|
r.url.path.split("/")[3]: r.url.params["startDate"]
|
|
for r in _price_requests(seen)
|
|
}
|
|
assert starts == {"SPY": "1993-01-29", "XLV": "1998-12-22", "IHF": "2006-05-05"}
|
|
|
|
|
|
def test_prices_dry_run_reports_budget(px, monkeypatch):
|
|
c, _, published, _, seen, _, state = px
|
|
T.ensure_tables(c)
|
|
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
|
|
T.write_links(c, [T.LinkRow("K1", "program", "PFS", "fts", "mention", "adsh", "a")])
|
|
state.mkdir(parents=True)
|
|
(state / "tiingo-2026-11.json").write_text(
|
|
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
|
|
)
|
|
monkeypatch.setattr(
|
|
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
|
)
|
|
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "1/450 symbols used, 449 remaining" in res.output
|
|
assert "1. SPY" in res.output and "used" in res.output
|
|
assert "AAA" in res.output and "K1" in res.output and "first pull" in res.output
|
|
assert _price_requests(seen) == [] and published == []
|
|
assert not (state / "tiingo-2026-11.json").read_text().count("AAA")
|
|
|
|
|
|
def test_prices_dry_run_without_market_tables_lists_benchmarks(px):
|
|
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "3 symbols ranked" in res.output
|
|
|
|
|
|
def test_prices_dry_run_shows_stored_span(px):
|
|
c, *_ = px
|
|
T.ensure_tables(c)
|
|
T.write_prices(
|
|
c,
|
|
"AAA",
|
|
tiingo.to_prices(
|
|
"AAA", [_eod("2024-01-02", 1)], pulled_on=DAY, source_file="f", overlap=None
|
|
),
|
|
)
|
|
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--dry-run"])
|
|
assert "stored to 2024-01-02" in res.output
|
|
|
|
|
|
def test_prices_without_key_records_gap(px, monkeypatch):
|
|
c, _, published, _, seen, _, _ = px
|
|
monkeypatch.delenv("TIINGO_API_KEY")
|
|
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
|
|
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"]
|
|
assert _count(c, "market.price_daily") == 0 and seen == [] and published == [True]
|
|
|
|
|
|
def test_prices_refusal_prints_gap(px, monkeypatch):
|
|
c, _, published, _, _, _, _ = px
|
|
monkeypatch.setattr(
|
|
http,
|
|
"client",
|
|
lambda source, key=None: httpx.Client(
|
|
transport=httpx.MockTransport(lambda r: httpx.Response(403))
|
|
),
|
|
)
|
|
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "gap: tiingo:prices:" in res.output and "403" in res.output
|
|
assert _count(c, "market.price_daily") == 0
|
|
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"] and published == [
|
|
True
|
|
]
|
|
|
|
|
|
def test_prices_refusal_on_prices_after_supported_ok(px, monkeypatch):
|
|
c, _, published, _, _, _, _ = px
|
|
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
|
|
routes = [
|
|
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
|
|
(r"/prices", lambda m: httpx.Response(401)),
|
|
]
|
|
_online(monkeypatch, routes)
|
|
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA"])
|
|
assert (
|
|
res.exit_code == 0
|
|
and "gap: tiingo:prices:" in res.output
|
|
and "401" in res.output
|
|
)
|
|
assert published == []
|
|
|
|
|
|
def test_prices_nothing_selected_exits(px):
|
|
res = runner.invoke(app, ["market", "prices"])
|
|
assert res.exit_code == 2 and "nothing to price" in res.output
|
|
|
|
|
|
def test_prices_unknown_ticker_and_window_gap_become_gap_note(px):
|
|
c, _, _, _, _, series, _ = px
|
|
T.ensure_tables(c)
|
|
T.write_securities(
|
|
c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, date(2030, 1, 1), "s")]
|
|
)
|
|
res = runner.invoke(
|
|
app, ["market", "prices", "--ticker", "BBB", "--ticker", "ZZZ", "--write"]
|
|
)
|
|
assert res.exit_code == 0, res.output
|
|
assert "ZZZ: not found at Tiingo" in res.output
|
|
(gap,) = T.read_gaps(c)
|
|
assert (
|
|
"ZZZ: not found at Tiingo" in gap.note
|
|
and "BBB: Tiingo ends 2026-11-12" in gap.note
|
|
)
|
|
# BBB later prices cleanly: its entry goes, ZZZ's stays
|
|
T.write_securities(c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, None, "s")])
|
|
runner.invoke(app, ["market", "prices", "--ticker", "BBB", "--refresh", "--write"])
|
|
(gap,) = T.read_gaps(c)
|
|
assert gap.note == "ZZZ: not found at Tiingo"
|
|
|
|
|
|
def test_prices_clean_run_clears_gap_and_window_clips(px):
|
|
c, _, _, _, _, _, _ = px
|
|
T.ensure_tables(c)
|
|
T.write_gap(c, "tiingo:prices", "AAA: not found at Tiingo")
|
|
T.write_securities(
|
|
c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", date(2024, 1, 3), None, "s")]
|
|
)
|
|
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert [r.day for r in T.read_prices(c, "AAA")] == [
|
|
date(2024, 1, 3),
|
|
date(2024, 1, 4),
|
|
]
|
|
assert T.read_gaps(c) == []
|
|
|
|
|
|
def test_prices_limit_stops_downloads(px):
|
|
c, _, _, _, seen, _, _ = px
|
|
res = runner.invoke(
|
|
app, ["market", "prices", "--benchmarks", "--limit", "1", "--write"]
|
|
)
|
|
assert res.exit_code == 0 and len(_price_requests(seen)) == 1
|
|
assert [r.ticker for r in [*T.read_prices(c, "SPY")][:1]] == ["SPY"]
|
|
|
|
|
|
def test_prices_budget_exhausted_stops_and_writes_what_it_has(px):
|
|
c, _, published, _, seen, _, state = px
|
|
state.mkdir(parents=True)
|
|
(state / "tiingo-2026-11.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"month": "2026-11",
|
|
"symbols": [f"S{i}" for i in range(449)] + ["AAA"],
|
|
"day_counts": {},
|
|
}
|
|
)
|
|
)
|
|
res = runner.invoke(
|
|
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
|
|
)
|
|
assert res.exit_code == 0, res.output
|
|
assert "budget: stopped, BBB" in res.output
|
|
assert [r.ticker for r in T.read_prices(c, "AAA")][:1] == ["AAA"]
|
|
assert T.read_prices(c, "BBB") == [] and published == [True]
|
|
|
|
|
|
def test_prices_without_write_stores_nothing(px, monkeypatch):
|
|
c, _, published, _, _, _, _ = px
|
|
monkeypatch.setattr(
|
|
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
|
)
|
|
res = runner.invoke(app, ["market", "prices", "--ticker", "ZZZ"])
|
|
assert res.exit_code == 0, res.output
|
|
assert (
|
|
"dry run: pass --write" in res.output
|
|
and "gap: tiingo:prices: ZZZ" in res.output
|
|
)
|
|
assert published == []
|
|
|
|
|
|
def test_merge_gap_keeps_other_tickers_and_orders():
|
|
assert mk._merge_gap("", {}, set()) == ""
|
|
assert mk._merge_gap("A: x | B: y", {"C": "z"}, {"A"}) == "B: y | C: z"
|
|
|
|
|
|
def test_or_empty_reraises_other_errors():
|
|
def boom(_):
|
|
raise RuntimeError("disk")
|
|
|
|
with pytest.raises(RuntimeError):
|
|
mk._or_empty(boom, None, {})
|
|
|
|
|
|
def test_prices_ticker_refuses_unpriced_record_without_a_request(px):
|
|
c, _, _, _, seen, _, state = px
|
|
T.ensure_tables(c)
|
|
T.write_securities(
|
|
c, "K9", [T.SecurityRow("K9", "AAA", "NYSE", None, None, "s", False)]
|
|
)
|
|
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "skipped AAA: symbol record only" in res.output
|
|
assert _price_requests(seen) == [] and _count(c, "market.price_daily") == 0
|
|
assert not (state / "tiingo-2026-11.json").exists()
|
|
assert T.read_gaps(c) == []
|
|
|
|
|
|
def test_prices_unsupported_named_ticker_is_gap_not_request(px):
|
|
c, _, _, _, seen, _, _ = px
|
|
res = runner.invoke(app, ["market", "prices", "--ticker", "QQQQ", "--write"])
|
|
assert "skipped QQQQ: not on Tiingo's supported list" in res.output
|
|
assert _price_requests(seen) == []
|
|
(gap,) = T.read_gaps(c)
|
|
assert gap.note == "QQQQ: not on Tiingo's supported list"
|
|
|
|
|
|
def test_prices_universe_records_unsupported_seed_ticker_gap(px):
|
|
c, _, _, _, seen, _, _ = px
|
|
T.ensure_tables(c)
|
|
T.write_securities(
|
|
c, "0001800682", [T.SecurityRow("0001800682", "CANO", "NYSE", None, None, "s")]
|
|
)
|
|
res = runner.invoke(app, ["market", "prices", "--universe", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "skipped CANO: not on Tiingo's supported list" in res.output
|
|
assert all("/CANO/" not in r.url.path for r in seen)
|
|
(gap,) = T.read_gaps(c)
|
|
assert gap.source == "tiingo:prices"
|
|
assert gap.note == "CANO: not on Tiingo's supported list"
|
|
# rerun with everything already priced today keeps the gap
|
|
again = runner.invoke(app, ["market", "prices", "--universe", "--write"])
|
|
assert again.exit_code == 0
|
|
assert [g.note for g in T.read_gaps(c)] == [gap.note]
|
|
|
|
|
|
def test_prices_missing_overlap_day_repulls_further_back(px, monkeypatch):
|
|
c, _, _, _, seen, _, _ = px
|
|
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
|
|
assert runner.invoke(app, cmd).exit_code == 0
|
|
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
|
|
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
|
|
|
|
def eod(m):
|
|
# the 2024-01-04 overlap day is absent from the narrow pull
|
|
start = next(r for r in seen[::-1] if "/prices" in r.url.path).url.params[
|
|
"startDate"
|
|
]
|
|
if start == "2024-01-04":
|
|
return httpx.Response(200, json=[_eod("2024-01-05", 54.45)])
|
|
return httpx.Response(
|
|
200, json=[_eod("2024-01-03", 55), _eod("2024-01-05", 60.5)]
|
|
)
|
|
|
|
handler = _handler(
|
|
[
|
|
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
|
|
(r"/daily/AAA/prices", eod),
|
|
]
|
|
)
|
|
|
|
def record(request):
|
|
seen.append(request)
|
|
return handler(request)
|
|
|
|
monkeypatch.setattr(
|
|
http,
|
|
"client",
|
|
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(record)),
|
|
)
|
|
res = runner.invoke(app, cmd)
|
|
assert res.exit_code == 0, res.output
|
|
assert "overlap day 2024-01-04 missing" in res.output
|
|
assert "re-pulling from 2023-12-25" in res.output
|
|
row = {r.day: r for r in T.read_prices(c, "AAA")}[date(2024, 1, 5)]
|
|
assert row.ret == pytest.approx(60.5 / 55 - 1)
|
|
|
|
|
|
# ── fts ──────────────────────────────────────────────────────────────
|
|
MPFS = '"Medicare Physician Fee Schedule"'
|
|
SNIPPET = (FX / "filing_snippet.htm").read_bytes()
|
|
LICENSED = "a licensed descriptor that no filing quote should ever copy word for word"
|
|
|
|
|
|
def _hit_json(cik, adsh, filename, sics, date_="2023-03-01"):
|
|
return {
|
|
"_id": f"{adsh}:{filename}",
|
|
"_source": {
|
|
"ciks": [cik],
|
|
"adsh": adsh,
|
|
"form": "10-K",
|
|
"file_date": date_,
|
|
"period_ending": "2022-12-31",
|
|
"sics": sics,
|
|
"display_names": [f"Filer {cik} (CIK {cik})"],
|
|
},
|
|
}
|
|
|
|
|
|
HITS = [
|
|
_hit_json("0000012345", "0000012345-23-000001", "a10k.htm", ["8071"]),
|
|
_hit_json("0000000002", "0000000002-23-000001", "reit.htm", ["6798"]),
|
|
_hit_json("0000012346", "0000012346-23-000001", "ex99.pdf", ["8082"]),
|
|
_hit_json("0000012347", "0000012347-23-000001", "gone.htm", ["8082"]),
|
|
]
|
|
|
|
|
|
def _fts_net(
|
|
monkeypatch, hits=HITS, docs=None, forbid_docs=False, status=200, broken=None
|
|
):
|
|
"""A mock EDGAR: the term "Medicare Physician Fee Schedule" has *hits* in
|
|
2023; every other query is empty."""
|
|
docs = docs if docs is not None else {"a10k.htm": SNIPPET}
|
|
requested: list[str] = []
|
|
|
|
def efts(m):
|
|
p = dict(httpx.URL(m.group(0)).params)
|
|
mine = p["q"] == MPFS and p["from"] == "0"
|
|
return httpx.Response(
|
|
status,
|
|
json={
|
|
"hits": {
|
|
"total": {"value": len(hits) if mine else 0},
|
|
"hits": hits if mine else [],
|
|
}
|
|
},
|
|
)
|
|
|
|
def doc(m):
|
|
requested.append(m.group(1))
|
|
if broken and m.group(1) in broken:
|
|
if broken[m.group(1)] == "timeout":
|
|
raise httpx.ReadTimeout("slow")
|
|
return httpx.Response(broken[m.group(1)])
|
|
body = docs.get(m.group(1))
|
|
if forbid_docs and body:
|
|
raise AssertionError("no document download expected")
|
|
return httpx.Response(200, content=body) if body else httpx.Response(404)
|
|
|
|
def sub(m):
|
|
return httpx.Response(200, json={**_sub_for(m.group(1)), "tickers": ["ZZZ"]})
|
|
|
|
_online(
|
|
monkeypatch,
|
|
[
|
|
(r"efts\.sec\.gov/.*", efts),
|
|
(r"/Archives/edgar/data/\d+/\d+/([^/?]+)$", doc),
|
|
(
|
|
r"company_tickers\.json",
|
|
lambda m: httpx.Response(
|
|
200, content=(FX / "company_tickers.json").read_bytes()
|
|
),
|
|
),
|
|
(r"CIK(\d{10})\.json", sub),
|
|
],
|
|
)
|
|
return requested
|
|
|
|
|
|
FTS = ["market", "fts", "--program", "PFS", "--from", "2023", "--to", "2023"]
|
|
|
|
|
|
def test_fts_cli_dry_run_lists_queries(env, monkeypatch):
|
|
c, _, published, hoard = env
|
|
monkeypatch.setattr(
|
|
http, "client", lambda *a, **k: (_ for _ in ()).throw(AssertionError("net"))
|
|
)
|
|
res = runner.invoke(app, [*FTS, "--dry-run"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "program PFS: 5 terms, 5 queries" in res.output
|
|
assert "5 queries (0 already hoarded)" in res.output
|
|
assert published == []
|
|
full = runner.invoke(app, ["market", "fts", "--dry-run"])
|
|
assert "program MSSP: 1 terms" in full.output
|
|
assert "program CARE-MANAGEMENT: 2 terms" in full.output
|
|
assert f"{(5 + 2 + 1 + 1 + 1 + 2 + 2) * (dt_year() - 2000)} queries" in full.output
|
|
|
|
|
|
def dt_year():
|
|
import datetime
|
|
|
|
return datetime.date.today().year
|
|
|
|
|
|
def test_fts_cli_family_scope_and_bad_keys(env, monkeypatch):
|
|
monkeypatch.setattr("pfs.families.refresh_from", lambda con: 0)
|
|
res = runner.invoke(
|
|
app,
|
|
[
|
|
"market",
|
|
"fts",
|
|
"--family",
|
|
"HOME-VISITS",
|
|
"--dry-run",
|
|
"--from",
|
|
"2023",
|
|
"--to",
|
|
"2023",
|
|
],
|
|
)
|
|
assert res.exit_code == 0, res.output
|
|
assert "family HOME-VISITS: 8 terms, 8 queries" in res.output
|
|
assert (
|
|
runner.invoke(app, ["market", "fts", "--family", "NOPE", "--dry-run"]).exit_code
|
|
== 2
|
|
)
|
|
assert (
|
|
runner.invoke(
|
|
app, ["market", "fts", "--program", "NOPE", "--dry-run"]
|
|
).exit_code
|
|
== 2
|
|
)
|
|
|
|
|
|
def test_fts_cli_without_write_stores_nothing(env, monkeypatch):
|
|
c, _, published, hoard = env
|
|
_fts_net(monkeypatch)
|
|
monkeypatch.setattr(
|
|
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
|
)
|
|
res = runner.invoke(app, FTS)
|
|
assert res.exit_code == 0, res.output
|
|
assert "Medicare Physician Fee Schedule 10-K 2023: 4 hits" in res.output
|
|
assert "passages kept 1," in res.output and "new entities 1" in res.output
|
|
assert "fts links 1 (PFS 1)" in res.output and "dry run" in res.output
|
|
assert (
|
|
"gap: sec-edgar:fts: 0000012346-23-000001:ex99.pdf: format not readable"
|
|
in res.output
|
|
)
|
|
assert (
|
|
"gap: sec-edgar:fts: 0000012347-23-000001:gone.htm: document not found"
|
|
in res.output
|
|
)
|
|
assert published == []
|
|
# hoarded: the pages and the one downloaded readable document
|
|
assert (
|
|
hoard / "sec/fts/medicare-physician-fee-schedule/10-K/2023.0.json"
|
|
).is_file()
|
|
assert (hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm").is_file()
|
|
|
|
|
|
def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch):
|
|
c, store, published, hoard = env
|
|
curated = T.LinkRow(
|
|
"0000012345", "program", "APM", "curated", "curated", "adsh", "x", "", "K"
|
|
)
|
|
T.write_links(c, [curated])
|
|
_fts_net(monkeypatch)
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
(ent,) = T.read_entities(c, "company")
|
|
assert ent.key == "0000012345" and ent.sic == "8082"
|
|
assert [s.ticker for s in T.read_securities(c, ent.key)] == ["ZZZ"]
|
|
assert [f.form for f in T.read_filings(c, ent.key)] == ["10-K"]
|
|
assert [l.target for l in T.read_links(c, basis="sic")] == ["8082"]
|
|
assert T.read_links(c, basis="curated") == [curated] # survives
|
|
(fts_link,) = T.read_links(c, basis="fts")
|
|
assert (fts_link.scope, fts_link.target, fts_link.strength) == (
|
|
"program",
|
|
"PFS",
|
|
"mention",
|
|
)
|
|
assert fts_link.item_key and fts_link.passage_id.startswith(
|
|
"0000012345-23-000001:a10k.htm:"
|
|
)
|
|
# the link's passage is verbatim in the hoarded filing, short, and cited
|
|
(p,) = [p for p in T.read_passages(c) if p.passage_id == fts_link.passage_id]
|
|
assert len(p.passage) <= 400 and p.item_key == fts_link.item_key
|
|
text = edgar.read_doc(hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm")
|
|
assert p.passage in text and p.label == "" and p.label_version == ""
|
|
assert len(T.read_hits(c)) == 4 and len(T.read_passages(c)) == 1
|
|
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
|
|
assert "2 piece(s) missing" in T.read_gaps(c)[0].note
|
|
logs = {
|
|
t
|
|
for (t,) in c.execute(
|
|
"SELECT table_name FROM cms.ingest_log WHERE module = 'market.edgar'"
|
|
).fetchall()
|
|
}
|
|
assert {"market.filing_hit", "market.filing_passage", "market.entity_link"} <= logs
|
|
assert published == [True] and store.count() == 1
|
|
# a rerun is idempotent and downloads nothing new
|
|
n_docs = _fts_net(monkeypatch, forbid_docs=True)
|
|
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
|
|
assert (
|
|
_count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1
|
|
)
|
|
assert n_docs == ["gone.htm"] # a 404 is retried, never stored
|
|
|
|
|
|
def test_fts_cli_refusal_prints_gap(env, monkeypatch):
|
|
c, _, published, _ = env
|
|
_fts_net(monkeypatch, status=403)
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
|
|
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
|
|
assert _count(c, "market.filing_hit") == 0 and _count(c, "market.entity") == 0
|
|
assert published == [True]
|
|
|
|
|
|
def test_fts_cli_withholds_ama_passages_and_drops_unsupported_links(env, monkeypatch):
|
|
c, _, _, _ = env
|
|
doc = f"<p>The Medicare Physician Fee Schedule: {LICENSED}.</p>".encode()
|
|
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": doc})
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "passages kept 0, withheld by the AMA guard 1" in res.output
|
|
assert "fts links 0 (none)" in res.output
|
|
assert T.read_links(c, basis="fts") == [] and T.read_passages(c) == []
|
|
assert len(T.read_hits(c)) == 1 # the hit stays
|
|
|
|
|
|
def test_fts_cli_unverifiable_quote_drops_link(env, monkeypatch):
|
|
c, _, _, hoard = env
|
|
_fts_net(monkeypatch, hits=HITS[:1])
|
|
real = mk._verified
|
|
|
|
def tamper(links, passages):
|
|
for p in passages.values():
|
|
p.__dict__["passage"] = "not in the filing"
|
|
return real(links, passages)
|
|
|
|
monkeypatch.setattr(mk, "_verified", tamper)
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert "fts links 0" in res.output and "quote not found in the hoard" in res.output
|
|
assert T.read_links(c, basis="fts") == []
|
|
|
|
|
|
def test_verified_handles_missing_filing_and_unpassaged_links(env):
|
|
_, _, _, hoard = env
|
|
link = T.LinkRow(
|
|
"1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "A:b.htm:0"
|
|
)
|
|
p = T.PassageRow("A:b.htm:0", "1", "A", "b.htm", "t", 0, "x")
|
|
assert mk._verified([link], {("A:b.htm:0", "t"): p}) == ([], 1)
|
|
bare = T.LinkRow("1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "")
|
|
assert mk._verified([bare], {}) == ([], 1)
|
|
|
|
|
|
def test_fts_cli_limit_defers_downloads_and_writes_nothing(env, monkeypatch):
|
|
c, _, published, _ = env
|
|
hits = [
|
|
HITS[0],
|
|
_hit_json(
|
|
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
|
|
),
|
|
]
|
|
_fts_net(monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET, "b.htm": SNIPPET})
|
|
res = runner.invoke(app, [*FTS, "--limit", "1", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "1 filing(s) not downloaded yet" in res.output
|
|
assert "incomplete (limit or time): nothing written" in res.output
|
|
assert published == []
|
|
# the newest filing was fetched first
|
|
assert "a10k.htm" in str(list(http.hoard("sec", "docs").rglob("*.htm")))
|
|
|
|
|
|
def test_fts_cli_deadline_stops_fetching(env, monkeypatch):
|
|
c, _, published, _ = env
|
|
_fts_net(monkeypatch)
|
|
ticks = iter(range(0, 10_000, 100))
|
|
monkeypatch.setattr(mk.time, "monotonic", lambda: next(ticks))
|
|
res = runner.invoke(app, [*FTS, "--max-minutes", "1", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "incomplete" in res.output and published == []
|
|
|
|
|
|
def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch):
|
|
c, _, _, _ = env
|
|
_fts_net(monkeypatch)
|
|
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
|
|
(p,) = [p for p in T.read_passages(c) if p.passage.startswith("Substantially")]
|
|
c.execute(
|
|
"UPDATE market.filing_passage SET label = 'risk_factor', label_version = 'v' "
|
|
"WHERE passage_id = ?",
|
|
[p.passage_id],
|
|
)
|
|
_fts_net(monkeypatch, forbid_docs=True)
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert res.exit_code == 0 and "new entities 0" in res.output
|
|
got = {q.passage_id: q for q in T.read_passages(c)}
|
|
assert (
|
|
got[p.passage_id].label == "risk_factor"
|
|
and got[p.passage_id].label_version == "v"
|
|
)
|
|
(link,) = T.read_links(c, basis="fts")
|
|
assert link.strength == "risk_factor" and link.passage_id == p.passage_id
|
|
|
|
|
|
def test_fts_cli_missing_submissions_is_a_gap(env, monkeypatch):
|
|
c, _, _, _ = env
|
|
_fts_net(monkeypatch, hits=HITS[:1])
|
|
_online(
|
|
monkeypatch,
|
|
[
|
|
(
|
|
r"efts\.sec\.gov/.*",
|
|
lambda m: (
|
|
httpx.Response(
|
|
200, json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
|
|
)
|
|
if "Medicare+Physician" in m.group(0)
|
|
or "Medicare%20Physician" in m.group(0)
|
|
else httpx.Response(
|
|
200, json={"hits": {"total": {"value": 0}, "hits": []}}
|
|
)
|
|
),
|
|
),
|
|
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
|
|
(
|
|
r"company_tickers\.json",
|
|
lambda m: httpx.Response(
|
|
200, content=(FX / "company_tickers.json").read_bytes()
|
|
),
|
|
),
|
|
],
|
|
)
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "CIK0000012345: no submissions, no entity" in res.output
|
|
assert T.read_entities(c) == []
|
|
|
|
|
|
# classification: the local model, resumable through the ledger
|
|
class _Ask:
|
|
model = "qwen-test"
|
|
|
|
def __init__(self, answers):
|
|
self.answers = answers
|
|
self.calls = 0
|
|
|
|
def __call__(self, system, user, schema):
|
|
self.calls += 1
|
|
return self.answers.pop(0)
|
|
|
|
|
|
TWO = [
|
|
HITS[0],
|
|
_hit_json("0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"),
|
|
]
|
|
TWO_DOCS = {"a10k.htm": SNIPPET, "b.htm": SNIPPET}
|
|
|
|
|
|
def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path):
|
|
c, _, _, _ = env
|
|
monkeypatch.setattr(
|
|
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / f"{task}-{fam}.jsonl"
|
|
)
|
|
# a passage of a term outside this run's scope is never sent to the model
|
|
T.write_passages(
|
|
c, [T.PassageRow("X:y.htm:1", "9", "X", "y.htm", "other term", 1, "z")]
|
|
)
|
|
ask = _Ask(
|
|
[
|
|
{"label": "revenue_exposure", "reason": "r"},
|
|
None,
|
|
{"label": "incidental", "reason": "i"},
|
|
]
|
|
)
|
|
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
|
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
|
|
assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first
|
|
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "classified with qwen-test: {'done': 1" in res.output and ask.calls == 1
|
|
labelled = [p for p in T.read_passages(c) if p.label]
|
|
assert len(labelled) == 1 and labelled[0].label == "revenue_exposure"
|
|
assert labelled[0].label_version == "model-reading exposure-v1 qwen-test"
|
|
# resume: the done unit is skipped; the next call fails (None) and stays unlabelled
|
|
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
|
assert res.exit_code == 0 and ask.calls == 2
|
|
assert "'skipped': 1" in res.output and "'failed': 1" in res.output
|
|
# a rerun without --classify keeps what is stored
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert len([p for p in T.read_passages(c) if p.label]) == 1
|
|
got = sorted(l.strength for l in T.read_links(c, basis="fts"))
|
|
assert got == ["mention", "revenue_exposure"]
|
|
|
|
|
|
def test_fts_cli_incidental_label_makes_no_link(env, monkeypatch, tmp_path):
|
|
c, _, _, _ = env
|
|
monkeypatch.setattr(
|
|
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
|
|
)
|
|
ask = _Ask([{"label": "incidental", "reason": "i"}] * 2)
|
|
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
|
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
|
|
res = runner.invoke(app, [*FTS, "--classify", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert T.read_links(c, basis="fts") == [] and len(T.read_passages(c)) == 2
|
|
|
|
|
|
def test_asker_builds_json_extractor(monkeypatch):
|
|
sentinel = object()
|
|
seen = {}
|
|
monkeypatch.setattr("llm.config.load", lambda: "cfg")
|
|
monkeypatch.setattr(
|
|
"llm.pool.HostPool.from_config", classmethod(lambda cls, cfg: ("pool", cfg))
|
|
)
|
|
|
|
def make(cfg, pool):
|
|
seen["args"] = (cfg, pool)
|
|
return sentinel
|
|
|
|
monkeypatch.setattr("llm.structured.json_extractor", make)
|
|
assert mk._asker() is sentinel
|
|
assert seen["args"] == ("cfg", ("pool", "cfg"))
|
|
|
|
|
|
def test_gap_note_truncates():
|
|
note = mk._gap_note([f"i{n}" for n in range(20)])
|
|
assert note.startswith("20 piece(s) missing: i0; i1") and "... 5 more" in note
|
|
|
|
|
|
def test_fts_cli_submissions_refusal_is_a_gap(env, monkeypatch):
|
|
c, _, published, _ = env
|
|
_fts_net(monkeypatch, hits=HITS[:1])
|
|
_online(
|
|
monkeypatch,
|
|
[
|
|
(
|
|
r"efts\.sec\.gov/.*",
|
|
lambda m: httpx.Response(
|
|
200,
|
|
json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
|
|
if "Medicare%20Physician" in m.group(0)
|
|
or "Medicare+Physician" in m.group(0)
|
|
else {"hits": {"total": {"value": 0}, "hits": []}},
|
|
),
|
|
),
|
|
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
|
|
(r"CIK\d{10}\.json", lambda m: httpx.Response(403)),
|
|
(r"company_tickers\.json", lambda m: httpx.Response(403)),
|
|
],
|
|
)
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
|
|
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
|
|
assert _count(c, "market.filing_hit") == 0 and published == [True]
|
|
|
|
|
|
def test_fts_cli_long_gap_list_is_abridged(env, monkeypatch):
|
|
c, _, _, _ = env
|
|
hits = [
|
|
_hit_json("0000012346", f"0000012346-23-{n:06d}", "x.pdf", ["8082"])
|
|
for n in range(22)
|
|
]
|
|
_fts_net(monkeypatch, hits=hits)
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "gap: sec-edgar:fts: ... 2 more" in res.output
|
|
assert "... 7 more" in T.read_gaps(c)[0].note
|
|
|
|
|
|
def test_fts_cli_flaky_document_is_skipped(env, monkeypatch):
|
|
hits = [
|
|
HITS[0],
|
|
_hit_json(
|
|
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
|
|
),
|
|
]
|
|
_fts_net(
|
|
monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET}, broken={"b.htm": "timeout"}
|
|
)
|
|
monkeypatch.setattr("time.sleep", lambda s: None)
|
|
res = runner.invoke(app, FTS)
|
|
assert res.exit_code == 0, res.output
|
|
assert "b.htm: sec-edgar: HTTP 0 ReadTimeout" in res.output
|
|
assert "passages kept 1," in res.output
|
|
|
|
|
|
def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch):
|
|
c, _, published, _ = env
|
|
_fts_net(monkeypatch, hits=HITS[:1], broken={"a10k.htm": 403})
|
|
res = runner.invoke(app, [*FTS, "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "gap: sec-edgar:fts: sec-edgar: HTTP 403" in res.output
|
|
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [
|
|
True
|
|
]
|