Files
stack/tests/cli/test_market_cli.py
kert 6a45806759
Some checks failed
CI / lint (push) Successful in 57s
CI / notebooks-smoke (push) Successful in 2m45s
Deploy / changes (push) Successful in 18s
Infra CI / notebooks (push) Successful in 1m14s
CI / test (push) Failing after 5m26s
Infra CI / zotero (push) Successful in 21s
Infra CI / docs (push) Successful in 1m35s
Infra CI / api (push) Successful in 1m22s
Infra CI / llm (push) Successful in 1m14s
Infra CI / mc (push) Successful in 36s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
feat(market): EDGAR full-text exposure passages per program and family, cited and AMA-guarded
2026-11-13 02:19:01 -05:00

1513 lines
54 KiB
Python

"""stack market — entities, gaps."""
from __future__ import annotations
import json
import re
from datetime import date
from pathlib import Path
import duckdb
import httpx
import pytest
from typer.testing import CliRunner
import cli.market as mk
from bib.store import Store
from cli import app
from market import edgar, http
from market import tables as T
from market.seed import SEAT_LABEL_KEYS, load_seed
from pfs.codetables import SeatRow, write_seats
runner = CliRunner()
FX = Path(__file__).parent.parent / "market" / "fixtures"
class _Pacer:
def wait(self):
pass
def _seat(org, specialty, kind="specialty", year=2024):
return SeatRow("RUC", year, org, specialty, "permanent", "u", "K", "s", kind)
def _specialties() -> set[str]:
companies, orgs = load_seed()
out = {o.specialty for o in orgs if o.specialty}
out |= {
link.target
for e in (*companies, *orgs)
for link in e.links
if link.scope == "specialty"
}
return out - set(SEAT_LABEL_KEYS)
def _build_db() -> duckdb.DuckDBPyConnection:
c = duckdb.connect(":memory:")
c.execute("CREATE SCHEMA terminology")
c.execute(
"CREATE TABLE terminology.cms_acceptable_provider_specialty_codes "
"(specialty_description VARCHAR)"
)
c.executemany(
"INSERT INTO terminology.cms_acceptable_provider_specialty_codes VALUES (?)",
[(s.title(),) for s in sorted(_specialties())],
)
c.execute("CREATE SCHEMA pfs")
c.execute("CREATE TABLE pfs.cpt_code (code VARCHAR, descriptor VARCHAR)")
c.execute(
"INSERT INTO pfs.cpt_code VALUES ('99999', 'a licensed descriptor that no "
"filing quote should ever copy word for word')"
)
write_seats(
c,
"RUC",
[
_seat("Family Medicine", "FAMILY PRACTICE"),
_seat("Cardiology", "CARDIOLOGY"),
_seat("RUC Chair", "", "category"),
],
)
return c
class _Batch:
def __init__(self, c):
self.c = c
def __enter__(self):
return self.c
def __exit__(self, *a):
return False
@pytest.fixture
def env(monkeypatch, tmp_path):
c = _build_db()
monkeypatch.setattr(mk, "_batch", lambda: _Batch(c))
monkeypatch.setattr(mk, "_read", lambda: c.cursor())
store = Store(":memory:", storage_dir=tmp_path / "st")
monkeypatch.setattr(mk, "_store", lambda: store)
published: list[bool] = []
monkeypatch.setattr(mk, "_publish", lambda: published.append(True))
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
monkeypatch.setattr(edgar, "new_pacer", lambda: _Pacer())
try:
yield c, store, published, tmp_path / "market"
finally:
store.close()
c.close()
def _cache_offline(hoard: Path) -> None:
sec = hoard / "sec"
(sec / "submissions").mkdir(parents=True)
(sec / "company_tickers.json").write_bytes(
(FX / "company_tickers.json").read_bytes()
)
for name in (
"submissions_CIK0001564406.json",
"CIK0001564406-submissions-001.json",
):
target = name.replace("submissions_", "")
if name.startswith("submissions_"):
target = "CIK0001564406.json"
(sec / "submissions" / target).write_bytes((FX / name).read_bytes())
_cache_filings(hoard)
def _cache_filings(hoard: Path, companies=None) -> None:
"""A hoarded filing per cited accession, holding exactly the quotes the
seed cites to it."""
import html
companies = companies or load_seed()[0]
quotes: dict[tuple[str, str], list[str]] = {}
for _who, c, cite in edgar.filing_cites(companies):
if cite.adsh:
quotes.setdefault((c.cik, cite.adsh), []).append(cite.quote)
for (cik, adsh), qs in quotes.items():
p = hoard / "sec" / "filings" / cik / f"{adsh}.htm"
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(
"<html><body>"
+ "".join(f"<p>{html.escape(q)}</p>" for q in qs)
+ "</body></html>"
)
def _count(c, table):
return c.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
def test_entities_offline_writes_and_logs(env):
c, store, published, hoard = env
_cache_offline(hoard)
# a stale curated link the seed no longer carries
T.write_links(
c, [T.LinkRow("0001564406", "program", "QPP-MIPS", "curated", "x", "url", "u")]
)
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 0, res.output
companies, orgs = load_seed()
assert _count(c, "market.entity") == len(companies) + len(orgs)
osh = next(e for e in T.read_entities(c, "company") if e.key == "0001564406")
assert osh.status == "acquired" and osh.ended_on == date(2023, 5, 2)
assert osh.sic == "8000" and osh.item_key
assert "no submissions for AMED" in res.output
# the five fixture filings, and the seed's dated ticker
assert len(T.read_filings(c, "0001564406")) == 5
assert [s.source for s in T.read_securities(c, "0001564406")] == ["seed"]
links = T.read_links(c, basis="curated")
assert all(l.item_key for l in links)
assert ("program", "QPP-MIPS") not in {(l.scope, l.target) for l in links}
assert [l.target for l in T.read_links(c, basis="sic")] == ["8000"]
logs = c.execute(
"SELECT table_name, rows FROM cms.ingest_log WHERE module = 'market.edgar' "
"ORDER BY table_name"
).fetchall()
assert [t for t, _ in logs] == [
"market.entity",
"market.entity_link",
"market.filing",
"market.security",
]
assert published == [True]
assert "retired 1" in res.output
assert store.count() > 0
def test_entities_rerun_is_idempotent(env):
c, _, published, hoard = env
_cache_offline(hoard)
for _ in range(2):
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
companies, orgs = load_seed()
assert _count(c, "market.entity") == len(companies) + len(orgs)
assert _count(c, "market.filing") == 5
def test_entities_no_write_prints_counts(env, monkeypatch):
c, _, published, hoard = env
_cache_offline(hoard)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 0, res.output
companies, orgs = load_seed()
assert f"entities {len(companies) + len(orgs)}" in res.output
assert (
"company/acquired" in res.output
and "organization/specialty_society" in res.output
)
assert "dry run" in res.output and published == []
assert "warning" not in res.output # every seat specialty has a society
def test_entities_warns_about_uncovered_seat_specialty(env):
c, _, _, hoard = env
_cache_offline(hoard)
write_seats(c, "RUC", [_seat("Podiatry", "PODIATRY", year=2025)])
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 0
assert "seat specialties without a society: ['PODIATRY']" in res.output
def _handler(routes):
def handler(request: httpx.Request):
url = str(request.url)
for pattern, make in routes:
m = re.search(pattern, url)
if m:
return make(m)
return httpx.Response(404)
return handler
def _online(monkeypatch, routes):
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(_handler(routes))
),
)
def test_entities_refusal_prints_gap(env, monkeypatch):
c, _, published, hoard = env
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
res = runner.invoke(app, ["market", "entities", "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:entities:" in res.output and "403" in res.output
assert _count(c, "market.entity") == 0 and _count(c, "market.filing") == 0
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:entities"]
assert published == [True]
def test_entities_refusal_without_write_records_nothing(env, monkeypatch):
c, _, published, _ = env
_online(monkeypatch, [("", lambda m: httpx.Response(403))])
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "entities"])
assert res.exit_code == 0 and "gap: sec-edgar:entities:" in res.output
assert published == []
def _sub_for(cik: str) -> dict:
return {
"cik": cik,
"name": f"Filer {cik}",
"sic": "8082",
"sicDescription": "Home Health",
"tickers": [],
"formerNames": [],
"filings": {
"recent": {
"accessionNumber": ["A-1", "A-2"],
"filingDate": ["2024-01-01", "2024-01-02"],
"reportDate": ["", ""],
"acceptanceDateTime": ["", ""],
"form": ["10-K", "4"],
"items": ["", ""],
"primaryDocument": ["a.htm", "b.xml"],
},
"files": [],
},
}
def test_sic_sweep_registers_filers_and_sic_links(env, monkeypatch):
c, _, published, hoard = env
_cache_filings(hoard)
atom = (FX / "sic_8082.atom").read_bytes().replace(b"0001468328", b"0000000042")
seen: list[str] = []
def sub(m):
seen.append(m.group(1))
cik = m.group(1)
if cik == "0001564406":
return httpx.Response(
200, content=(FX / "submissions_CIK0001564406.json").read_bytes()
)
if cik == "0000000042":
return httpx.Response(404) # no submissions file: skipped
return httpx.Response(200, json=_sub_for(cik))
def page(m):
return httpx.Response(
200, content=(FX / "CIK0001564406-submissions-001.json").read_bytes()
)
_online(
monkeypatch,
[
(
r"company_tickers\.json",
lambda m: httpx.Response(
200, content=(FX / "company_tickers.json").read_bytes()
),
),
(r"SIC=\d+.*start=0", lambda m: httpx.Response(200, content=atom)),
(r"submissions-001\.json", page),
(r"CIK(\d{10})\.json", sub),
],
)
res = runner.invoke(app, ["market", "entities", "--sic-sweep", "--write"])
assert res.exit_code == 0, res.output
assert res.output.count("SIC ") == len(edgar.HEALTH_SICS)
keys = {e.key for e in T.read_entities(c, "company")}
assert {"0001725255", "0001468328"} <= keys and "0000000042" not in keys
# a sweep filer keeps only the sweep forms; the seed's filings keep everything
assert [f.form for f in T.read_filings(c, "0001725255")] == ["10-K"]
assert len(T.read_filings(c, "0001468328")) == 2
assert len(T.read_filings(c, "0001564406")) == 5
sic = T.read_links(c, basis="sic")
assert {l.target for l in sic} == {"8082", "8000"}
assert any(l.entity_key == "0001725255" for l in sic)
def test_entities_seed_errors_abort(env, monkeypatch):
c, _, published, _ = env
companies, orgs = load_seed()
bad = companies[0].__class__(
**{
**companies[0].__dict__,
"links": (
companies[0].links[0].__class__("family", "NOPE", companies[0].cite),
),
}
)
monkeypatch.setattr(mk, "load_seed", lambda: ((bad, *companies[1:]), orgs))
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 1 and "seed error: OSH" in res.output
assert c.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
).fetchone() == (0,)
assert published == []
def test_entities_missing_vocabulary_tables_abort(env, monkeypatch):
bare = duckdb.connect(":memory:")
monkeypatch.setattr(mk, "_read", lambda: bare)
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 1 and "cannot validate the seed" in res.output
bare.close()
def test_entities_other_read_errors_propagate(env, monkeypatch):
def boom():
raise RuntimeError("disk on fire")
monkeypatch.setattr(mk, "_read", boom)
res = runner.invoke(app, ["market", "entities", "--offline"])
assert isinstance(res.exception, RuntimeError)
def test_require_key_missing_writes_only_gap(env, monkeypatch):
c, _, published, _ = env
monkeypatch.delenv("TIINGO_API_KEY", raising=False)
assert mk._require_key("tiingo", "tiingo:prices", write=True) is None
assert [(g.source, g.note) for g in T.read_gaps(c)] == [
("tiingo:prices", "TIINGO_API_KEY not in .env")
]
assert all(_count(c, f"market.{t}") == 0 for t in ("entity", "security", "filing"))
assert published == [True]
def test_require_key_present_returns_it(env, monkeypatch):
monkeypatch.setenv("TIINGO_API_KEY", "abc")
assert mk._require_key("tiingo", "tiingo:prices", write=True) == "abc"
def test_require_key_missing_prints_gap_without_write(env, monkeypatch, capsys):
monkeypatch.delenv("HUD_API_TOKEN", raising=False)
assert mk._require_key("hud", "hud:hud", write=False) is None
assert "gap: hud:hud: HUD_API_TOKEN not in .env" in capsys.readouterr().out
c, _, published, _ = env
assert published == []
def test_gap_cleared_after_success(env):
c, _, _, hoard = env
_cache_offline(hoard)
T.write_gap(c, "sec-edgar:entities", "403")
T.write_gap(c, "tiingo:prices", "no key")
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
assert {g.source for g in T.read_gaps(c)} == {
"tiingo:prices",
"web:renalmd.org",
"web:asco.org",
"web:ahip.org",
}
def test_gaps_command_lists_rows(env):
c, _, _, _ = env
T.write_gap(c, "tiingo:prices", "TIINGO_API_KEY not in .env")
res = runner.invoke(app, ["market", "gaps"])
assert res.exit_code == 0
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
def test_gaps_command_empty_and_missing_tables(env, monkeypatch):
c, _, _, _ = env
T.ensure_tables(c)
assert "no gaps recorded" in runner.invoke(app, ["market", "gaps"]).output
bare = duckdb.connect(":memory:")
monkeypatch.setattr(mk, "_read", lambda: bare)
assert "no market tables yet" in runner.invoke(app, ["market", "gaps"]).output
class _Broken:
def execute(self, *a, **k):
raise RuntimeError("x")
def close(self):
pass
monkeypatch.setattr(mk, "_read", lambda: _Broken())
assert isinstance(runner.invoke(app, ["market", "gaps"]).exception, RuntimeError)
bare.close()
def test_default_indirections_resolve_lazily(monkeypatch):
import conf.connect as cc
calls: list[str] = []
monkeypatch.setattr(cc, "duckdb_batch", lambda n: calls.append(f"batch {n}") or n)
monkeypatch.setattr(
cc, "duckdb", lambda n, read_only=True: calls.append(f"read {n}") or n
)
monkeypatch.setattr(cc, "bib", lambda: calls.append("bib") or "store")
monkeypatch.setattr(
cc, "publish_replica", lambda n: calls.append(f"pub {n}") or "p"
)
assert mk._batch() == "aco" and mk._read() == "aco" and mk._store() == "store"
mk._publish()
assert calls == ["batch aco", "read aco", "bib", "pub aco"]
def test_entities_aborts_when_a_quote_is_not_in_the_filing(env):
c, _, published, hoard = env
_cache_offline(hoard)
victim = next(hoard.glob("sec/filings/*/*.htm"))
victim.write_text("<p>nothing the seed quotes</p>")
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 1 and "quote error:" in res.output
assert published == []
assert c.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
).fetchone() == (0,)
def test_entities_aborts_when_a_cited_filing_is_not_hoarded(env):
c, _, published, hoard = env
_cache_offline(hoard)
next(hoard.glob("sec/filings/*/*.htm")).unlink()
res = runner.invoke(app, ["market", "entities", "--offline"])
assert res.exit_code == 1 and "not hoarded" in res.output
def test_entities_fetches_cited_filings_online(env, monkeypatch):
c, _, _, hoard = env
_cache_offline(hoard)
victim = next(hoard.glob("sec/filings/*/*.htm"))
cik, adsh = victim.parent.name, victim.stem
body = victim.read_bytes()
victim.unlink()
(hoard / "sec" / "submissions" / f"CIK{cik}.json").write_text(
json.dumps(
_sub_for(cik)
| {
"filings": {
"recent": {
**_sub_for(cik)["filings"]["recent"],
"accessionNumber": [adsh, "Z"],
"primaryDocument": ["doc.htm", "z.htm"],
},
"files": [],
}
}
)
)
seen = []
def serve(m):
seen.append(m.group(0))
return httpx.Response(200, content=body)
_online(monkeypatch, [(r"/Archives/edgar/data/\d+/\d+/doc\.htm", serve)])
res = runner.invoke(app, ["market", "entities"])
assert res.exit_code == 0, res.output
assert len(seen) == 1 and victim.exists()
def test_entities_manifest_and_ingest_log_sources(env):
c, _, _, hoard = env
_cache_offline(hoard)
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
manifest = hoard / "MANIFEST.jsonl"
assert manifest.exists() and manifest.read_text().count("\n") >= 6
rows = c.execute(
"SELECT table_name, source_file, sha256 FROM cms.ingest_log WHERE module = 'market.edgar'"
).fetchall()
assert len(rows) == 4 and all(sf and sha and len(sha) == 64 for _, sf, sha in rows)
def test_entities_records_unreachable_site_gaps_and_pseudo_orgs(env):
c, _, _, hoard = env
_cache_offline(hoard)
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
gaps = {g.source for g in T.read_gaps(c)}
assert {"web:renalmd.org", "web:asco.org", "web:ahip.org"} <= gaps
assert "sec-edgar:entities" not in gaps
pseudo = {e.key for e in T.read_entities(c, "organization") if e.pseudo}
assert pseudo == {"PRIMARY-CARE-SEAT", "ONCOLOGY-SEAT"}
def test_entities_retires_organizations_the_seed_dropped(env):
c, _, _, hoard = env
_cache_offline(hoard)
T.write_entities(c, [T.EntityRow("ASCO", "ASCO", "organization")])
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
assert "ASCO" not in {e.key for e in T.read_entities(c, "organization")}
def test_entities_aborts_when_a_quote_copies_a_cpt_descriptor(env):
c, _, published, hoard = env
_cache_offline(hoard)
victim = load_seed()[0][0].cite.quote
c.execute("DELETE FROM pfs.cpt_code")
c.execute("INSERT INTO pfs.cpt_code VALUES ('1', ?)", [victim])
res = runner.invoke(app, ["market", "entities", "--offline", "--write"])
assert res.exit_code == 1 and "ama error: OSH" in res.output
assert published == []
assert c.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'market'"
).fetchone() == (0,)
def test_entities_stores_ticker_history_and_link_dates(env):
c, _, _, hoard = env
_cache_offline(hoard)
assert (
runner.invoke(app, ["market", "entities", "--offline", "--write"]).exit_code
== 0
)
assert [
(s.ticker, s.valid_to, s.priced) for s in T.read_securities(c, "0001156039")
] == [("ANTM", date(2022, 6, 27), False), ("ELV", None, True)]
assert [s.priced for s in T.read_securities(c, "0001739940")] == [False]
(clov,) = [
l
for l in T.read_links(c, basis="curated", target="APM")
if l.entity_key == "0001801170"
]
assert clov.valid_to == date(2023, 12, 31)
# ── stack market prices ──────────────────────────────────────────────
import io # noqa: E402
import zipfile # noqa: E402
from market import tiingo # noqa: E402
DAY = date(2026, 11, 12)
def _supported_zip(*rows: str) -> bytes:
head = "ticker,exchange,assetType,priceCurrency,startDate,endDate\n"
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as z:
z.writestr("supported_tickers.csv", head + "\n".join(rows) + "\n")
return buf.getvalue()
def _eod(day: str, close: float, adj: float | None = None) -> dict:
return {
"date": f"{day}T00:00:00.000Z",
"open": close,
"high": close,
"low": close,
"close": close,
"volume": 5,
"adjClose": close if adj is None else adj,
"divCash": 0.0,
"splitFactor": 1.0,
}
@pytest.fixture
def px(env, monkeypatch, tmp_path):
c, store, published, hoard = env
monkeypatch.setenv("TIINGO_API_KEY", "secret-key")
monkeypatch.setattr(tiingo, "state_dir", lambda: tmp_path / "state")
monkeypatch.setattr(tiingo, "new_pacer", lambda: _Pacer())
monkeypatch.setattr(tiingo, "today", lambda: DAY)
seen: list[httpx.Request] = []
series = {
"AAA": [
_eod("2024-01-02", 100),
_eod("2024-01-03", 110),
_eod("2024-01-04", 99),
],
"BBB": [_eod("2024-01-02", 10), _eod("2024-01-03", 11)],
"SPY": [_eod("2024-01-02", 400), _eod("2024-01-03", 404)],
"XLV": [_eod("2024-01-02", 100)],
"IHF": [_eod("2024-01-02", 50)],
}
z = _supported_zip(
"AAA,NYSE,Stock,USD,2000-01-01,2026-11-12",
"BBB,NYSE,Stock,USD,2000-01-01,2026-11-12",
"SPY,NYSE,ETF,USD,1993-01-29,2026-11-12",
"XLV,NYSE,ETF,USD,1998-12-22,2026-11-12",
"IHF,NYSE,ETF,USD,2006-05-05,2026-11-12",
"ZZZ,NYSE,Stock,USD,2000-01-01,2026-11-12",
)
def prices_for(m):
if m.group(1) not in series:
return httpx.Response(404, json={"detail": "Not found."})
return httpx.Response(200, json=series[m.group(1)])
routes = [
(r"supported_tickers\.zip", lambda m: httpx.Response(200, content=z)),
(r"/daily/(\w+)/prices", prices_for),
]
def handler(request):
seen.append(request)
return _handler(routes)(request)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(handler),
headers={"Authorization": f"Token {key}"} if key else {},
),
)
return c, store, published, hoard, seen, series, tmp_path / "state"
def _price_requests(seen):
return [r for r in seen if "/prices" in r.url.path]
def test_prices_cli_fetch_write(px):
c, _, published, hoard, seen, _, state = px
res = runner.invoke(
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
)
assert res.exit_code == 0, res.output
assert _count(c, "market.price_daily") == 5
assert [r.url.params["startDate"] for r in _price_requests(seen)] == [
"1993-01-29"
] * 2
assert all("secret-key" not in str(r.url) for r in seen)
keyed = [r for r in seen if "/prices" in r.url.path]
assert all(r.headers["authorization"] == "Token secret-key" for r in keyed)
free = [r for r in seen if "supported" in r.url.path]
assert free and "authorization" not in free[0].headers
assert {
(t, n)
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
} == {("market.price_daily", 5), ("market.series", 2)}
assert [s.value for s in T.read_series(c, ["AAA"])] == [pytest.approx(99.0)]
assert published == [True]
used = json.loads((state / "tiingo-2026-11.json").read_text())
assert used["symbols"] == ["AAA", "BBB"] and used["day_counts"] == {"2026-11-12": 2}
assert (hoard / "tiingo" / "eod" / "AAA" / "1993-01-29_2026-11-12.json").is_file()
assert "priced 2 symbols, 5 rows" in res.output and "2/450" in res.output
assert T.read_gaps(c) == []
def test_prices_incremental_pull_and_same_day_skip(px, monkeypatch):
c, _, _, _, seen, series, _ = px
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
assert runner.invoke(app, cmd).exit_code == 0
before = len(_price_requests(seen))
again = runner.invoke(app, cmd)
assert len(_price_requests(seen)) == before # priced today: nothing to do
assert "priced 0 symbols" in again.output
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
series["AAA"] = [_eod("2024-01-04", 99, 49.5), _eod("2024-01-05", 108.9, 54.45)]
assert runner.invoke(app, cmd).exit_code == 0
last = _price_requests(seen)[-1]
assert last.url.params["startDate"] == "2024-01-04"
rows = {r.day: r for r in T.read_prices(c, "AAA")}
assert len(rows) == 4 and rows[date(2024, 1, 4)].adj_close == 99
assert rows[date(2024, 1, 5)].ret == pytest.approx(0.1)
assert [m.value for m in T.read_series(c, ["AAA"])] == [pytest.approx(108.9)]
def test_prices_benchmarks_start_at_listing(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(app, ["market", "prices", "--benchmarks", "--write"])
assert res.exit_code == 0, res.output
starts = {
r.url.path.split("/")[3]: r.url.params["startDate"]
for r in _price_requests(seen)
}
assert starts == {"SPY": "1993-01-29", "XLV": "1998-12-22", "IHF": "2006-05-05"}
def test_prices_dry_run_reports_budget(px, monkeypatch):
c, _, published, _, seen, _, state = px
T.ensure_tables(c)
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
T.write_links(c, [T.LinkRow("K1", "program", "PFS", "fts", "mention", "adsh", "a")])
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
assert res.exit_code == 0, res.output
assert "1/450 symbols used, 449 remaining" in res.output
assert "1. SPY" in res.output and "used" in res.output
assert "AAA" in res.output and "K1" in res.output and "first pull" in res.output
assert _price_requests(seen) == [] and published == []
assert not (state / "tiingo-2026-11.json").read_text().count("AAA")
def test_prices_dry_run_without_market_tables_lists_benchmarks(px):
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
assert res.exit_code == 0, res.output
assert "3 symbols ranked" in res.output
def test_prices_dry_run_shows_stored_span(px):
c, *_ = px
T.ensure_tables(c)
T.write_prices(
c,
"AAA",
tiingo.to_prices(
"AAA", [_eod("2024-01-02", 1)], pulled_on=DAY, source_file="f", overlap=None
),
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--dry-run"])
assert "stored to 2024-01-02" in res.output
def test_prices_without_key_records_gap(px, monkeypatch):
c, _, published, _, seen, _, _ = px
monkeypatch.delenv("TIINGO_API_KEY")
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"]
assert _count(c, "market.price_daily") == 0 and seen == [] and published == [True]
def test_prices_refusal_prints_gap(px, monkeypatch):
c, _, published, _, _, _, _ = px
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(lambda r: httpx.Response(403))
),
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "gap: tiingo:prices:" in res.output and "403" in res.output
assert _count(c, "market.price_daily") == 0
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"] and published == [
True
]
def test_prices_refusal_on_prices_after_supported_ok(px, monkeypatch):
c, _, published, _, _, _, _ = px
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
routes = [
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
(r"/prices", lambda m: httpx.Response(401)),
]
_online(monkeypatch, routes)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA"])
assert (
res.exit_code == 0
and "gap: tiingo:prices:" in res.output
and "401" in res.output
)
assert published == []
def test_prices_nothing_selected_exits(px):
res = runner.invoke(app, ["market", "prices"])
assert res.exit_code == 2 and "nothing to price" in res.output
def test_prices_unknown_ticker_and_window_gap_become_gap_note(px):
c, _, _, _, _, series, _ = px
T.ensure_tables(c)
T.write_securities(
c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, date(2030, 1, 1), "s")]
)
res = runner.invoke(
app, ["market", "prices", "--ticker", "BBB", "--ticker", "ZZZ", "--write"]
)
assert res.exit_code == 0, res.output
assert "ZZZ: not found at Tiingo" in res.output
(gap,) = T.read_gaps(c)
assert (
"ZZZ: not found at Tiingo" in gap.note
and "BBB: Tiingo ends 2026-11-12" in gap.note
)
# BBB later prices cleanly: its entry goes, ZZZ's stays
T.write_securities(c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, None, "s")])
runner.invoke(app, ["market", "prices", "--ticker", "BBB", "--refresh", "--write"])
(gap,) = T.read_gaps(c)
assert gap.note == "ZZZ: not found at Tiingo"
def test_prices_clean_run_clears_gap_and_window_clips(px):
c, _, _, _, _, _, _ = px
T.ensure_tables(c)
T.write_gap(c, "tiingo:prices", "AAA: not found at Tiingo")
T.write_securities(
c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", date(2024, 1, 3), None, "s")]
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert [r.day for r in T.read_prices(c, "AAA")] == [
date(2024, 1, 3),
date(2024, 1, 4),
]
assert T.read_gaps(c) == []
def test_prices_limit_stops_downloads(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(
app, ["market", "prices", "--benchmarks", "--limit", "1", "--write"]
)
assert res.exit_code == 0 and len(_price_requests(seen)) == 1
assert [r.ticker for r in [*T.read_prices(c, "SPY")][:1]] == ["SPY"]
def test_prices_budget_exhausted_stops_and_writes_what_it_has(px):
c, _, published, _, seen, _, state = px
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps(
{
"month": "2026-11",
"symbols": [f"S{i}" for i in range(449)] + ["AAA"],
"day_counts": {},
}
)
)
res = runner.invoke(
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
)
assert res.exit_code == 0, res.output
assert "budget: stopped, BBB" in res.output
assert [r.ticker for r in T.read_prices(c, "AAA")][:1] == ["AAA"]
assert T.read_prices(c, "BBB") == [] and published == [True]
def test_prices_without_write_stores_nothing(px, monkeypatch):
c, _, published, _, _, _, _ = px
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "prices", "--ticker", "ZZZ"])
assert res.exit_code == 0, res.output
assert (
"dry run: pass --write" in res.output
and "gap: tiingo:prices: ZZZ" in res.output
)
assert published == []
def test_merge_gap_keeps_other_tickers_and_orders():
assert mk._merge_gap("", {}, set()) == ""
assert mk._merge_gap("A: x | B: y", {"C": "z"}, {"A"}) == "B: y | C: z"
def test_or_empty_reraises_other_errors():
def boom(_):
raise RuntimeError("disk")
with pytest.raises(RuntimeError):
mk._or_empty(boom, None, {})
def test_prices_ticker_refuses_unpriced_record_without_a_request(px):
c, _, _, _, seen, _, state = px
T.ensure_tables(c)
T.write_securities(
c, "K9", [T.SecurityRow("K9", "AAA", "NYSE", None, None, "s", False)]
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "skipped AAA: symbol record only" in res.output
assert _price_requests(seen) == [] and _count(c, "market.price_daily") == 0
assert not (state / "tiingo-2026-11.json").exists()
assert T.read_gaps(c) == []
def test_prices_unsupported_named_ticker_is_gap_not_request(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(app, ["market", "prices", "--ticker", "QQQQ", "--write"])
assert "skipped QQQQ: not on Tiingo's supported list" in res.output
assert _price_requests(seen) == []
(gap,) = T.read_gaps(c)
assert gap.note == "QQQQ: not on Tiingo's supported list"
def test_prices_universe_records_unsupported_seed_ticker_gap(px):
c, _, _, _, seen, _, _ = px
T.ensure_tables(c)
T.write_securities(
c, "0001800682", [T.SecurityRow("0001800682", "CANO", "NYSE", None, None, "s")]
)
res = runner.invoke(app, ["market", "prices", "--universe", "--write"])
assert res.exit_code == 0, res.output
assert "skipped CANO: not on Tiingo's supported list" in res.output
assert all("/CANO/" not in r.url.path for r in seen)
(gap,) = T.read_gaps(c)
assert gap.source == "tiingo:prices"
assert gap.note == "CANO: not on Tiingo's supported list"
# rerun with everything already priced today keeps the gap
again = runner.invoke(app, ["market", "prices", "--universe", "--write"])
assert again.exit_code == 0
assert [g.note for g in T.read_gaps(c)] == [gap.note]
def test_prices_missing_overlap_day_repulls_further_back(px, monkeypatch):
c, _, _, _, seen, _, _ = px
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
assert runner.invoke(app, cmd).exit_code == 0
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
def eod(m):
# the 2024-01-04 overlap day is absent from the narrow pull
start = next(r for r in seen[::-1] if "/prices" in r.url.path).url.params[
"startDate"
]
if start == "2024-01-04":
return httpx.Response(200, json=[_eod("2024-01-05", 54.45)])
return httpx.Response(
200, json=[_eod("2024-01-03", 55), _eod("2024-01-05", 60.5)]
)
handler = _handler(
[
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
(r"/daily/AAA/prices", eod),
]
)
def record(request):
seen.append(request)
return handler(request)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(transport=httpx.MockTransport(record)),
)
res = runner.invoke(app, cmd)
assert res.exit_code == 0, res.output
assert "overlap day 2024-01-04 missing" in res.output
assert "re-pulling from 2023-12-25" in res.output
row = {r.day: r for r in T.read_prices(c, "AAA")}[date(2024, 1, 5)]
assert row.ret == pytest.approx(60.5 / 55 - 1)
# ── fts ──────────────────────────────────────────────────────────────
MPFS = '"Medicare Physician Fee Schedule"'
SNIPPET = (FX / "filing_snippet.htm").read_bytes()
LICENSED = "a licensed descriptor that no filing quote should ever copy word for word"
def _hit_json(cik, adsh, filename, sics, date_="2023-03-01"):
return {
"_id": f"{adsh}:{filename}",
"_source": {
"ciks": [cik],
"adsh": adsh,
"form": "10-K",
"file_date": date_,
"period_ending": "2022-12-31",
"sics": sics,
"display_names": [f"Filer {cik} (CIK {cik})"],
},
}
HITS = [
_hit_json("0000012345", "0000012345-23-000001", "a10k.htm", ["8071"]),
_hit_json("0000000002", "0000000002-23-000001", "reit.htm", ["6798"]),
_hit_json("0000012346", "0000012346-23-000001", "ex99.pdf", ["8082"]),
_hit_json("0000012347", "0000012347-23-000001", "gone.htm", ["8082"]),
]
def _fts_net(
monkeypatch, hits=HITS, docs=None, forbid_docs=False, status=200, broken=None
):
"""A mock EDGAR: the term "Medicare Physician Fee Schedule" has *hits* in
2023; every other query is empty."""
docs = docs if docs is not None else {"a10k.htm": SNIPPET}
requested: list[str] = []
def efts(m):
p = dict(httpx.URL(m.group(0)).params)
mine = p["q"] == MPFS and p["from"] == "0"
return httpx.Response(
status,
json={
"hits": {
"total": {"value": len(hits) if mine else 0},
"hits": hits if mine else [],
}
},
)
def doc(m):
requested.append(m.group(1))
if broken and m.group(1) in broken:
if broken[m.group(1)] == "timeout":
raise httpx.ReadTimeout("slow")
return httpx.Response(broken[m.group(1)])
body = docs.get(m.group(1))
if forbid_docs and body:
raise AssertionError("no document download expected")
return httpx.Response(200, content=body) if body else httpx.Response(404)
def sub(m):
return httpx.Response(200, json={**_sub_for(m.group(1)), "tickers": ["ZZZ"]})
_online(
monkeypatch,
[
(r"efts\.sec\.gov/.*", efts),
(r"/Archives/edgar/data/\d+/\d+/([^/?]+)$", doc),
(
r"company_tickers\.json",
lambda m: httpx.Response(
200, content=(FX / "company_tickers.json").read_bytes()
),
),
(r"CIK(\d{10})\.json", sub),
],
)
return requested
FTS = ["market", "fts", "--program", "PFS", "--from", "2023", "--to", "2023"]
def test_fts_cli_dry_run_lists_queries(env, monkeypatch):
c, _, published, hoard = env
monkeypatch.setattr(
http, "client", lambda *a, **k: (_ for _ in ()).throw(AssertionError("net"))
)
res = runner.invoke(app, [*FTS, "--dry-run"])
assert res.exit_code == 0, res.output
assert "program PFS: 5 terms, 5 queries" in res.output
assert "5 queries (0 already hoarded)" in res.output
assert published == []
full = runner.invoke(app, ["market", "fts", "--dry-run"])
assert "program MSSP: 1 terms" in full.output
assert "program CARE-MANAGEMENT: 2 terms" in full.output
assert f"{(5 + 2 + 1 + 1 + 1 + 2 + 2) * (dt_year() - 2000)} queries" in full.output
def dt_year():
import datetime
return datetime.date.today().year
def test_fts_cli_family_scope_and_bad_keys(env, monkeypatch):
monkeypatch.setattr("pfs.families.refresh_from", lambda con: 0)
res = runner.invoke(
app,
[
"market",
"fts",
"--family",
"HOME-VISITS",
"--dry-run",
"--from",
"2023",
"--to",
"2023",
],
)
assert res.exit_code == 0, res.output
assert "family HOME-VISITS: 8 terms, 8 queries" in res.output
assert (
runner.invoke(app, ["market", "fts", "--family", "NOPE", "--dry-run"]).exit_code
== 2
)
assert (
runner.invoke(
app, ["market", "fts", "--program", "NOPE", "--dry-run"]
).exit_code
== 2
)
def test_fts_cli_without_write_stores_nothing(env, monkeypatch):
c, _, published, hoard = env
_fts_net(monkeypatch)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, FTS)
assert res.exit_code == 0, res.output
assert "Medicare Physician Fee Schedule 10-K 2023: 4 hits" in res.output
assert "passages kept 1," in res.output and "new entities 1" in res.output
assert "fts links 1 (PFS 1)" in res.output and "dry run" in res.output
assert (
"gap: sec-edgar:fts: 0000012346-23-000001:ex99.pdf: format not readable"
in res.output
)
assert (
"gap: sec-edgar:fts: 0000012347-23-000001:gone.htm: document not found"
in res.output
)
assert published == []
# hoarded: the pages and the one downloaded readable document
assert (
hoard / "sec/fts/medicare-physician-fee-schedule/10-K/2023.0.json"
).is_file()
assert (hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm").is_file()
def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch):
c, store, published, hoard = env
curated = T.LinkRow(
"0000012345", "program", "APM", "curated", "curated", "adsh", "x", "", "K"
)
T.write_links(c, [curated])
_fts_net(monkeypatch)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
(ent,) = T.read_entities(c, "company")
assert ent.key == "0000012345" and ent.sic == "8082"
assert [s.ticker for s in T.read_securities(c, ent.key)] == ["ZZZ"]
assert [f.form for f in T.read_filings(c, ent.key)] == ["10-K"]
assert [l.target for l in T.read_links(c, basis="sic")] == ["8082"]
assert T.read_links(c, basis="curated") == [curated] # survives
(fts_link,) = T.read_links(c, basis="fts")
assert (fts_link.scope, fts_link.target, fts_link.strength) == (
"program",
"PFS",
"mention",
)
assert fts_link.item_key and fts_link.passage_id.startswith(
"0000012345-23-000001:a10k.htm:"
)
# the link's passage is verbatim in the hoarded filing, short, and cited
(p,) = [p for p in T.read_passages(c) if p.passage_id == fts_link.passage_id]
assert len(p.passage) <= 400 and p.item_key == fts_link.item_key
text = edgar.read_doc(hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm")
assert p.passage in text and p.label == "" and p.label_version == ""
assert len(T.read_hits(c)) == 4 and len(T.read_passages(c)) == 1
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
assert "2 piece(s) missing" in T.read_gaps(c)[0].note
logs = {
t
for (t,) in c.execute(
"SELECT table_name FROM cms.ingest_log WHERE module = 'market.edgar'"
).fetchall()
}
assert {"market.filing_hit", "market.filing_passage", "market.entity_link"} <= logs
assert published == [True] and store.count() == 1
# a rerun is idempotent and downloads nothing new
n_docs = _fts_net(monkeypatch, forbid_docs=True)
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
assert (
_count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1
)
assert n_docs == ["gone.htm"] # a 404 is retried, never stored
def test_fts_cli_refusal_prints_gap(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch, status=403)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
assert _count(c, "market.filing_hit") == 0 and _count(c, "market.entity") == 0
assert published == [True]
def test_fts_cli_withholds_ama_passages_and_drops_unsupported_links(env, monkeypatch):
c, _, _, _ = env
doc = f"<p>The Medicare Physician Fee Schedule: {LICENSED}.</p>".encode()
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": doc})
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "passages kept 0, withheld by the AMA guard 1" in res.output
assert "fts links 0 (none)" in res.output
assert T.read_links(c, basis="fts") == [] and T.read_passages(c) == []
assert len(T.read_hits(c)) == 1 # the hit stays
def test_fts_cli_unverifiable_quote_drops_link(env, monkeypatch):
c, _, _, hoard = env
_fts_net(monkeypatch, hits=HITS[:1])
real = mk._verified
def tamper(links, passages):
for p in passages.values():
p.__dict__["passage"] = "not in the filing"
return real(links, passages)
monkeypatch.setattr(mk, "_verified", tamper)
res = runner.invoke(app, [*FTS, "--write"])
assert "fts links 0" in res.output and "quote not found in the hoard" in res.output
assert T.read_links(c, basis="fts") == []
def test_verified_handles_missing_filing_and_unpassaged_links(env):
_, _, _, hoard = env
link = T.LinkRow(
"1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "A:b.htm:0"
)
p = T.PassageRow("A:b.htm:0", "1", "A", "b.htm", "t", 0, "x")
assert mk._verified([link], {("A:b.htm:0", "t"): p}) == ([], 1)
bare = T.LinkRow("1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "")
assert mk._verified([bare], {}) == ([], 1)
def test_fts_cli_limit_defers_downloads_and_writes_nothing(env, monkeypatch):
c, _, published, _ = env
hits = [
HITS[0],
_hit_json(
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
),
]
_fts_net(monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET, "b.htm": SNIPPET})
res = runner.invoke(app, [*FTS, "--limit", "1", "--write"])
assert res.exit_code == 0, res.output
assert "1 filing(s) not downloaded yet" in res.output
assert "incomplete (limit or time): nothing written" in res.output
assert published == []
# the newest filing was fetched first
assert "a10k.htm" in str(list(http.hoard("sec", "docs").rglob("*.htm")))
def test_fts_cli_deadline_stops_fetching(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch)
ticks = iter(range(0, 10_000, 100))
monkeypatch.setattr(mk.time, "monotonic", lambda: next(ticks))
res = runner.invoke(app, [*FTS, "--max-minutes", "1", "--write"])
assert res.exit_code == 0, res.output
assert "incomplete" in res.output and published == []
def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch):
c, _, _, _ = env
_fts_net(monkeypatch)
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
(p,) = [p for p in T.read_passages(c) if p.passage.startswith("Substantially")]
c.execute(
"UPDATE market.filing_passage SET label = 'risk_factor', label_version = 'v' "
"WHERE passage_id = ?",
[p.passage_id],
)
_fts_net(monkeypatch, forbid_docs=True)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0 and "new entities 0" in res.output
got = {q.passage_id: q for q in T.read_passages(c)}
assert (
got[p.passage_id].label == "risk_factor"
and got[p.passage_id].label_version == "v"
)
(link,) = T.read_links(c, basis="fts")
assert link.strength == "risk_factor" and link.passage_id == p.passage_id
def test_fts_cli_missing_submissions_is_a_gap(env, monkeypatch):
c, _, _, _ = env
_fts_net(monkeypatch, hits=HITS[:1])
_online(
monkeypatch,
[
(
r"efts\.sec\.gov/.*",
lambda m: (
httpx.Response(
200, json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
)
if "Medicare+Physician" in m.group(0)
or "Medicare%20Physician" in m.group(0)
else httpx.Response(
200, json={"hits": {"total": {"value": 0}, "hits": []}}
)
),
),
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
(
r"company_tickers\.json",
lambda m: httpx.Response(
200, content=(FX / "company_tickers.json").read_bytes()
),
),
],
)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "CIK0000012345: no submissions, no entity" in res.output
assert T.read_entities(c) == []
# classification: the local model, resumable through the ledger
class _Ask:
model = "qwen-test"
def __init__(self, answers):
self.answers = answers
self.calls = 0
def __call__(self, system, user, schema):
self.calls += 1
return self.answers.pop(0)
TWO = [
HITS[0],
_hit_json("0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"),
]
TWO_DOCS = {"a10k.htm": SNIPPET, "b.htm": SNIPPET}
def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path):
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / f"{task}-{fam}.jsonl"
)
# a passage of a term outside this run's scope is never sent to the model
T.write_passages(
c, [T.PassageRow("X:y.htm:1", "9", "X", "y.htm", "other term", 1, "z")]
)
ask = _Ask(
[
{"label": "revenue_exposure", "reason": "r"},
None,
{"label": "incidental", "reason": "i"},
]
)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0, res.output
assert "classified with qwen-test: {'done': 1" in res.output and ask.calls == 1
labelled = [p for p in T.read_passages(c) if p.label]
assert len(labelled) == 1 and labelled[0].label == "revenue_exposure"
assert labelled[0].label_version == "model-reading exposure-v1 qwen-test"
# resume: the done unit is skipped; the next call fails (None) and stays unlabelled
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0 and ask.calls == 2
assert "'skipped': 1" in res.output and "'failed': 1" in res.output
# a rerun without --classify keeps what is stored
res = runner.invoke(app, [*FTS, "--write"])
assert len([p for p in T.read_passages(c) if p.label]) == 1
got = sorted(l.strength for l in T.read_links(c, basis="fts"))
assert got == ["mention", "revenue_exposure"]
def test_fts_cli_incidental_label_makes_no_link(env, monkeypatch, tmp_path):
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
)
ask = _Ask([{"label": "incidental", "reason": "i"}] * 2)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
res = runner.invoke(app, [*FTS, "--classify", "--write"])
assert res.exit_code == 0, res.output
assert T.read_links(c, basis="fts") == [] and len(T.read_passages(c)) == 2
def test_asker_builds_json_extractor(monkeypatch):
sentinel = object()
seen = {}
monkeypatch.setattr("llm.config.load", lambda: "cfg")
monkeypatch.setattr(
"llm.pool.HostPool.from_config", classmethod(lambda cls, cfg: ("pool", cfg))
)
def make(cfg, pool):
seen["args"] = (cfg, pool)
return sentinel
monkeypatch.setattr("llm.structured.json_extractor", make)
assert mk._asker() is sentinel
assert seen["args"] == ("cfg", ("pool", "cfg"))
def test_gap_note_truncates():
note = mk._gap_note([f"i{n}" for n in range(20)])
assert note.startswith("20 piece(s) missing: i0; i1") and "... 5 more" in note
def test_fts_cli_submissions_refusal_is_a_gap(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch, hits=HITS[:1])
_online(
monkeypatch,
[
(
r"efts\.sec\.gov/.*",
lambda m: httpx.Response(
200,
json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
if "Medicare%20Physician" in m.group(0)
or "Medicare+Physician" in m.group(0)
else {"hits": {"total": {"value": 0}, "hits": []}},
),
),
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
(r"CIK\d{10}\.json", lambda m: httpx.Response(403)),
(r"company_tickers\.json", lambda m: httpx.Response(403)),
],
)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
assert _count(c, "market.filing_hit") == 0 and published == [True]
def test_fts_cli_long_gap_list_is_abridged(env, monkeypatch):
c, _, _, _ = env
hits = [
_hit_json("0000012346", f"0000012346-23-{n:06d}", "x.pdf", ["8082"])
for n in range(22)
]
_fts_net(monkeypatch, hits=hits)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts: ... 2 more" in res.output
assert "... 7 more" in T.read_gaps(c)[0].note
def test_fts_cli_flaky_document_is_skipped(env, monkeypatch):
hits = [
HITS[0],
_hit_json(
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
),
]
_fts_net(
monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET}, broken={"b.htm": "timeout"}
)
monkeypatch.setattr("time.sleep", lambda s: None)
res = runner.invoke(app, FTS)
assert res.exit_code == 0, res.output
assert "b.htm: sec-edgar: HTTP 0 ReadTimeout" in res.output
assert "passages kept 1," in res.output
def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch, hits=HITS[:1], broken={"a10k.htm": 403})
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts: sec-edgar: HTTP 403" in res.output
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [
True
]