feat(market): Tiingo daily prices for a ranked, capped universe; benchmarks SPY/XLV/IHF; monthly series (#797)

This commit is contained in:
kert
2026-11-13 00:45:29 -05:00
parent a9a4a16e90
commit 37dc7d74b5
8 changed files with 1646 additions and 3 deletions

View File

@@ -17,7 +17,7 @@ from typing import Any
import typer
from market import edgar, http
from market import edgar, http, tiingo
from market.seed import (
SEAT_LABEL_KEYS,
SEED_PATH,
@@ -34,19 +34,23 @@ from market.tables import (
clear_gap,
ensure_tables,
read_gaps,
read_prices,
retire_links,
retire_organizations,
write_entities,
write_filings,
write_gap,
write_links,
write_prices,
write_securities,
write_series,
)
from pfs.codetables import is_missing_table_error
app = typer.Typer(no_args_is_help=True)
ENTITIES_GAP = "sec-edgar:entities"
PRICES_GAP = "tiingo:prices"
# ── indirections the tests monkeypatch ───────────────────────────────
@@ -326,6 +330,217 @@ def entities(
)
# ── prices ───────────────────────────────────────────────────────────
def _or_empty(read: Any, con: Any, empty: Any) -> Any:
"""``read(con)``, or ``empty`` when a market table does not exist yet."""
try:
return read(con)
except Exception as exc:
if not is_missing_table_error(exc):
raise
return empty
def _merge_gap(existing: str, issues: dict[str, str], fetched: set[str]) -> str:
"""Earlier gap entries (``TICKER: reason`` joined by `` | ``) stay unless
that ticker was priced this run; this run's issues are added."""
kept = {}
for part in filter(None, existing.split(" | ")):
kept[part.split(":", 1)[0]] = part
for t in fetched:
kept.pop(t, None)
kept.update({t: f"{t}: {note}" for t, note in issues.items()})
return " | ".join(kept[t] for t in sorted(kept))
@app.command()
def prices(
ticker: list[str] = typer.Option(
[], "--ticker", help="Price this ticker (repeatable)."
),
universe: bool = typer.Option(
False, "--universe", help="The ranked, capped universe (seed + filing-linked)."
),
benchmarks: bool = typer.Option(False, "--benchmarks", help="SPY, XLV and IHF."),
limit: int = typer.Option(
0, "--limit", help="Download at most this many symbols (0 = no limit)."
),
refresh: bool = typer.Option(
False, "--refresh", help="Download again even if priced or hoarded today."
),
dry_run: bool = typer.Option(
False, "--dry-run", help="Print the ranked universe and the budget only."
),
write: bool = typer.Option(
False, "--write", help="Store prices, returns and the monthly series."
),
) -> None:
"""Tiingo daily prices and total returns (market.price_daily) and the
month-end total-return series (market.series, #797). Prices are internal
use only. The monthly symbol allowance is 450; a first pull of a symbol
starts at its listing, later pulls at the last stored day."""
if not (ticker or universe or benchmarks):
typer.echo("nothing to price: pass --ticker, --universe or --benchmarks")
raise typer.Exit(2)
key = None
if not dry_run:
key = _require_key("tiingo", PRICES_GAP, write=write)
if key is None:
return
day = tiingo.today()
budget = tiingo.Budget.load(day)
pacer = tiingo.new_pacer()
try:
with http.client("tiingo") as free:
supported = tiingo.fetch_supported(free, pacer=pacer, refresh=refresh)
except http.Refused as exc:
_gap(PRICES_GAP, str(exc), write=write)
return
import duckdb
con = _read()
try:
state = _or_empty(tiingo.last_pulls, con, {})
targets = _or_empty(
lambda c: tiingo.plan(
c,
tickers=ticker,
universe_on=universe,
benchmarks_on=benchmarks,
used=budget.symbols,
supported=supported,
),
con,
None,
)
finally:
con.close()
if targets is None:
mem = duckdb.connect(":memory:")
ensure_tables(mem)
targets = tiingo.plan(
mem,
tickers=ticker,
universe_on=False,
benchmarks_on=benchmarks or universe,
used=budget.symbols,
supported=supported,
)
mem.close()
typer.echo(
f"budget {budget.month}: {len(budget.symbols)}/{tiingo.MONTHLY_SYMBOLS} "
f"symbols used, {budget.remaining_symbols} remaining; requests today "
f"{budget.day_counts.get(day.isoformat(), 0)}/{tiingo.DAILY_REQUESTS}"
)
if dry_run:
for i, t in enumerate(targets, 1):
seen = state.get(t.ticker)
note = f"stored to {seen[0]}" if seen else f"first pull from {t.start}"
used = "used" if t.ticker in budget.symbols else "new"
typer.echo(f"{i:>4}. {t.ticker:<8} {t.entity_key:<14} {used:<4} {note}")
typer.echo(f"{len(targets)} symbols ranked")
return
fetched: dict[str, list[Any]] = {}
issues: dict[str, str] = {}
n_downloaded = 0
cl = http.client("tiingo", key)
try:
for t in targets:
if limit and n_downloaded >= limit:
break
last = state.get(t.ticker)
if last and last[1] == day and not refresh:
continue
start = last[0] if last else t.start
try:
records, path = tiingo.fetch_eod(
cl,
t.ticker,
start,
day,
pacer=pacer,
budget=budget,
today=day,
refresh=refresh,
)
except tiingo.BudgetExhausted as exc:
typer.echo(f"budget: stopped, {exc}")
break
n_downloaded += 1
if records is None:
issues[t.ticker] = "not found at Tiingo"
typer.echo(f"{t.ticker}: not found at Tiingo")
continue
new = tiingo.clip_windows(
tiingo.to_prices(
t.ticker,
records,
pulled_on=day,
source_file=str(path),
overlap=last[0] if last else None,
),
t.windows,
)
meta = {
k: v
for k, v in zip(
("startDate", "endDate"),
(supported.get(t.ticker) or (None, None)),
strict=True,
)
}
notes = [n for s in t.securities if (n := tiingo.window_gap(meta, s))]
if notes:
issues[t.ticker] = "; ".join(notes)
fetched[t.ticker] = new
span = f"{new[0].day}..{new[-1].day}" if new else "no new rows"
typer.echo(f"{t.ticker}: +{len(new)} rows {span}")
except http.Refused as exc:
_gap(PRICES_GAP, str(exc), write=write)
return
finally:
cl.close()
n_rows = sum(len(r) for r in fetched.values())
typer.echo(
f"priced {len(fetched)} symbols, {n_rows} rows; budget "
f"{len(budget.symbols)}/{tiingo.MONTHLY_SYMBOLS} symbols"
)
for t, note in sorted(issues.items()):
typer.echo(f"gap: {PRICES_GAP}: {t}: {note}")
if not write:
typer.echo("dry run: pass --write to store")
return
from cms.ingest_log import log_ingest
store = _store()
item_keys = {t: tiingo.cite(store, t) for t in fetched}
n_series = 0
with _batch() as con:
ensure_tables(con)
for t, rows in fetched.items():
write_prices(con, t, rows)
series = tiingo.tri_monthly(t, read_prices(con, t), item_key=item_keys[t])
n_series += write_series(con, f"tiingo:{t}:tri", series)
old = {g.source: g.note for g in read_gaps(con)}.get(PRICES_GAP, "")
merged = _merge_gap(old, issues, set(fetched) - set(issues))
if merged:
write_gap(con, PRICES_GAP, merged)
else:
clear_gap(con, PRICES_GAP)
src = str(http.hoard(http.MANIFEST))
for table, n in (("market.price_daily", n_rows), ("market.series", n_series)):
log_ingest(
con, module="market.tiingo", table_name=table, rows=n, source_file=src
)
_publish()
typer.echo(f"stored: {n_rows} price rows, {n_series} monthly rows")
@app.command()
def gaps() -> None:
"""Sources that could not be read (market.source_gap)."""

View File

@@ -35,6 +35,16 @@ CREATE TABLE IF NOT EXISTS market.filing (
);
CREATE TABLE IF NOT EXISTS market.source_gap (
source VARCHAR, note VARCHAR, recorded_on DATE
);
CREATE TABLE IF NOT EXISTS market.price_daily (
ticker VARCHAR, day DATE, open DOUBLE, high DOUBLE, low DOUBLE,
close DOUBLE, volume DOUBLE, adj_close DOUBLE, div_cash DOUBLE,
split_factor DOUBLE, ret DOUBLE, ret_raw DOUBLE, pulled_on DATE,
source_file VARCHAR
);
CREATE TABLE IF NOT EXISTS market.series (
series_id VARCHAR, symbol VARCHAR, source VARCHAR, year INTEGER,
month INTEGER, value DOUBLE, item_key VARCHAR
)
"""
@@ -125,6 +135,42 @@ class GapRow:
recorded_on: date
@dataclass(frozen=True)
class PriceRow:
"""One trading day of one ticker (internal use only: Tiingo's licence)."""
ticker: str
day: date
open: float
high: float
low: float
close: float
volume: float
adj_close: float
div_cash: float
split_factor: float
#: adjClose_t / adjClose_{t-1} - 1, within one pull (``None`` on its
#: first row)
ret: float | None
#: (close_t + divCash_t) * splitFactor_t / close_{t-1} - 1
ret_raw: float | None
pulled_on: date
source_file: str
@dataclass(frozen=True)
class MonthlyRow:
"""A month-end value of a series (``market.series``)."""
series_id: str
symbol: str
source: str
year: int
month: int
value: float
item_key: str
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
a statement per row, far too slow for filing lists)."""
@@ -335,6 +381,50 @@ def read_gaps(con: Any) -> list[GapRow]:
return sorted((GapRow(*r) for r in rows), key=lambda g: g.source)
# ── price_daily / series ─────────────────────────────────────────────
def write_prices(con: Any, ticker: str, rows: Sequence[PriceRow]) -> int:
"""Delete *ticker*'s rows over the days of *rows*, then insert them."""
ensure_tables(con)
if not rows:
return 0
days = [r.day for r in rows]
con.execute(
"DELETE FROM market.price_daily WHERE ticker = ? AND day BETWEEN ? AND ?",
[ticker, min(days), max(days)],
)
return _insert(con, "market.price_daily", rows)
def read_prices(con: Any, ticker: str) -> list[PriceRow]:
rows = _select(con, PriceRow, "market.price_daily", "ticker = ?", [ticker])
return sorted((PriceRow(*r) for r in rows), key=lambda r: r.day)
def write_series(con: Any, series_id: str, rows: Sequence[MonthlyRow]) -> int:
"""Replace the whole series *series_id*."""
ensure_tables(con)
con.execute("DELETE FROM market.series WHERE series_id = ?", [series_id])
return _insert(con, "market.series", rows)
def read_series(con: Any, symbols: Sequence[str]) -> list[MonthlyRow]:
if not symbols:
return []
rows = _select(
con,
MonthlyRow,
"market.series",
f"symbol IN ({','.join('?' * len(symbols))})",
list(symbols),
)
return sorted(
(MonthlyRow(*r) for r in rows),
key=lambda m: (m.series_id, m.year, m.month),
)
# ── districts ────────────────────────────────────────────────────────

529
src/market/tiingo.py Normal file
View File

@@ -0,0 +1,529 @@
"""Tiingo daily prices and total returns: the index ETFs, the seed companies
and the filing-linked companies, ranked and capped to the monthly symbol
allowance, plus a month-end total-return series (#797).
Prices are internal use only (Tiingo's licence): they live in
``data/market/tiingo/`` and ``market.price_daily`` / ``market.series`` and are
never attached to bib, committed or shared. The key travels only in the
``Authorization`` header (:func:`market.http.client`).
Tiingo keeps a company's full history under its current ticker, so a
security is pulled by the provider's ticker and clipped only by a window in
which that ticker belonged to a different company (:func:`clip`).
"""
from __future__ import annotations
import csv
import datetime as dt
import io
import json
import zipfile
from collections import defaultdict
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from datetime import date
from pathlib import Path
from typing import Any
import httpx
from market import http
from market.tables import ( # noqa: F401 (re-exported)
MonthlyRow,
PriceRow,
SecurityRow,
read_links,
read_prices,
read_securities,
read_series,
write_prices,
write_series,
)
SOURCE = "tiingo"
EOD = "https://api.tiingo.com/tiingo/daily/{ticker}/prices"
META = "https://api.tiingo.com/tiingo/daily/{ticker}"
SUPPORTED = "https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip"
BENCHMARKS = {
"SPY": ("market", date(1993, 1, 29)),
"XLV": ("sector", date(1998, 12, 22)),
"IHF": ("providers", date(2006, 5, 5)),
}
#: no history is pulled from before SPY's first day
FLOOR = date(1993, 1, 29)
MONTHLY_SYMBOLS = 450
DAILY_REQUESTS = 950
PACE_SECONDS = 75.0
_STRENGTH = {"revenue_exposure": 0, "risk_factor": 1, "mention": 2}
def new_pacer() -> http.Pacer:
return http.Pacer(PACE_SECONDS)
def today() -> date:
return dt.date.today()
def state_dir() -> Path:
return Path(__file__).resolve().parents[2] / ".state" / "market"
# ── budget ───────────────────────────────────────────────────────────
class BudgetExhausted(Exception):
"""The monthly symbol allowance or the daily request cap is spent."""
@dataclass
class Budget:
"""Tiingo usage this month: distinct symbols and requests per day."""
month: str
symbols: set[str] = field(default_factory=set)
day_counts: dict[str, int] = field(default_factory=dict)
path: Path | None = None
@classmethod
def load(cls, day: date) -> Budget:
month = f"{day:%Y-%m}"
path = state_dir() / f"tiingo-{month}.json"
if path.is_file():
raw = json.loads(path.read_text())
return cls(
month,
set(raw.get("symbols", [])),
{k: int(v) for k, v in raw.get("day_counts", {}).items()},
path,
)
return cls(month, path=path)
def save(self) -> None:
if self.path is None:
return
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(
json.dumps(
{
"month": self.month,
"symbols": sorted(self.symbols),
"day_counts": self.day_counts,
},
indent=1,
)
)
@property
def remaining_symbols(self) -> int:
return MONTHLY_SYMBOLS - len(self.symbols)
def allow(self, ticker: str, day: date) -> bool:
if self.day_counts.get(day.isoformat(), 0) >= DAILY_REQUESTS:
return False
return ticker in self.symbols or len(self.symbols) < MONTHLY_SYMBOLS
def spend(self, ticker: str, day: date) -> None:
self.symbols.add(ticker)
key = day.isoformat()
self.day_counts[key] = self.day_counts.get(key, 0) + 1
self.save()
# ── supported tickers ────────────────────────────────────────────────
def _d(text: Any) -> date | None:
text = str(text or "").strip()[:10]
return date.fromisoformat(text) if text else None
def parse_supported(zip_bytes: bytes) -> dict[str, tuple[date | None, date | None]]:
"""``supported_tickers.csv`` USD rows as ``{ticker: (start, end)}``
(a ticker listed twice keeps its widest span)."""
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as z:
text = z.read("supported_tickers.csv").decode("utf-8-sig")
out: dict[str, tuple[date | None, date | None]] = {}
for row in csv.DictReader(io.StringIO(text)):
if row.get("priceCurrency") != "USD":
continue
ticker = row["ticker"].strip().upper()
start, end = _d(row.get("startDate")), _d(row.get("endDate"))
if ticker in out:
s0, e0 = out[ticker]
start = min((d for d in (s0, start) if d), default=None)
end = max((d for d in (e0, end) if d), default=None)
out[ticker] = (start, end)
return out
def fetch_supported(
cl: httpx.Client | None,
*,
pacer: Any,
refresh: bool = False,
offline: bool = False,
) -> dict[str, tuple[date | None, date | None]]:
"""The supported-ticker list (keyless, outside the budget), hoarded at
``data/market/tiingo/supported_tickers.zip``."""
path = http.hoard("tiingo", "supported_tickers.zip")
have = http.newest(path)
if have.is_file() and (offline or not refresh):
return parse_supported(have.read_bytes())
if offline or cl is None:
return {}
r = http.get(cl, SUPPORTED, pacer=pacer, source=SOURCE)
saved = http.save(path, r.content)
http.manifest(saved, SUPPORTED)
return parse_supported(saved.read_bytes())
# ── fetching ─────────────────────────────────────────────────────────
def fetch_eod(
cl: httpx.Client,
ticker: str,
start: date,
end: date,
*,
pacer: Any,
budget: Budget,
today: date,
refresh: bool = False,
) -> tuple[list[dict[str, Any]] | None, Path | None]:
"""End-of-day records of *ticker* for ``start..end``, hoarded at
``data/market/tiingo/eod/<T>/<start>_<end>.json`` (a changed body lands
beside the old one). A hoarded file costs nothing; a download spends the
budget first. An unknown ticker is ``(None, None)``."""
path = http.hoard("tiingo", "eod", ticker.replace("/", "_"), f"{start}_{end}.json")
have = http.newest(path)
if have.is_file() and not refresh:
return json.loads(have.read_bytes()), have
if not budget.allow(ticker, today):
raise BudgetExhausted(
f"{ticker}: {len(budget.symbols)}/{MONTHLY_SYMBOLS} symbols this month, "
f"{budget.day_counts.get(today.isoformat(), 0)}/{DAILY_REQUESTS} requests today"
)
budget.spend(ticker, today)
url = EOD.format(ticker=ticker)
r = http.get(
cl,
url,
params={
"startDate": start.isoformat(),
"endDate": end.isoformat(),
"format": "json",
"resampleFreq": "daily",
},
pacer=pacer,
source=SOURCE,
ok_404=True,
)
if r.status_code == 404:
return None, None
saved = http.save(path, r.content)
http.manifest(saved, url)
return json.loads(saved.read_bytes()), saved
# ── returns ──────────────────────────────────────────────────────────
def to_prices(
ticker: str,
records: Sequence[Mapping[str, Any]],
*,
pulled_on: date,
source_file: str,
overlap: date | None,
) -> list[PriceRow]:
"""Price rows with returns computed within this pull only (adjusted
prices are rebased by later splits and dividends, so levels never mix
across pulls). The *overlap* day, already stored with its own return, is
the base of the next day's return and is not returned."""
rows: list[PriceRow] = []
prev: Mapping[str, Any] | None = None
for rec in sorted(records, key=lambda r: str(r["date"])):
close = float(rec["close"])
adj = float(rec["adjClose"])
div = float(rec.get("divCash") or 0.0)
split = float(rec.get("splitFactor") or 1.0)
ret = ret_raw = None
if prev is not None:
ret = adj / float(prev["adjClose"]) - 1.0
ret_raw = (close + div) * split / float(prev["close"]) - 1.0
prev = rec
day = date.fromisoformat(str(rec["date"])[:10])
if overlap is not None and day == overlap:
continue
rows.append(
PriceRow(
ticker,
day,
float(rec["open"]),
float(rec["high"]),
float(rec["low"]),
close,
float(rec.get("volume") or 0.0),
adj,
div,
split,
ret,
ret_raw,
pulled_on,
source_file,
)
)
return rows
def clip(
rows: Sequence[PriceRow], valid_from: date | None, valid_to: date | None
) -> list[PriceRow]:
"""Rows inside ``valid_from..valid_to`` (either end open)."""
return [
r
for r in rows
if (valid_from is None or r.day >= valid_from)
and (valid_to is None or r.day <= valid_to)
]
def window_gap(
meta: Mapping[str, Any], sec: SecurityRow, tol_days: int = 7
) -> str | None:
"""A note when Tiingo's ``startDate``/``endDate`` fall short of the
security's window by more than *tol_days*."""
tol = dt.timedelta(days=tol_days)
start, end = _d(meta.get("startDate")), _d(meta.get("endDate"))
notes = []
if sec.valid_from and start and start > sec.valid_from + tol:
notes.append(f"Tiingo starts {start}, after valid_from {sec.valid_from}")
if sec.valid_to and end and end < sec.valid_to - tol:
notes.append(f"Tiingo ends {end}, before valid_to {sec.valid_to}")
return "; ".join(notes) or None
def tri_monthly(
ticker: str, rows: Sequence[PriceRow], *, item_key: str
) -> list[MonthlyRow]:
"""Month-end total-return index, 100 on the first stored day, chained
from the stored daily returns (so a later rebasing of adjusted prices
cannot break it)."""
level = 100.0
last: dict[tuple[int, int], float] = {}
for r in sorted(rows, key=lambda r: r.day):
if r.ret is not None:
level *= 1.0 + r.ret
last[(r.day.year, r.day.month)] = level
return [
MonthlyRow(f"tiingo:{ticker}:tri", ticker, SOURCE, y, m, v, item_key)
for (y, m), v in sorted(last.items())
]
def cite(store: Any, ticker: str) -> str:
"""A bib ``source`` item for *ticker*'s Tiingo series: a citation only,
never an attachment (the prices are internal-use)."""
from bib.item import Source
from bib.tag import Tag
item = Source(
title=f"Tiingo end-of-day prices: {ticker}",
url=EOD.format(ticker=ticker),
doc_type="dataset",
abstract="Internal use only (Tiingo licence); raw prices stay in "
"data/market/tiingo and market.price_daily.",
access_date=today().isoformat(),
)
return store.upsert(
item,
tags=[
Tag.module("market"),
Tag.table("market.price_daily"),
Tag.source("tiingo"),
],
)
# ── readers over market.price_daily ──────────────────────────────────
def read_returns(
con: Any, tickers: Sequence[str], start: date, end: date
) -> dict[str, dict[date, float]]:
"""``{ticker: {day: ret}}`` over ``start..end``; days without a return
are dropped."""
out: dict[str, dict[date, float]] = {t: {} for t in tickers}
if not tickers:
return out
marks = ",".join("?" * len(tickers))
for t, day, ret in con.execute(
"SELECT ticker, day, ret FROM market.price_daily "
f"WHERE ticker IN ({marks}) AND day BETWEEN ? AND ? AND ret IS NOT NULL "
"ORDER BY day",
[*tickers, start, end],
).fetchall():
out[t][day] = ret
return out
def trading_days(con: Any, ticker: str = "SPY") -> list[date]:
return [
d
for (d,) in con.execute(
"SELECT day FROM market.price_daily WHERE ticker = ? ORDER BY day",
[ticker],
).fetchall()
]
def last_pulls(con: Any) -> dict[str, tuple[date, date]]:
"""``{ticker: (last stored day, last pulled_on)}``."""
return {
t: (d, p)
for t, d, p in con.execute(
"SELECT ticker, max(day), max(pulled_on) FROM market.price_daily "
"GROUP BY ticker"
).fetchall()
}
# ── the ranked universe (M4) ─────────────────────────────────────────
def _seed_ciks() -> list[str]:
from market.seed import load_seed
return [c.cik for c in load_seed()[0]]
def universe(
con: Any,
*,
used: set[str],
cap: int = MONTHLY_SYMBOLS,
supported: Mapping[str, Any] | None = None,
seed_order: Sequence[str] | None = None,
) -> list[tuple[str, str]]:
"""``(ticker, entity_key)`` in rank order: the benchmarks, the seed
securities in seed order, then the securities of entities with a
filing-text (``fts``) link, ranked by strongest link, number of distinct
link targets and newest 10-K. SIC-only entities and unsupported or
unpriced tickers are excluded; at most *cap* symbols not already in
*used* are returned."""
by_entity: dict[str, list[str]] = defaultdict(list)
for s in read_securities(con):
if s.priced:
by_entity[s.entity_key].append(s.ticker)
ranked: list[tuple[str, str]] = [(t, f"index:{t}") for t in BENCHMARKS]
for key in seed_order if seed_order is not None else _seed_ciks():
ranked += [(t, key) for t in by_entity.get(key, [])]
best: dict[str, int] = {}
targets: dict[str, set[str]] = defaultdict(set)
for link in read_links(con, basis="fts"):
rank = _STRENGTH.get(link.strength, len(_STRENGTH))
best[link.entity_key] = min(best.get(link.entity_key, rank), rank)
targets[link.entity_key].add(link.target)
newest = {
cik: filed
for cik, filed in con.execute(
"SELECT cik, max(filed) FROM market.filing "
"WHERE form IN ('10-K', '10-K/A') GROUP BY cik"
).fetchall()
}
for key in sorted(
best,
key=lambda k: (
best[k],
-len(targets[k]),
-(newest[k].toordinal() if newest.get(k) else 0),
k,
),
):
ranked += [(t, key) for t in by_entity.get(key, [])]
out: list[tuple[str, str]] = []
seen: set[str] = set()
fresh = 0
for ticker, key in ranked:
if ticker in seen or (supported is not None and ticker not in supported):
continue
if ticker not in used:
if fresh >= cap:
continue
fresh += 1
seen.add(ticker)
out.append((ticker, key))
return out
# ── what to pull ─────────────────────────────────────────────────────
@dataclass(frozen=True)
class Target:
ticker: str
entity_key: str
#: ``(valid_from, valid_to)`` of the securities carrying this ticker;
#: empty when none is known
windows: tuple[tuple[date | None, date | None], ...]
#: where a first pull starts
start: date
securities: tuple[SecurityRow, ...] = ()
def plan(
con: Any,
*,
tickers: Sequence[str],
universe_on: bool,
benchmarks_on: bool,
used: set[str],
supported: Mapping[str, Any] | None = None,
cap: int = MONTHLY_SYMBOLS,
seed_order: Sequence[str] | None = None,
) -> list[Target]:
"""Benchmarks, then the named tickers, then the ranked universe."""
names: list[tuple[str, str]] = []
if benchmarks_on:
names += [(t, f"index:{t}") for t in BENCHMARKS]
secs: dict[str, list[SecurityRow]] = defaultdict(list)
for s in read_securities(con):
secs[s.ticker].append(s)
for t in tickers:
t = t.upper()
names.append((t, secs[t][0].entity_key if secs.get(t) else ""))
if universe_on:
names += universe(
con, used=used, cap=cap, supported=supported, seed_order=seed_order
)
out: list[Target] = []
seen: set[str] = set()
for ticker, key in names:
if ticker in seen:
continue
seen.add(ticker)
rows = secs.get(ticker, []) if ticker not in BENCHMARKS else []
windows = tuple((s.valid_from, s.valid_to) for s in rows)
if ticker in BENCHMARKS:
start = BENCHMARKS[ticker][1]
elif windows and all(w[0] for w in windows):
start = max(FLOOR, min(w[0] for w in windows))
else:
start = FLOOR
out.append(Target(ticker, key, windows, start, tuple(rows)))
return out
def clip_windows(
rows: Sequence[PriceRow], windows: Sequence[tuple[date | None, date | None]]
) -> list[PriceRow]:
"""Rows inside any of *windows* (all rows when there is none)."""
if not windows:
return list(rows)
return [r for r in rows if any(clip([r], a, b) for a, b in windows)]

View File

@@ -587,3 +587,335 @@ def test_entities_stores_ticker_history_and_link_dates(env):
if l.entity_key == "0001801170"
]
assert clov.valid_to == date(2023, 12, 31)
# ── stack market prices ──────────────────────────────────────────────
import io # noqa: E402
import zipfile # noqa: E402
from market import tiingo # noqa: E402
DAY = date(2026, 11, 12)
def _supported_zip(*rows: str) -> bytes:
head = "ticker,exchange,assetType,priceCurrency,startDate,endDate\n"
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as z:
z.writestr("supported_tickers.csv", head + "\n".join(rows) + "\n")
return buf.getvalue()
def _eod(day: str, close: float, adj: float | None = None) -> dict:
return {
"date": f"{day}T00:00:00.000Z",
"open": close,
"high": close,
"low": close,
"close": close,
"volume": 5,
"adjClose": close if adj is None else adj,
"divCash": 0.0,
"splitFactor": 1.0,
}
@pytest.fixture
def px(env, monkeypatch, tmp_path):
c, store, published, hoard = env
monkeypatch.setenv("TIINGO_API_KEY", "secret-key")
monkeypatch.setattr(tiingo, "state_dir", lambda: tmp_path / "state")
monkeypatch.setattr(tiingo, "new_pacer", lambda: _Pacer())
monkeypatch.setattr(tiingo, "today", lambda: DAY)
seen: list[httpx.Request] = []
series = {
"AAA": [
_eod("2024-01-02", 100),
_eod("2024-01-03", 110),
_eod("2024-01-04", 99),
],
"BBB": [_eod("2024-01-02", 10), _eod("2024-01-03", 11)],
"SPY": [_eod("2024-01-02", 400), _eod("2024-01-03", 404)],
"XLV": [_eod("2024-01-02", 100)],
"IHF": [_eod("2024-01-02", 50)],
}
z = _supported_zip(
"AAA,NYSE,Stock,USD,2000-01-01,2026-11-12",
"BBB,NYSE,Stock,USD,2000-01-01,2026-11-12",
"SPY,NYSE,ETF,USD,1993-01-29,2026-11-12",
"XLV,NYSE,ETF,USD,1998-12-22,2026-11-12",
"IHF,NYSE,ETF,USD,2006-05-05,2026-11-12",
)
def prices_for(m):
if m.group(1) not in series:
return httpx.Response(404, json={"detail": "Not found."})
return httpx.Response(200, json=series[m.group(1)])
routes = [
(r"supported_tickers\.zip", lambda m: httpx.Response(200, content=z)),
(r"/daily/(\w+)/prices", prices_for),
]
def handler(request):
seen.append(request)
return _handler(routes)(request)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(handler),
headers={"Authorization": f"Token {key}"} if key else {},
),
)
return c, store, published, hoard, seen, series, tmp_path / "state"
def _price_requests(seen):
return [r for r in seen if "/prices" in r.url.path]
def test_prices_cli_fetch_write(px):
c, _, published, hoard, seen, _, state = px
res = runner.invoke(
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
)
assert res.exit_code == 0, res.output
assert _count(c, "market.price_daily") == 5
assert [r.url.params["startDate"] for r in _price_requests(seen)] == [
"1993-01-29"
] * 2
assert all("secret-key" not in str(r.url) for r in seen)
keyed = [r for r in seen if "/prices" in r.url.path]
assert all(r.headers["authorization"] == "Token secret-key" for r in keyed)
free = [r for r in seen if "supported" in r.url.path]
assert free and "authorization" not in free[0].headers
assert {
(t, n)
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
} == {("market.price_daily", 5), ("market.series", 2)}
assert [s.value for s in T.read_series(c, ["AAA"])] == [pytest.approx(99.0)]
assert published == [True]
used = json.loads((state / "tiingo-2026-11.json").read_text())
assert used["symbols"] == ["AAA", "BBB"] and used["day_counts"] == {"2026-11-12": 2}
assert (hoard / "tiingo" / "eod" / "AAA" / "1993-01-29_2026-11-12.json").is_file()
assert "priced 2 symbols, 5 rows" in res.output and "2/450" in res.output
assert T.read_gaps(c) == []
def test_prices_incremental_pull_and_same_day_skip(px, monkeypatch):
c, _, _, _, seen, series, _ = px
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
assert runner.invoke(app, cmd).exit_code == 0
before = len(_price_requests(seen))
again = runner.invoke(app, cmd)
assert len(_price_requests(seen)) == before # priced today: nothing to do
assert "priced 0 symbols" in again.output
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
series["AAA"] = [_eod("2024-01-04", 99, 49.5), _eod("2024-01-05", 108.9, 54.45)]
assert runner.invoke(app, cmd).exit_code == 0
last = _price_requests(seen)[-1]
assert last.url.params["startDate"] == "2024-01-04"
rows = {r.day: r for r in T.read_prices(c, "AAA")}
assert len(rows) == 4 and rows[date(2024, 1, 4)].adj_close == 99
assert rows[date(2024, 1, 5)].ret == pytest.approx(0.1)
assert [m.value for m in T.read_series(c, ["AAA"])] == [pytest.approx(108.9)]
def test_prices_benchmarks_start_at_listing(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(app, ["market", "prices", "--benchmarks", "--write"])
assert res.exit_code == 0, res.output
starts = {
r.url.path.split("/")[3]: r.url.params["startDate"]
for r in _price_requests(seen)
}
assert starts == {"SPY": "1993-01-29", "XLV": "1998-12-22", "IHF": "2006-05-05"}
def test_prices_dry_run_reports_budget(px, monkeypatch):
c, _, published, _, seen, _, state = px
T.ensure_tables(c)
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
T.write_links(c, [T.LinkRow("K1", "program", "PFS", "fts", "mention", "adsh", "a")])
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
assert res.exit_code == 0, res.output
assert "1/450 symbols used, 449 remaining" in res.output
assert "1. SPY" in res.output and "used" in res.output
assert "AAA" in res.output and "K1" in res.output and "first pull" in res.output
assert _price_requests(seen) == [] and published == []
assert not (state / "tiingo-2026-11.json").read_text().count("AAA")
def test_prices_dry_run_without_market_tables_lists_benchmarks(px):
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
assert res.exit_code == 0, res.output
assert "3 symbols ranked" in res.output
def test_prices_dry_run_shows_stored_span(px):
c, *_ = px
T.ensure_tables(c)
T.write_prices(
c,
"AAA",
tiingo.to_prices(
"AAA", [_eod("2024-01-02", 1)], pulled_on=DAY, source_file="f", overlap=None
),
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--dry-run"])
assert "stored to 2024-01-02" in res.output
def test_prices_without_key_records_gap(px, monkeypatch):
c, _, published, _, seen, _, _ = px
monkeypatch.delenv("TIINGO_API_KEY")
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"]
assert _count(c, "market.price_daily") == 0 and seen == [] and published == [True]
def test_prices_refusal_prints_gap(px, monkeypatch):
c, _, published, _, _, _, _ = px
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(lambda r: httpx.Response(403))
),
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert "gap: tiingo:prices:" in res.output and "403" in res.output
assert _count(c, "market.price_daily") == 0
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"] and published == [
True
]
def test_prices_refusal_on_prices_after_supported_ok(px, monkeypatch):
c, _, published, _, _, _, _ = px
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
routes = [
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
(r"/prices", lambda m: httpx.Response(401)),
]
_online(monkeypatch, routes)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA"])
assert (
res.exit_code == 0
and "gap: tiingo:prices:" in res.output
and "401" in res.output
)
assert published == []
def test_prices_nothing_selected_exits(px):
res = runner.invoke(app, ["market", "prices"])
assert res.exit_code == 2 and "nothing to price" in res.output
def test_prices_unknown_ticker_and_window_gap_become_gap_note(px):
c, _, _, _, _, series, _ = px
T.ensure_tables(c)
T.write_securities(
c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, date(2030, 1, 1), "s")]
)
res = runner.invoke(
app, ["market", "prices", "--ticker", "BBB", "--ticker", "ZZZ", "--write"]
)
assert res.exit_code == 0, res.output
assert "ZZZ: not found at Tiingo" in res.output
(gap,) = T.read_gaps(c)
assert (
"ZZZ: not found at Tiingo" in gap.note
and "BBB: Tiingo ends 2026-11-12" in gap.note
)
# BBB later prices cleanly: its entry goes, ZZZ's stays
T.write_securities(c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, None, "s")])
runner.invoke(app, ["market", "prices", "--ticker", "BBB", "--refresh", "--write"])
(gap,) = T.read_gaps(c)
assert gap.note == "ZZZ: not found at Tiingo"
def test_prices_clean_run_clears_gap_and_window_clips(px):
c, _, _, _, _, _, _ = px
T.ensure_tables(c)
T.write_gap(c, "tiingo:prices", "AAA: not found at Tiingo")
T.write_securities(
c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", date(2024, 1, 3), None, "s")]
)
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
assert res.exit_code == 0, res.output
assert [r.day for r in T.read_prices(c, "AAA")] == [
date(2024, 1, 3),
date(2024, 1, 4),
]
assert T.read_gaps(c) == []
def test_prices_limit_stops_downloads(px):
c, _, _, _, seen, _, _ = px
res = runner.invoke(
app, ["market", "prices", "--benchmarks", "--limit", "1", "--write"]
)
assert res.exit_code == 0 and len(_price_requests(seen)) == 1
assert [r.ticker for r in [*T.read_prices(c, "SPY")][:1]] == ["SPY"]
def test_prices_budget_exhausted_stops_and_writes_what_it_has(px):
c, _, published, _, seen, _, state = px
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps(
{
"month": "2026-11",
"symbols": [f"S{i}" for i in range(449)] + ["AAA"],
"day_counts": {},
}
)
)
res = runner.invoke(
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
)
assert res.exit_code == 0, res.output
assert "budget: stopped, BBB" in res.output
assert [r.ticker for r in T.read_prices(c, "AAA")][:1] == ["AAA"]
assert T.read_prices(c, "BBB") == [] and published == [True]
def test_prices_without_write_stores_nothing(px, monkeypatch):
c, _, published, _, _, _, _ = px
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, ["market", "prices", "--ticker", "ZZZ"])
assert res.exit_code == 0, res.output
assert (
"dry run: pass --write" in res.output
and "gap: tiingo:prices: ZZZ" in res.output
)
assert published == []
def test_merge_gap_keeps_other_tickers_and_orders():
assert mk._merge_gap("", {}, set()) == ""
assert mk._merge_gap("A: x | B: y", {"C": "z"}, {"A"}) == "B: y | C: z"
def test_or_empty_reraises_other_errors():
def boom(_):
raise RuntimeError("disk")
with pytest.raises(RuntimeError):
mk._or_empty(boom, None, {})

View File

@@ -0,0 +1,7 @@
ticker,exchange,assetType,priceCurrency,startDate,endDate
000001,SHE,Stock,CNY,2007-01-04,2026-11-12
ELV,NYSE,Stock,USD,2001-10-30,2026-11-12
IHF,NYSE,ETF,USD,2006-05-05,2026-11-12
OSH,NYSE,Stock,USD,2020-08-06,2023-05-02
SPY,NYSE,ETF,USD,1993-01-29,2026-11-12
XLV,NYSE,ETF,USD,1998-12-22,2026-11-12
1 ticker exchange assetType priceCurrency startDate endDate
2 000001 SHE Stock CNY 2007-01-04 2026-11-12
3 ELV NYSE Stock USD 2001-10-30 2026-11-12
4 IHF NYSE ETF USD 2006-05-05 2026-11-12
5 OSH NYSE Stock USD 2020-08-06 2023-05-02
6 SPY NYSE ETF USD 1993-01-29 2026-11-12
7 XLV NYSE ETF USD 1998-12-22 2026-11-12

View File

@@ -6,7 +6,13 @@ from __future__ import annotations
import subprocess
import sys
MODULES = ["market.http", "market.tables", "market.seed", "market.edgar"]
MODULES = [
"market.http",
"market.tables",
"market.seed",
"market.edgar",
"market.tiingo",
]
_PROBE = """
import builtins, importlib, sys

View File

@@ -9,6 +9,7 @@ import duckdb
import pytest
from market import tables as T
from market import tiingo as TI
@pytest.fixture
@@ -30,7 +31,15 @@ def test_ensure_tables_idempotent():
"WHERE table_schema = 'market'"
).fetchall()
}
assert names == {"entity", "security", "entity_link", "filing", "source_gap"}
assert names == {
"entity",
"security",
"entity_link",
"filing",
"source_gap",
"price_daily",
"series",
}
ENTITY = T.EntityRow(
@@ -67,6 +76,24 @@ FILING = T.FilingRow(
)
GAP = T.GapRow("sec-edgar:entities", "403", dt.date.today())
PRICE = TI.PriceRow(
"OSH",
date(2021, 1, 4),
10.0,
11.0,
9.5,
10.5,
1000.0,
10.4,
0.0,
1.0,
0.01,
0.012,
date(2026, 11, 12),
"eod/OSH/x.json",
)
SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY")
ROUNDTRIP = {
"entity": (
ENTITY,
@@ -84,6 +111,16 @@ ROUNDTRIP = {
lambda c, r: T.write_filings(c, r.cik, [r]),
lambda c: T.read_filings(c, "0001564406"),
),
"price_daily": (
PRICE,
lambda c, r: TI.write_prices(c, r.ticker, [r]),
lambda c: TI.read_prices(c, "OSH"),
),
"series": (
SERIES,
lambda c, r: TI.write_series(c, r.series_id, [r]),
lambda c: TI.read_series(c, ["OSH"]),
),
"gap": (
GAP,
lambda c, r: T.write_gap(c, r.source, r.note),

427
tests/market/test_tiingo.py Normal file
View File

@@ -0,0 +1,427 @@
"""market.tiingo: returns, windows, budget, ranked universe, monthly series.
Prices here are synthetic (Tiingo's licence forbids sharing real ones)."""
from __future__ import annotations
import io
import zipfile
from datetime import date
from pathlib import Path
import duckdb
import httpx
import pytest
from market import http
from market import tables as T
from market import tiingo as TI
FX = Path(__file__).parent / "fixtures"
def rec(day, close, adj=None, div=0.0, split=1.0):
return {
"date": f"{day}T00:00:00.000Z",
"open": close,
"high": close,
"low": close,
"close": close,
"volume": 10,
"adjClose": close if adj is None else adj,
"divCash": div,
"splitFactor": split,
}
def pull(ticker, records, overlap=None):
return TI.to_prices(
ticker,
records,
pulled_on=date(2026, 11, 12),
source_file="f.json",
overlap=overlap,
)
def test_to_prices_returns_within_pull():
rows = pull(
"X",
[
rec("2024-01-02", 1, 100),
rec("2024-01-03", 1, 110),
rec("2024-01-04", 1, 99),
],
)
assert [r.ret for r in rows] == [None, pytest.approx(0.1), pytest.approx(-0.1)]
assert rows[0].day == date(2024, 1, 2) and rows[0].source_file == "f.json"
def test_ret_raw_with_split_and_dividend():
rows = pull("X", [rec("2024-01-02", 100), rec("2024-01-03", 50, split=2.0)])
assert rows[1].ret_raw == pytest.approx(0.0)
rows = pull("X", [rec("2024-01-02", 100), rec("2024-01-03", 99, div=1.0)])
assert rows[1].ret_raw == pytest.approx(0.0)
assert rows[1].div_cash == 1.0 and rows[1].split_factor == 1.0
def test_incremental_pull_rebased_adjclose_keeps_returns():
d1, d2, d3 = date(2024, 1, 2), date(2024, 1, 3), date(2024, 1, 4)
first = pull("X", [rec(str(d1), 100, 100), rec(str(d2), 110, 110)])
second = pull("X", [rec(str(d2), 110, 55), rec(str(d3), 121, 60.5)], overlap=d2)
assert [r.day for r in second] == [d3]
assert second[0].ret == pytest.approx(0.1)
con = duckdb.connect(":memory:")
T.ensure_tables(con)
TI.write_prices(con, "X", first)
TI.write_prices(con, "X", second)
stored = {r.day: r for r in TI.read_prices(con, "X")}
assert stored[d2].adj_close == 110 and stored[d2].ret == pytest.approx(0.1)
assert stored[d3].ret == pytest.approx(0.1)
def test_prices_clipped_to_security_window():
rows = pull("AGL", [rec("2015-06-01", 1), rec("2021-05-03", 1)])
kept = TI.clip(rows, date(2021, 4, 15), None)
assert [r.day for r in kept] == [date(2021, 5, 3)]
assert [r.day for r in TI.clip(rows, None, date(2016, 1, 1))] == [date(2015, 6, 1)]
assert len(TI.clip(rows, None, None)) == 2
def test_window_gap():
sec = T.SecurityRow(
"1", "AGL", "NYSE", date(2021, 4, 15), date(2023, 6, 30), "seed"
)
ok = {"startDate": "2021-04-15", "endDate": "2023-06-30"}
assert TI.window_gap(ok, sec) is None
note = TI.window_gap({"startDate": "2021-04-15", "endDate": "2023-05-02"}, sec)
assert "2023-05-02" in note and "2023-06-30" in note
late = TI.window_gap({"startDate": "2021-06-01", "endDate": None}, sec)
assert "2021-06-01" in late and "2021-04-15" in late
assert TI.window_gap({}, sec) is None
assert TI.window_gap({"startDate": "2021-04-20"}, sec) is None # within tolerance
def test_budget_monthly_symbols_and_daily_cap(tmp_path):
today = date(2026, 11, 12)
b = TI.Budget("2026-11", {f"S{i}" for i in range(450)}, {})
assert not b.allow("NEW", today)
assert b.allow("S7", today)
b2 = TI.Budget("2026-11", set(), {today.isoformat(): 950})
assert not b2.allow("S7", today)
b3 = TI.Budget("2026-11", set(), {})
b3.spend("A", today)
b3.spend("A", today)
assert b3.symbols == {"A"} and b3.day_counts == {"2026-11-12": 2}
def test_budget_persists_and_rolls_over(tmp_path, monkeypatch):
monkeypatch.setattr(TI, "state_dir", lambda: tmp_path)
today = date(2026, 11, 12)
b = TI.Budget.load(today)
b.spend("A", today)
again = TI.Budget.load(today)
assert again.symbols == {"A"} and again.remaining_symbols == 449
nxt = TI.Budget.load(date(2026, 12, 1))
assert nxt.symbols == set() and nxt.month == "2026-12"
assert (tmp_path / "tiingo-2026-11.json").is_file()
def test_monthly_tri_month_end():
rows = pull(
"X",
[
rec("2024-01-02", 1, 100),
rec("2024-01-31", 1, 110),
rec("2024-02-29", 1, 99),
],
)
out = TI.tri_monthly("X", rows, item_key="K")
assert [(m.year, m.month, m.value) for m in out] == [
(2024, 1, pytest.approx(110.0)),
(2024, 2, pytest.approx(99.0)),
]
assert out[0].series_id == "tiingo:X:tri" and out[0].item_key == "K"
assert out[0].symbol == "X" and out[0].source == "tiingo"
assert TI.tri_monthly("X", [], item_key="K") == []
def _universe_db():
con = duckdb.connect(":memory:")
T.ensure_tables(con)
def sec(key, ticker, priced=True):
T.write_securities(
con, key, [T.SecurityRow(key, ticker, "NYSE", None, None, "s", priced)]
)
def link(key, basis, target, strength):
T.write_links(
con, [T.LinkRow(key, "program", target, basis, strength, "adsh", "a")]
)
def tenk(key, filed):
T.write_filings(
con, key, [T.FilingRow(key, f"A-{key}", "10-K", filed, "", None, "", "")]
)
sec("SEED", "SEEDCO")
sec("OLD", "OLDCO", priced=False)
sec("REV", "REVCO")
sec("MEN", "MENCO")
sec("MEN2", "MEN2CO")
sec("SIC", "SICCO")
sec("GONE", "GONECO")
link("SEED", "curated", "PFS", "curated")
link("REV", "fts", "PFS", "revenue_exposure")
link("MEN", "fts", "PFS", "mention")
link("MEN", "fts", "APM", "mention")
link("MEN2", "fts", "PFS", "mention")
link("SIC", "sic", "8082", "sector")
link("GONE", "fts", "PFS", "revenue_exposure")
tenk("MEN", date(2020, 1, 1))
tenk("MEN2", date(2025, 1, 1))
return con
SUPPORTED = {
t: (date(2000, 1, 1), None)
for t in (
"SPY",
"XLV",
"IHF",
"SEEDCO",
"REVCO",
"MENCO",
"MEN2CO",
"SICCO",
"OLDCO",
)
}
def test_universe_ranked_capped_excludes_sic_only():
con = _universe_db()
kw = {"supported": SUPPORTED, "seed_order": ["SEED"]}
got = [t for t, _ in TI.universe(con, used=set(), **kw)]
assert got[:3] == ["SPY", "XLV", "IHF"]
assert got[3:] == ["SEEDCO", "REVCO", "MENCO", "MEN2CO"]
assert "SICCO" not in got and "OLDCO" not in got and "GONECO" not in got
keys = dict(TI.universe(con, used=set(), **kw))
assert keys["REVCO"] == "REV" and keys["SPY"] == "index:SPY"
assert [t for t, _ in TI.universe(con, used=set(), cap=5, **kw)] == got[:5]
capped = [t for t, _ in TI.universe(con, used={"SPY"}, cap=5, **kw)]
assert capped[:2] == ["SPY", "XLV"] and len(capped) == 6
assert "SEEDCO" in capped and "REVCO" in capped and "MEN2CO" not in capped
def test_universe_default_seed_order_reads_the_seed():
con = duckdb.connect(":memory:")
T.ensure_tables(con)
assert [t for t, _ in TI.universe(con, used=set())] == ["SPY", "XLV", "IHF"]
def _zip(csv_bytes: bytes) -> bytes:
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as z:
z.writestr("supported_tickers.csv", csv_bytes)
return buf.getvalue()
def test_parse_supported():
got = TI.parse_supported(_zip((FX / "supported_tickers_head.csv").read_bytes()))
assert got["OSH"] == (date(2020, 8, 6), date(2023, 5, 2))
assert "000001" not in got
assert got["SPY"][0] == date(1993, 1, 29)
extra = b"ticker,exchange,assetType,priceCurrency,startDate,endDate\nZZ,NYSE,Stock,USD,,\nZZ,NYSE,ETF,USD,2001-01-01,2002-01-01\n"
assert TI.parse_supported(_zip(extra))["ZZ"] == (date(2001, 1, 1), date(2002, 1, 1))
def _client(routes, seen):
def handler(request: httpx.Request):
seen.append(request)
for frag, make in routes:
if frag in str(request.url):
return make(request)
return httpx.Response(404)
return httpx.Client(transport=httpx.MockTransport(handler))
class _Pacer:
def wait(self):
pass
@pytest.fixture
def hoard(tmp_path, monkeypatch):
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
monkeypatch.setattr(TI, "state_dir", lambda: tmp_path / "state")
return tmp_path / "market"
def test_benchmark_first_pull_starts_at_listing(hoard):
seen = []
cl = _client(
[("/prices", lambda r: httpx.Response(200, json=[rec("2024-01-02", 1)]))], seen
)
budget = TI.Budget("2026-11", set(), {})
for t in ("XLV", "SPY"):
start = TI.BENCHMARKS[t][1]
TI.fetch_eod(
cl,
t,
start,
date(2026, 11, 12),
pacer=_Pacer(),
budget=budget,
today=date(2026, 11, 12),
)
starts = {r.url.path.split("/")[3]: r.url.params["startDate"] for r in seen}
assert starts == {"XLV": "1998-12-22", "SPY": "1993-01-29"}
assert all(r.url.params["resampleFreq"] == "daily" for r in seen)
assert all("token" not in str(r.url).lower() for r in seen)
assert TI.BENCHMARKS["IHF"] == ("providers", date(2006, 5, 5))
def test_fetch_eod_caches_and_never_overwrites(hoard):
seen = []
body = [rec("2024-01-02", 1)]
cl = _client([("/prices", lambda r: httpx.Response(200, json=body))], seen)
budget = TI.Budget("2026-11", set(), {})
args = (cl, "X", date(2024, 1, 1), date(2024, 1, 5))
kw = {"pacer": _Pacer(), "budget": budget, "today": date(2026, 11, 12)}
recs, path = TI.fetch_eod(*args, **kw)
assert recs == body and path.parent.name == "X"
again, _ = TI.fetch_eod(*args, **kw)
assert again == body and len(seen) == 1 # cache hit costs nothing
body[:] = [rec("2024-01-02", 2)]
changed, p2 = TI.fetch_eod(*args, refresh=True, **kw)
assert changed == body and p2 != path and path.is_file()
assert budget.symbols == {"X"} and budget.day_counts["2026-11-12"] == 2
def test_fetch_eod_unknown_ticker_and_budget(hoard):
cl = _client([], [])
budget = TI.Budget("2026-11", set(), {})
kw = {"pacer": _Pacer(), "today": date(2026, 11, 12)}
assert TI.fetch_eod(
cl, "NOPE", date(2024, 1, 1), date(2024, 1, 5), budget=budget, **kw
) == (None, None)
full = TI.Budget("2026-11", {f"S{i}" for i in range(450)}, {})
with pytest.raises(TI.BudgetExhausted):
TI.fetch_eod(cl, "NEW", date(2024, 1, 1), date(2024, 1, 5), budget=full, **kw)
def test_fetch_supported_hoards_and_reads_cache(hoard):
seen = []
z = _zip((FX / "supported_tickers_head.csv").read_bytes())
cl = _client(
[("supported_tickers.zip", lambda r: httpx.Response(200, content=z))], seen
)
got = TI.fetch_supported(cl, pacer=_Pacer())
assert got["OSH"][1] == date(2023, 5, 2)
assert (hoard / "tiingo" / "supported_tickers.zip").is_file()
assert TI.fetch_supported(None, pacer=_Pacer(), offline=True)["SPY"]
assert TI.fetch_supported(cl, pacer=_Pacer())["SPY"] and len(seen) == 1
assert len(TI.fetch_supported(cl, pacer=_Pacer(), refresh=True)) == len(got)
def test_fetch_supported_offline_without_cache(hoard):
assert TI.fetch_supported(None, pacer=_Pacer(), offline=True) == {}
class _Store:
def __init__(self):
self.items = []
self.attached = []
def upsert(self, item, tags=()):
self.items.append(item)
return "KEY1"
def attach_file(self, *a, **k): # pragma: no cover - must never run
self.attached.append(a)
def test_tiingo_cite_has_no_attachment():
store = _Store()
assert TI.cite(store, "SPY") == "KEY1"
assert store.attached == []
assert "token" not in store.items[0].url.lower() and "SPY" in store.items[0].url
def test_plan_targets_windows_and_start():
con = duckdb.connect(":memory:")
T.ensure_tables(con)
T.write_securities(
con, "C", [T.SecurityRow("C", "CI", "NYSE", date(2018, 12, 21), None, "s")]
)
T.write_securities(con, "E", [T.SecurityRow("E", "ELV", "NYSE", None, None, "s")])
by = {
t.ticker: t
for t in TI.plan(
con,
tickers=["CI", "ELV", "ZZZ"],
universe_on=False,
benchmarks_on=True,
used=set(),
)
}
assert by["CI"].start == date(2018, 12, 21) and by["CI"].windows == (
(date(2018, 12, 21), None),
)
assert by["ELV"].start == TI.FLOOR and by["ZZZ"].windows == ()
assert by["SPY"].start == date(1993, 1, 29) and by["XLV"].start == date(
1998, 12, 22
)
assert by["CI"].securities[0].ticker == "CI"
def test_roundtrip_helpers():
con = duckdb.connect(":memory:")
T.ensure_tables(con)
spy = pull("SPY", [rec("2024-01-02", 1, 100), rec("2024-01-03", 1, 101)])
aaa = pull("AAA", [rec("2024-01-02", 1, 100), rec("2024-01-03", 1, 90)])
TI.write_prices(con, "SPY", spy)
TI.write_prices(con, "AAA", aaa)
assert TI.write_prices(con, "AAA", []) == 0
got = TI.read_returns(
con, ["SPY", "AAA", "NONE"], date(2024, 1, 1), date(2024, 1, 31)
)
assert got["SPY"] == {date(2024, 1, 3): pytest.approx(0.01)}
assert got["AAA"] == {date(2024, 1, 3): pytest.approx(-0.1)} and got["NONE"] == {}
assert (
TI.read_returns(con, ["SPY"], date(2024, 1, 4), date(2024, 2, 1))["SPY"] == {}
)
assert TI.trading_days(con) == [date(2024, 1, 2), date(2024, 1, 3)]
assert TI.read_returns(con, [], date(2024, 1, 1), date(2024, 1, 2)) == {}
m = TI.tri_monthly("SPY", spy, item_key="K")
TI.write_series(con, "tiingo:SPY:tri", m)
TI.write_series(con, "tiingo:SPY:tri", m)
assert TI.read_series(con, ["SPY"]) == m and TI.read_series(con, []) == []
assert TI.write_series(con, "x", []) == 0
assert TI.last_pulls(con) == {
"AAA": (date(2024, 1, 3), date(2026, 11, 12)),
"SPY": (date(2024, 1, 3), date(2026, 11, 12)),
}
def test_pacer_and_state_dir_defaults():
assert TI.new_pacer().min_interval == 75.0
assert TI.state_dir().parts[-2:] == (".state", "market")
assert TI.today() == date.today()
def test_plan_deduplicates_names():
con = duckdb.connect(":memory:")
T.ensure_tables(con)
got = TI.plan(
con,
tickers=["spy", "SPY"],
universe_on=True,
benchmarks_on=True,
used=set(),
seed_order=[],
)
assert [t.ticker for t in got] == ["SPY", "XLV", "IHF"]
assert TI.clip_windows([], []) == []