From 37dc7d74b54d0344b2c62f0958891ea5b9ccea32 Mon Sep 17 00:00:00 2001 From: kert Date: Fri, 13 Nov 2026 00:45:29 -0500 Subject: [PATCH] feat(market): Tiingo daily prices for a ranked, capped universe; benchmarks SPY/XLV/IHF; monthly series (#797) --- src/cli/market.py | 217 ++++++- src/market/tables.py | 90 +++ src/market/tiingo.py | 529 ++++++++++++++++++ tests/cli/test_market_cli.py | 332 +++++++++++ .../fixtures/supported_tickers_head.csv | 7 + tests/market/test_imports.py | 8 +- tests/market/test_tables.py | 39 +- tests/market/test_tiingo.py | 427 ++++++++++++++ 8 files changed, 1646 insertions(+), 3 deletions(-) create mode 100644 src/market/tiingo.py create mode 100644 tests/market/fixtures/supported_tickers_head.csv create mode 100644 tests/market/test_tiingo.py diff --git a/src/cli/market.py b/src/cli/market.py index 7bde991..2d920da 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -17,7 +17,7 @@ from typing import Any import typer -from market import edgar, http +from market import edgar, http, tiingo from market.seed import ( SEAT_LABEL_KEYS, SEED_PATH, @@ -34,19 +34,23 @@ from market.tables import ( clear_gap, ensure_tables, read_gaps, + read_prices, retire_links, retire_organizations, write_entities, write_filings, write_gap, write_links, + write_prices, write_securities, + write_series, ) from pfs.codetables import is_missing_table_error app = typer.Typer(no_args_is_help=True) ENTITIES_GAP = "sec-edgar:entities" +PRICES_GAP = "tiingo:prices" # ── indirections the tests monkeypatch ─────────────────────────────── @@ -326,6 +330,217 @@ def entities( ) +# ── prices ─────────────────────────────────────────────────────────── +def _or_empty(read: Any, con: Any, empty: Any) -> Any: + """``read(con)``, or ``empty`` when a market table does not exist yet.""" + try: + return read(con) + except Exception as exc: + if not is_missing_table_error(exc): + raise + return empty + + +def _merge_gap(existing: str, issues: dict[str, str], fetched: set[str]) -> str: + """Earlier gap entries (``TICKER: reason`` joined by `` | ``) stay unless + that ticker was priced this run; this run's issues are added.""" + kept = {} + for part in filter(None, existing.split(" | ")): + kept[part.split(":", 1)[0]] = part + for t in fetched: + kept.pop(t, None) + kept.update({t: f"{t}: {note}" for t, note in issues.items()}) + return " | ".join(kept[t] for t in sorted(kept)) + + +@app.command() +def prices( + ticker: list[str] = typer.Option( + [], "--ticker", help="Price this ticker (repeatable)." + ), + universe: bool = typer.Option( + False, "--universe", help="The ranked, capped universe (seed + filing-linked)." + ), + benchmarks: bool = typer.Option(False, "--benchmarks", help="SPY, XLV and IHF."), + limit: int = typer.Option( + 0, "--limit", help="Download at most this many symbols (0 = no limit)." + ), + refresh: bool = typer.Option( + False, "--refresh", help="Download again even if priced or hoarded today." + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Print the ranked universe and the budget only." + ), + write: bool = typer.Option( + False, "--write", help="Store prices, returns and the monthly series." + ), +) -> None: + """Tiingo daily prices and total returns (market.price_daily) and the + month-end total-return series (market.series, #797). Prices are internal + use only. The monthly symbol allowance is 450; a first pull of a symbol + starts at its listing, later pulls at the last stored day.""" + if not (ticker or universe or benchmarks): + typer.echo("nothing to price: pass --ticker, --universe or --benchmarks") + raise typer.Exit(2) + key = None + if not dry_run: + key = _require_key("tiingo", PRICES_GAP, write=write) + if key is None: + return + day = tiingo.today() + budget = tiingo.Budget.load(day) + pacer = tiingo.new_pacer() + try: + with http.client("tiingo") as free: + supported = tiingo.fetch_supported(free, pacer=pacer, refresh=refresh) + except http.Refused as exc: + _gap(PRICES_GAP, str(exc), write=write) + return + + import duckdb + + con = _read() + try: + state = _or_empty(tiingo.last_pulls, con, {}) + targets = _or_empty( + lambda c: tiingo.plan( + c, + tickers=ticker, + universe_on=universe, + benchmarks_on=benchmarks, + used=budget.symbols, + supported=supported, + ), + con, + None, + ) + finally: + con.close() + if targets is None: + mem = duckdb.connect(":memory:") + ensure_tables(mem) + targets = tiingo.plan( + mem, + tickers=ticker, + universe_on=False, + benchmarks_on=benchmarks or universe, + used=budget.symbols, + supported=supported, + ) + mem.close() + + typer.echo( + f"budget {budget.month}: {len(budget.symbols)}/{tiingo.MONTHLY_SYMBOLS} " + f"symbols used, {budget.remaining_symbols} remaining; requests today " + f"{budget.day_counts.get(day.isoformat(), 0)}/{tiingo.DAILY_REQUESTS}" + ) + if dry_run: + for i, t in enumerate(targets, 1): + seen = state.get(t.ticker) + note = f"stored to {seen[0]}" if seen else f"first pull from {t.start}" + used = "used" if t.ticker in budget.symbols else "new" + typer.echo(f"{i:>4}. {t.ticker:<8} {t.entity_key:<14} {used:<4} {note}") + typer.echo(f"{len(targets)} symbols ranked") + return + + fetched: dict[str, list[Any]] = {} + issues: dict[str, str] = {} + n_downloaded = 0 + cl = http.client("tiingo", key) + try: + for t in targets: + if limit and n_downloaded >= limit: + break + last = state.get(t.ticker) + if last and last[1] == day and not refresh: + continue + start = last[0] if last else t.start + try: + records, path = tiingo.fetch_eod( + cl, + t.ticker, + start, + day, + pacer=pacer, + budget=budget, + today=day, + refresh=refresh, + ) + except tiingo.BudgetExhausted as exc: + typer.echo(f"budget: stopped, {exc}") + break + n_downloaded += 1 + if records is None: + issues[t.ticker] = "not found at Tiingo" + typer.echo(f"{t.ticker}: not found at Tiingo") + continue + new = tiingo.clip_windows( + tiingo.to_prices( + t.ticker, + records, + pulled_on=day, + source_file=str(path), + overlap=last[0] if last else None, + ), + t.windows, + ) + meta = { + k: v + for k, v in zip( + ("startDate", "endDate"), + (supported.get(t.ticker) or (None, None)), + strict=True, + ) + } + notes = [n for s in t.securities if (n := tiingo.window_gap(meta, s))] + if notes: + issues[t.ticker] = "; ".join(notes) + fetched[t.ticker] = new + span = f"{new[0].day}..{new[-1].day}" if new else "no new rows" + typer.echo(f"{t.ticker}: +{len(new)} rows {span}") + except http.Refused as exc: + _gap(PRICES_GAP, str(exc), write=write) + return + finally: + cl.close() + + n_rows = sum(len(r) for r in fetched.values()) + typer.echo( + f"priced {len(fetched)} symbols, {n_rows} rows; budget " + f"{len(budget.symbols)}/{tiingo.MONTHLY_SYMBOLS} symbols" + ) + for t, note in sorted(issues.items()): + typer.echo(f"gap: {PRICES_GAP}: {t}: {note}") + if not write: + typer.echo("dry run: pass --write to store") + return + + from cms.ingest_log import log_ingest + + store = _store() + item_keys = {t: tiingo.cite(store, t) for t in fetched} + n_series = 0 + with _batch() as con: + ensure_tables(con) + for t, rows in fetched.items(): + write_prices(con, t, rows) + series = tiingo.tri_monthly(t, read_prices(con, t), item_key=item_keys[t]) + n_series += write_series(con, f"tiingo:{t}:tri", series) + old = {g.source: g.note for g in read_gaps(con)}.get(PRICES_GAP, "") + merged = _merge_gap(old, issues, set(fetched) - set(issues)) + if merged: + write_gap(con, PRICES_GAP, merged) + else: + clear_gap(con, PRICES_GAP) + src = str(http.hoard(http.MANIFEST)) + for table, n in (("market.price_daily", n_rows), ("market.series", n_series)): + log_ingest( + con, module="market.tiingo", table_name=table, rows=n, source_file=src + ) + _publish() + typer.echo(f"stored: {n_rows} price rows, {n_series} monthly rows") + + @app.command() def gaps() -> None: """Sources that could not be read (market.source_gap).""" diff --git a/src/market/tables.py b/src/market/tables.py index df5af05..9077a14 100644 --- a/src/market/tables.py +++ b/src/market/tables.py @@ -35,6 +35,16 @@ CREATE TABLE IF NOT EXISTS market.filing ( ); CREATE TABLE IF NOT EXISTS market.source_gap ( source VARCHAR, note VARCHAR, recorded_on DATE +); +CREATE TABLE IF NOT EXISTS market.price_daily ( + ticker VARCHAR, day DATE, open DOUBLE, high DOUBLE, low DOUBLE, + close DOUBLE, volume DOUBLE, adj_close DOUBLE, div_cash DOUBLE, + split_factor DOUBLE, ret DOUBLE, ret_raw DOUBLE, pulled_on DATE, + source_file VARCHAR +); +CREATE TABLE IF NOT EXISTS market.series ( + series_id VARCHAR, symbol VARCHAR, source VARCHAR, year INTEGER, + month INTEGER, value DOUBLE, item_key VARCHAR ) """ @@ -125,6 +135,42 @@ class GapRow: recorded_on: date +@dataclass(frozen=True) +class PriceRow: + """One trading day of one ticker (internal use only: Tiingo's licence).""" + + ticker: str + day: date + open: float + high: float + low: float + close: float + volume: float + adj_close: float + div_cash: float + split_factor: float + #: adjClose_t / adjClose_{t-1} - 1, within one pull (``None`` on its + #: first row) + ret: float | None + #: (close_t + divCash_t) * splitFactor_t / close_{t-1} - 1 + ret_raw: float | None + pulled_on: date + source_file: str + + +@dataclass(frozen=True) +class MonthlyRow: + """A month-end value of a series (``market.series``).""" + + series_id: str + symbol: str + source: str + year: int + month: int + value: float + item_key: str + + def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int: """Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is a statement per row, far too slow for filing lists).""" @@ -335,6 +381,50 @@ def read_gaps(con: Any) -> list[GapRow]: return sorted((GapRow(*r) for r in rows), key=lambda g: g.source) +# ── price_daily / series ───────────────────────────────────────────── + + +def write_prices(con: Any, ticker: str, rows: Sequence[PriceRow]) -> int: + """Delete *ticker*'s rows over the days of *rows*, then insert them.""" + ensure_tables(con) + if not rows: + return 0 + days = [r.day for r in rows] + con.execute( + "DELETE FROM market.price_daily WHERE ticker = ? AND day BETWEEN ? AND ?", + [ticker, min(days), max(days)], + ) + return _insert(con, "market.price_daily", rows) + + +def read_prices(con: Any, ticker: str) -> list[PriceRow]: + rows = _select(con, PriceRow, "market.price_daily", "ticker = ?", [ticker]) + return sorted((PriceRow(*r) for r in rows), key=lambda r: r.day) + + +def write_series(con: Any, series_id: str, rows: Sequence[MonthlyRow]) -> int: + """Replace the whole series *series_id*.""" + ensure_tables(con) + con.execute("DELETE FROM market.series WHERE series_id = ?", [series_id]) + return _insert(con, "market.series", rows) + + +def read_series(con: Any, symbols: Sequence[str]) -> list[MonthlyRow]: + if not symbols: + return [] + rows = _select( + con, + MonthlyRow, + "market.series", + f"symbol IN ({','.join('?' * len(symbols))})", + list(symbols), + ) + return sorted( + (MonthlyRow(*r) for r in rows), + key=lambda m: (m.series_id, m.year, m.month), + ) + + # ── districts ──────────────────────────────────────────────────────── diff --git a/src/market/tiingo.py b/src/market/tiingo.py new file mode 100644 index 0000000..03bd99e --- /dev/null +++ b/src/market/tiingo.py @@ -0,0 +1,529 @@ +"""Tiingo daily prices and total returns: the index ETFs, the seed companies +and the filing-linked companies, ranked and capped to the monthly symbol +allowance, plus a month-end total-return series (#797). + +Prices are internal use only (Tiingo's licence): they live in +``data/market/tiingo/`` and ``market.price_daily`` / ``market.series`` and are +never attached to bib, committed or shared. The key travels only in the +``Authorization`` header (:func:`market.http.client`). + +Tiingo keeps a company's full history under its current ticker, so a +security is pulled by the provider's ticker and clipped only by a window in +which that ticker belonged to a different company (:func:`clip`). +""" + +from __future__ import annotations + +import csv +import datetime as dt +import io +import json +import zipfile +from collections import defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path +from typing import Any + +import httpx + +from market import http +from market.tables import ( # noqa: F401 (re-exported) + MonthlyRow, + PriceRow, + SecurityRow, + read_links, + read_prices, + read_securities, + read_series, + write_prices, + write_series, +) + +SOURCE = "tiingo" +EOD = "https://api.tiingo.com/tiingo/daily/{ticker}/prices" +META = "https://api.tiingo.com/tiingo/daily/{ticker}" +SUPPORTED = "https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip" +BENCHMARKS = { + "SPY": ("market", date(1993, 1, 29)), + "XLV": ("sector", date(1998, 12, 22)), + "IHF": ("providers", date(2006, 5, 5)), +} +#: no history is pulled from before SPY's first day +FLOOR = date(1993, 1, 29) +MONTHLY_SYMBOLS = 450 +DAILY_REQUESTS = 950 +PACE_SECONDS = 75.0 +_STRENGTH = {"revenue_exposure": 0, "risk_factor": 1, "mention": 2} + + +def new_pacer() -> http.Pacer: + return http.Pacer(PACE_SECONDS) + + +def today() -> date: + return dt.date.today() + + +def state_dir() -> Path: + return Path(__file__).resolve().parents[2] / ".state" / "market" + + +# ── budget ─────────────────────────────────────────────────────────── + + +class BudgetExhausted(Exception): + """The monthly symbol allowance or the daily request cap is spent.""" + + +@dataclass +class Budget: + """Tiingo usage this month: distinct symbols and requests per day.""" + + month: str + symbols: set[str] = field(default_factory=set) + day_counts: dict[str, int] = field(default_factory=dict) + path: Path | None = None + + @classmethod + def load(cls, day: date) -> Budget: + month = f"{day:%Y-%m}" + path = state_dir() / f"tiingo-{month}.json" + if path.is_file(): + raw = json.loads(path.read_text()) + return cls( + month, + set(raw.get("symbols", [])), + {k: int(v) for k, v in raw.get("day_counts", {}).items()}, + path, + ) + return cls(month, path=path) + + def save(self) -> None: + if self.path is None: + return + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text( + json.dumps( + { + "month": self.month, + "symbols": sorted(self.symbols), + "day_counts": self.day_counts, + }, + indent=1, + ) + ) + + @property + def remaining_symbols(self) -> int: + return MONTHLY_SYMBOLS - len(self.symbols) + + def allow(self, ticker: str, day: date) -> bool: + if self.day_counts.get(day.isoformat(), 0) >= DAILY_REQUESTS: + return False + return ticker in self.symbols or len(self.symbols) < MONTHLY_SYMBOLS + + def spend(self, ticker: str, day: date) -> None: + self.symbols.add(ticker) + key = day.isoformat() + self.day_counts[key] = self.day_counts.get(key, 0) + 1 + self.save() + + +# ── supported tickers ──────────────────────────────────────────────── + + +def _d(text: Any) -> date | None: + text = str(text or "").strip()[:10] + return date.fromisoformat(text) if text else None + + +def parse_supported(zip_bytes: bytes) -> dict[str, tuple[date | None, date | None]]: + """``supported_tickers.csv`` USD rows as ``{ticker: (start, end)}`` + (a ticker listed twice keeps its widest span).""" + with zipfile.ZipFile(io.BytesIO(zip_bytes)) as z: + text = z.read("supported_tickers.csv").decode("utf-8-sig") + out: dict[str, tuple[date | None, date | None]] = {} + for row in csv.DictReader(io.StringIO(text)): + if row.get("priceCurrency") != "USD": + continue + ticker = row["ticker"].strip().upper() + start, end = _d(row.get("startDate")), _d(row.get("endDate")) + if ticker in out: + s0, e0 = out[ticker] + start = min((d for d in (s0, start) if d), default=None) + end = max((d for d in (e0, end) if d), default=None) + out[ticker] = (start, end) + return out + + +def fetch_supported( + cl: httpx.Client | None, + *, + pacer: Any, + refresh: bool = False, + offline: bool = False, +) -> dict[str, tuple[date | None, date | None]]: + """The supported-ticker list (keyless, outside the budget), hoarded at + ``data/market/tiingo/supported_tickers.zip``.""" + path = http.hoard("tiingo", "supported_tickers.zip") + have = http.newest(path) + if have.is_file() and (offline or not refresh): + return parse_supported(have.read_bytes()) + if offline or cl is None: + return {} + r = http.get(cl, SUPPORTED, pacer=pacer, source=SOURCE) + saved = http.save(path, r.content) + http.manifest(saved, SUPPORTED) + return parse_supported(saved.read_bytes()) + + +# ── fetching ───────────────────────────────────────────────────────── + + +def fetch_eod( + cl: httpx.Client, + ticker: str, + start: date, + end: date, + *, + pacer: Any, + budget: Budget, + today: date, + refresh: bool = False, +) -> tuple[list[dict[str, Any]] | None, Path | None]: + """End-of-day records of *ticker* for ``start..end``, hoarded at + ``data/market/tiingo/eod//_.json`` (a changed body lands + beside the old one). A hoarded file costs nothing; a download spends the + budget first. An unknown ticker is ``(None, None)``.""" + path = http.hoard("tiingo", "eod", ticker.replace("/", "_"), f"{start}_{end}.json") + have = http.newest(path) + if have.is_file() and not refresh: + return json.loads(have.read_bytes()), have + if not budget.allow(ticker, today): + raise BudgetExhausted( + f"{ticker}: {len(budget.symbols)}/{MONTHLY_SYMBOLS} symbols this month, " + f"{budget.day_counts.get(today.isoformat(), 0)}/{DAILY_REQUESTS} requests today" + ) + budget.spend(ticker, today) + url = EOD.format(ticker=ticker) + r = http.get( + cl, + url, + params={ + "startDate": start.isoformat(), + "endDate": end.isoformat(), + "format": "json", + "resampleFreq": "daily", + }, + pacer=pacer, + source=SOURCE, + ok_404=True, + ) + if r.status_code == 404: + return None, None + saved = http.save(path, r.content) + http.manifest(saved, url) + return json.loads(saved.read_bytes()), saved + + +# ── returns ────────────────────────────────────────────────────────── + + +def to_prices( + ticker: str, + records: Sequence[Mapping[str, Any]], + *, + pulled_on: date, + source_file: str, + overlap: date | None, +) -> list[PriceRow]: + """Price rows with returns computed within this pull only (adjusted + prices are rebased by later splits and dividends, so levels never mix + across pulls). The *overlap* day, already stored with its own return, is + the base of the next day's return and is not returned.""" + rows: list[PriceRow] = [] + prev: Mapping[str, Any] | None = None + for rec in sorted(records, key=lambda r: str(r["date"])): + close = float(rec["close"]) + adj = float(rec["adjClose"]) + div = float(rec.get("divCash") or 0.0) + split = float(rec.get("splitFactor") or 1.0) + ret = ret_raw = None + if prev is not None: + ret = adj / float(prev["adjClose"]) - 1.0 + ret_raw = (close + div) * split / float(prev["close"]) - 1.0 + prev = rec + day = date.fromisoformat(str(rec["date"])[:10]) + if overlap is not None and day == overlap: + continue + rows.append( + PriceRow( + ticker, + day, + float(rec["open"]), + float(rec["high"]), + float(rec["low"]), + close, + float(rec.get("volume") or 0.0), + adj, + div, + split, + ret, + ret_raw, + pulled_on, + source_file, + ) + ) + return rows + + +def clip( + rows: Sequence[PriceRow], valid_from: date | None, valid_to: date | None +) -> list[PriceRow]: + """Rows inside ``valid_from..valid_to`` (either end open).""" + return [ + r + for r in rows + if (valid_from is None or r.day >= valid_from) + and (valid_to is None or r.day <= valid_to) + ] + + +def window_gap( + meta: Mapping[str, Any], sec: SecurityRow, tol_days: int = 7 +) -> str | None: + """A note when Tiingo's ``startDate``/``endDate`` fall short of the + security's window by more than *tol_days*.""" + tol = dt.timedelta(days=tol_days) + start, end = _d(meta.get("startDate")), _d(meta.get("endDate")) + notes = [] + if sec.valid_from and start and start > sec.valid_from + tol: + notes.append(f"Tiingo starts {start}, after valid_from {sec.valid_from}") + if sec.valid_to and end and end < sec.valid_to - tol: + notes.append(f"Tiingo ends {end}, before valid_to {sec.valid_to}") + return "; ".join(notes) or None + + +def tri_monthly( + ticker: str, rows: Sequence[PriceRow], *, item_key: str +) -> list[MonthlyRow]: + """Month-end total-return index, 100 on the first stored day, chained + from the stored daily returns (so a later rebasing of adjusted prices + cannot break it).""" + level = 100.0 + last: dict[tuple[int, int], float] = {} + for r in sorted(rows, key=lambda r: r.day): + if r.ret is not None: + level *= 1.0 + r.ret + last[(r.day.year, r.day.month)] = level + return [ + MonthlyRow(f"tiingo:{ticker}:tri", ticker, SOURCE, y, m, v, item_key) + for (y, m), v in sorted(last.items()) + ] + + +def cite(store: Any, ticker: str) -> str: + """A bib ``source`` item for *ticker*'s Tiingo series: a citation only, + never an attachment (the prices are internal-use).""" + from bib.item import Source + from bib.tag import Tag + + item = Source( + title=f"Tiingo end-of-day prices: {ticker}", + url=EOD.format(ticker=ticker), + doc_type="dataset", + abstract="Internal use only (Tiingo licence); raw prices stay in " + "data/market/tiingo and market.price_daily.", + access_date=today().isoformat(), + ) + return store.upsert( + item, + tags=[ + Tag.module("market"), + Tag.table("market.price_daily"), + Tag.source("tiingo"), + ], + ) + + +# ── readers over market.price_daily ────────────────────────────────── + + +def read_returns( + con: Any, tickers: Sequence[str], start: date, end: date +) -> dict[str, dict[date, float]]: + """``{ticker: {day: ret}}`` over ``start..end``; days without a return + are dropped.""" + out: dict[str, dict[date, float]] = {t: {} for t in tickers} + if not tickers: + return out + marks = ",".join("?" * len(tickers)) + for t, day, ret in con.execute( + "SELECT ticker, day, ret FROM market.price_daily " + f"WHERE ticker IN ({marks}) AND day BETWEEN ? AND ? AND ret IS NOT NULL " + "ORDER BY day", + [*tickers, start, end], + ).fetchall(): + out[t][day] = ret + return out + + +def trading_days(con: Any, ticker: str = "SPY") -> list[date]: + return [ + d + for (d,) in con.execute( + "SELECT day FROM market.price_daily WHERE ticker = ? ORDER BY day", + [ticker], + ).fetchall() + ] + + +def last_pulls(con: Any) -> dict[str, tuple[date, date]]: + """``{ticker: (last stored day, last pulled_on)}``.""" + return { + t: (d, p) + for t, d, p in con.execute( + "SELECT ticker, max(day), max(pulled_on) FROM market.price_daily " + "GROUP BY ticker" + ).fetchall() + } + + +# ── the ranked universe (M4) ───────────────────────────────────────── + + +def _seed_ciks() -> list[str]: + from market.seed import load_seed + + return [c.cik for c in load_seed()[0]] + + +def universe( + con: Any, + *, + used: set[str], + cap: int = MONTHLY_SYMBOLS, + supported: Mapping[str, Any] | None = None, + seed_order: Sequence[str] | None = None, +) -> list[tuple[str, str]]: + """``(ticker, entity_key)`` in rank order: the benchmarks, the seed + securities in seed order, then the securities of entities with a + filing-text (``fts``) link, ranked by strongest link, number of distinct + link targets and newest 10-K. SIC-only entities and unsupported or + unpriced tickers are excluded; at most *cap* symbols not already in + *used* are returned.""" + by_entity: dict[str, list[str]] = defaultdict(list) + for s in read_securities(con): + if s.priced: + by_entity[s.entity_key].append(s.ticker) + + ranked: list[tuple[str, str]] = [(t, f"index:{t}") for t in BENCHMARKS] + for key in seed_order if seed_order is not None else _seed_ciks(): + ranked += [(t, key) for t in by_entity.get(key, [])] + + best: dict[str, int] = {} + targets: dict[str, set[str]] = defaultdict(set) + for link in read_links(con, basis="fts"): + rank = _STRENGTH.get(link.strength, len(_STRENGTH)) + best[link.entity_key] = min(best.get(link.entity_key, rank), rank) + targets[link.entity_key].add(link.target) + newest = { + cik: filed + for cik, filed in con.execute( + "SELECT cik, max(filed) FROM market.filing " + "WHERE form IN ('10-K', '10-K/A') GROUP BY cik" + ).fetchall() + } + for key in sorted( + best, + key=lambda k: ( + best[k], + -len(targets[k]), + -(newest[k].toordinal() if newest.get(k) else 0), + k, + ), + ): + ranked += [(t, key) for t in by_entity.get(key, [])] + + out: list[tuple[str, str]] = [] + seen: set[str] = set() + fresh = 0 + for ticker, key in ranked: + if ticker in seen or (supported is not None and ticker not in supported): + continue + if ticker not in used: + if fresh >= cap: + continue + fresh += 1 + seen.add(ticker) + out.append((ticker, key)) + return out + + +# ── what to pull ───────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Target: + ticker: str + entity_key: str + #: ``(valid_from, valid_to)`` of the securities carrying this ticker; + #: empty when none is known + windows: tuple[tuple[date | None, date | None], ...] + #: where a first pull starts + start: date + securities: tuple[SecurityRow, ...] = () + + +def plan( + con: Any, + *, + tickers: Sequence[str], + universe_on: bool, + benchmarks_on: bool, + used: set[str], + supported: Mapping[str, Any] | None = None, + cap: int = MONTHLY_SYMBOLS, + seed_order: Sequence[str] | None = None, +) -> list[Target]: + """Benchmarks, then the named tickers, then the ranked universe.""" + names: list[tuple[str, str]] = [] + if benchmarks_on: + names += [(t, f"index:{t}") for t in BENCHMARKS] + secs: dict[str, list[SecurityRow]] = defaultdict(list) + for s in read_securities(con): + secs[s.ticker].append(s) + for t in tickers: + t = t.upper() + names.append((t, secs[t][0].entity_key if secs.get(t) else "")) + if universe_on: + names += universe( + con, used=used, cap=cap, supported=supported, seed_order=seed_order + ) + out: list[Target] = [] + seen: set[str] = set() + for ticker, key in names: + if ticker in seen: + continue + seen.add(ticker) + rows = secs.get(ticker, []) if ticker not in BENCHMARKS else [] + windows = tuple((s.valid_from, s.valid_to) for s in rows) + if ticker in BENCHMARKS: + start = BENCHMARKS[ticker][1] + elif windows and all(w[0] for w in windows): + start = max(FLOOR, min(w[0] for w in windows)) + else: + start = FLOOR + out.append(Target(ticker, key, windows, start, tuple(rows))) + return out + + +def clip_windows( + rows: Sequence[PriceRow], windows: Sequence[tuple[date | None, date | None]] +) -> list[PriceRow]: + """Rows inside any of *windows* (all rows when there is none).""" + if not windows: + return list(rows) + return [r for r in rows if any(clip([r], a, b) for a, b in windows)] diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index 5b6b654..79a538e 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -587,3 +587,335 @@ def test_entities_stores_ticker_history_and_link_dates(env): if l.entity_key == "0001801170" ] assert clov.valid_to == date(2023, 12, 31) + + +# ── stack market prices ────────────────────────────────────────────── +import io # noqa: E402 +import zipfile # noqa: E402 + +from market import tiingo # noqa: E402 + +DAY = date(2026, 11, 12) + + +def _supported_zip(*rows: str) -> bytes: + head = "ticker,exchange,assetType,priceCurrency,startDate,endDate\n" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + z.writestr("supported_tickers.csv", head + "\n".join(rows) + "\n") + return buf.getvalue() + + +def _eod(day: str, close: float, adj: float | None = None) -> dict: + return { + "date": f"{day}T00:00:00.000Z", + "open": close, + "high": close, + "low": close, + "close": close, + "volume": 5, + "adjClose": close if adj is None else adj, + "divCash": 0.0, + "splitFactor": 1.0, + } + + +@pytest.fixture +def px(env, monkeypatch, tmp_path): + c, store, published, hoard = env + monkeypatch.setenv("TIINGO_API_KEY", "secret-key") + monkeypatch.setattr(tiingo, "state_dir", lambda: tmp_path / "state") + monkeypatch.setattr(tiingo, "new_pacer", lambda: _Pacer()) + monkeypatch.setattr(tiingo, "today", lambda: DAY) + seen: list[httpx.Request] = [] + series = { + "AAA": [ + _eod("2024-01-02", 100), + _eod("2024-01-03", 110), + _eod("2024-01-04", 99), + ], + "BBB": [_eod("2024-01-02", 10), _eod("2024-01-03", 11)], + "SPY": [_eod("2024-01-02", 400), _eod("2024-01-03", 404)], + "XLV": [_eod("2024-01-02", 100)], + "IHF": [_eod("2024-01-02", 50)], + } + z = _supported_zip( + "AAA,NYSE,Stock,USD,2000-01-01,2026-11-12", + "BBB,NYSE,Stock,USD,2000-01-01,2026-11-12", + "SPY,NYSE,ETF,USD,1993-01-29,2026-11-12", + "XLV,NYSE,ETF,USD,1998-12-22,2026-11-12", + "IHF,NYSE,ETF,USD,2006-05-05,2026-11-12", + ) + + def prices_for(m): + if m.group(1) not in series: + return httpx.Response(404, json={"detail": "Not found."}) + return httpx.Response(200, json=series[m.group(1)]) + + routes = [ + (r"supported_tickers\.zip", lambda m: httpx.Response(200, content=z)), + (r"/daily/(\w+)/prices", prices_for), + ] + + def handler(request): + seen.append(request) + return _handler(routes)(request) + + monkeypatch.setattr( + http, + "client", + lambda source, key=None: httpx.Client( + transport=httpx.MockTransport(handler), + headers={"Authorization": f"Token {key}"} if key else {}, + ), + ) + return c, store, published, hoard, seen, series, tmp_path / "state" + + +def _price_requests(seen): + return [r for r in seen if "/prices" in r.url.path] + + +def test_prices_cli_fetch_write(px): + c, _, published, hoard, seen, _, state = px + res = runner.invoke( + app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"] + ) + assert res.exit_code == 0, res.output + assert _count(c, "market.price_daily") == 5 + assert [r.url.params["startDate"] for r in _price_requests(seen)] == [ + "1993-01-29" + ] * 2 + assert all("secret-key" not in str(r.url) for r in seen) + keyed = [r for r in seen if "/prices" in r.url.path] + assert all(r.headers["authorization"] == "Token secret-key" for r in keyed) + free = [r for r in seen if "supported" in r.url.path] + assert free and "authorization" not in free[0].headers + assert { + (t, n) + for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall() + } == {("market.price_daily", 5), ("market.series", 2)} + assert [s.value for s in T.read_series(c, ["AAA"])] == [pytest.approx(99.0)] + assert published == [True] + used = json.loads((state / "tiingo-2026-11.json").read_text()) + assert used["symbols"] == ["AAA", "BBB"] and used["day_counts"] == {"2026-11-12": 2} + assert (hoard / "tiingo" / "eod" / "AAA" / "1993-01-29_2026-11-12.json").is_file() + assert "priced 2 symbols, 5 rows" in res.output and "2/450" in res.output + assert T.read_gaps(c) == [] + + +def test_prices_incremental_pull_and_same_day_skip(px, monkeypatch): + c, _, _, _, seen, series, _ = px + cmd = ["market", "prices", "--ticker", "AAA", "--write"] + assert runner.invoke(app, cmd).exit_code == 0 + before = len(_price_requests(seen)) + again = runner.invoke(app, cmd) + assert len(_price_requests(seen)) == before # priced today: nothing to do + assert "priced 0 symbols" in again.output + monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13)) + series["AAA"] = [_eod("2024-01-04", 99, 49.5), _eod("2024-01-05", 108.9, 54.45)] + assert runner.invoke(app, cmd).exit_code == 0 + last = _price_requests(seen)[-1] + assert last.url.params["startDate"] == "2024-01-04" + rows = {r.day: r for r in T.read_prices(c, "AAA")} + assert len(rows) == 4 and rows[date(2024, 1, 4)].adj_close == 99 + assert rows[date(2024, 1, 5)].ret == pytest.approx(0.1) + assert [m.value for m in T.read_series(c, ["AAA"])] == [pytest.approx(108.9)] + + +def test_prices_benchmarks_start_at_listing(px): + c, _, _, _, seen, _, _ = px + res = runner.invoke(app, ["market", "prices", "--benchmarks", "--write"]) + assert res.exit_code == 0, res.output + starts = { + r.url.path.split("/")[3]: r.url.params["startDate"] + for r in _price_requests(seen) + } + assert starts == {"SPY": "1993-01-29", "XLV": "1998-12-22", "IHF": "2006-05-05"} + + +def test_prices_dry_run_reports_budget(px, monkeypatch): + c, _, published, _, seen, _, state = px + T.ensure_tables(c) + T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")]) + T.write_links(c, [T.LinkRow("K1", "program", "PFS", "fts", "mention", "adsh", "a")]) + state.mkdir(parents=True) + (state / "tiingo-2026-11.json").write_text( + json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}}) + ) + monkeypatch.setattr( + mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch")) + ) + res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"]) + assert res.exit_code == 0, res.output + assert "1/450 symbols used, 449 remaining" in res.output + assert "1. SPY" in res.output and "used" in res.output + assert "AAA" in res.output and "K1" in res.output and "first pull" in res.output + assert _price_requests(seen) == [] and published == [] + assert not (state / "tiingo-2026-11.json").read_text().count("AAA") + + +def test_prices_dry_run_without_market_tables_lists_benchmarks(px): + res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"]) + assert res.exit_code == 0, res.output + assert "3 symbols ranked" in res.output + + +def test_prices_dry_run_shows_stored_span(px): + c, *_ = px + T.ensure_tables(c) + T.write_prices( + c, + "AAA", + tiingo.to_prices( + "AAA", [_eod("2024-01-02", 1)], pulled_on=DAY, source_file="f", overlap=None + ), + ) + res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--dry-run"]) + assert "stored to 2024-01-02" in res.output + + +def test_prices_without_key_records_gap(px, monkeypatch): + c, _, published, _, seen, _, _ = px + monkeypatch.delenv("TIINGO_API_KEY") + res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"]) + assert res.exit_code == 0, res.output + assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output + assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"] + assert _count(c, "market.price_daily") == 0 and seen == [] and published == [True] + + +def test_prices_refusal_prints_gap(px, monkeypatch): + c, _, published, _, _, _, _ = px + monkeypatch.setattr( + http, + "client", + lambda source, key=None: httpx.Client( + transport=httpx.MockTransport(lambda r: httpx.Response(403)) + ), + ) + res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"]) + assert res.exit_code == 0, res.output + assert "gap: tiingo:prices:" in res.output and "403" in res.output + assert _count(c, "market.price_daily") == 0 + assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"] and published == [ + True + ] + + +def test_prices_refusal_on_prices_after_supported_ok(px, monkeypatch): + c, _, published, _, _, _, _ = px + z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12") + routes = [ + (r"supported_tickers", lambda m: httpx.Response(200, content=z)), + (r"/prices", lambda m: httpx.Response(401)), + ] + _online(monkeypatch, routes) + res = runner.invoke(app, ["market", "prices", "--ticker", "AAA"]) + assert ( + res.exit_code == 0 + and "gap: tiingo:prices:" in res.output + and "401" in res.output + ) + assert published == [] + + +def test_prices_nothing_selected_exits(px): + res = runner.invoke(app, ["market", "prices"]) + assert res.exit_code == 2 and "nothing to price" in res.output + + +def test_prices_unknown_ticker_and_window_gap_become_gap_note(px): + c, _, _, _, _, series, _ = px + T.ensure_tables(c) + T.write_securities( + c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, date(2030, 1, 1), "s")] + ) + res = runner.invoke( + app, ["market", "prices", "--ticker", "BBB", "--ticker", "ZZZ", "--write"] + ) + assert res.exit_code == 0, res.output + assert "ZZZ: not found at Tiingo" in res.output + (gap,) = T.read_gaps(c) + assert ( + "ZZZ: not found at Tiingo" in gap.note + and "BBB: Tiingo ends 2026-11-12" in gap.note + ) + # BBB later prices cleanly: its entry goes, ZZZ's stays + T.write_securities(c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, None, "s")]) + runner.invoke(app, ["market", "prices", "--ticker", "BBB", "--refresh", "--write"]) + (gap,) = T.read_gaps(c) + assert gap.note == "ZZZ: not found at Tiingo" + + +def test_prices_clean_run_clears_gap_and_window_clips(px): + c, _, _, _, _, _, _ = px + T.ensure_tables(c) + T.write_gap(c, "tiingo:prices", "AAA: not found at Tiingo") + T.write_securities( + c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", date(2024, 1, 3), None, "s")] + ) + res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"]) + assert res.exit_code == 0, res.output + assert [r.day for r in T.read_prices(c, "AAA")] == [ + date(2024, 1, 3), + date(2024, 1, 4), + ] + assert T.read_gaps(c) == [] + + +def test_prices_limit_stops_downloads(px): + c, _, _, _, seen, _, _ = px + res = runner.invoke( + app, ["market", "prices", "--benchmarks", "--limit", "1", "--write"] + ) + assert res.exit_code == 0 and len(_price_requests(seen)) == 1 + assert [r.ticker for r in [*T.read_prices(c, "SPY")][:1]] == ["SPY"] + + +def test_prices_budget_exhausted_stops_and_writes_what_it_has(px): + c, _, published, _, seen, _, state = px + state.mkdir(parents=True) + (state / "tiingo-2026-11.json").write_text( + json.dumps( + { + "month": "2026-11", + "symbols": [f"S{i}" for i in range(449)] + ["AAA"], + "day_counts": {}, + } + ) + ) + res = runner.invoke( + app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"] + ) + assert res.exit_code == 0, res.output + assert "budget: stopped, BBB" in res.output + assert [r.ticker for r in T.read_prices(c, "AAA")][:1] == ["AAA"] + assert T.read_prices(c, "BBB") == [] and published == [True] + + +def test_prices_without_write_stores_nothing(px, monkeypatch): + c, _, published, _, _, _, _ = px + monkeypatch.setattr( + mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch")) + ) + res = runner.invoke(app, ["market", "prices", "--ticker", "ZZZ"]) + assert res.exit_code == 0, res.output + assert ( + "dry run: pass --write" in res.output + and "gap: tiingo:prices: ZZZ" in res.output + ) + assert published == [] + + +def test_merge_gap_keeps_other_tickers_and_orders(): + assert mk._merge_gap("", {}, set()) == "" + assert mk._merge_gap("A: x | B: y", {"C": "z"}, {"A"}) == "B: y | C: z" + + +def test_or_empty_reraises_other_errors(): + def boom(_): + raise RuntimeError("disk") + + with pytest.raises(RuntimeError): + mk._or_empty(boom, None, {}) diff --git a/tests/market/fixtures/supported_tickers_head.csv b/tests/market/fixtures/supported_tickers_head.csv new file mode 100644 index 0000000..dd07cad --- /dev/null +++ b/tests/market/fixtures/supported_tickers_head.csv @@ -0,0 +1,7 @@ +ticker,exchange,assetType,priceCurrency,startDate,endDate +000001,SHE,Stock,CNY,2007-01-04,2026-11-12 +ELV,NYSE,Stock,USD,2001-10-30,2026-11-12 +IHF,NYSE,ETF,USD,2006-05-05,2026-11-12 +OSH,NYSE,Stock,USD,2020-08-06,2023-05-02 +SPY,NYSE,ETF,USD,1993-01-29,2026-11-12 +XLV,NYSE,ETF,USD,1998-12-22,2026-11-12 diff --git a/tests/market/test_imports.py b/tests/market/test_imports.py index 194062a..272132d 100644 --- a/tests/market/test_imports.py +++ b/tests/market/test_imports.py @@ -6,7 +6,13 @@ from __future__ import annotations import subprocess import sys -MODULES = ["market.http", "market.tables", "market.seed", "market.edgar"] +MODULES = [ + "market.http", + "market.tables", + "market.seed", + "market.edgar", + "market.tiingo", +] _PROBE = """ import builtins, importlib, sys diff --git a/tests/market/test_tables.py b/tests/market/test_tables.py index 855984e..948924a 100644 --- a/tests/market/test_tables.py +++ b/tests/market/test_tables.py @@ -9,6 +9,7 @@ import duckdb import pytest from market import tables as T +from market import tiingo as TI @pytest.fixture @@ -30,7 +31,15 @@ def test_ensure_tables_idempotent(): "WHERE table_schema = 'market'" ).fetchall() } - assert names == {"entity", "security", "entity_link", "filing", "source_gap"} + assert names == { + "entity", + "security", + "entity_link", + "filing", + "source_gap", + "price_daily", + "series", + } ENTITY = T.EntityRow( @@ -67,6 +76,24 @@ FILING = T.FilingRow( ) GAP = T.GapRow("sec-edgar:entities", "403", dt.date.today()) +PRICE = TI.PriceRow( + "OSH", + date(2021, 1, 4), + 10.0, + 11.0, + 9.5, + 10.5, + 1000.0, + 10.4, + 0.0, + 1.0, + 0.01, + 0.012, + date(2026, 11, 12), + "eod/OSH/x.json", +) +SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY") + ROUNDTRIP = { "entity": ( ENTITY, @@ -84,6 +111,16 @@ ROUNDTRIP = { lambda c, r: T.write_filings(c, r.cik, [r]), lambda c: T.read_filings(c, "0001564406"), ), + "price_daily": ( + PRICE, + lambda c, r: TI.write_prices(c, r.ticker, [r]), + lambda c: TI.read_prices(c, "OSH"), + ), + "series": ( + SERIES, + lambda c, r: TI.write_series(c, r.series_id, [r]), + lambda c: TI.read_series(c, ["OSH"]), + ), "gap": ( GAP, lambda c, r: T.write_gap(c, r.source, r.note), diff --git a/tests/market/test_tiingo.py b/tests/market/test_tiingo.py new file mode 100644 index 0000000..aeac177 --- /dev/null +++ b/tests/market/test_tiingo.py @@ -0,0 +1,427 @@ +"""market.tiingo: returns, windows, budget, ranked universe, monthly series. +Prices here are synthetic (Tiingo's licence forbids sharing real ones).""" + +from __future__ import annotations + +import io +import zipfile +from datetime import date +from pathlib import Path + +import duckdb +import httpx +import pytest + +from market import http +from market import tables as T +from market import tiingo as TI + +FX = Path(__file__).parent / "fixtures" + + +def rec(day, close, adj=None, div=0.0, split=1.0): + return { + "date": f"{day}T00:00:00.000Z", + "open": close, + "high": close, + "low": close, + "close": close, + "volume": 10, + "adjClose": close if adj is None else adj, + "divCash": div, + "splitFactor": split, + } + + +def pull(ticker, records, overlap=None): + return TI.to_prices( + ticker, + records, + pulled_on=date(2026, 11, 12), + source_file="f.json", + overlap=overlap, + ) + + +def test_to_prices_returns_within_pull(): + rows = pull( + "X", + [ + rec("2024-01-02", 1, 100), + rec("2024-01-03", 1, 110), + rec("2024-01-04", 1, 99), + ], + ) + assert [r.ret for r in rows] == [None, pytest.approx(0.1), pytest.approx(-0.1)] + assert rows[0].day == date(2024, 1, 2) and rows[0].source_file == "f.json" + + +def test_ret_raw_with_split_and_dividend(): + rows = pull("X", [rec("2024-01-02", 100), rec("2024-01-03", 50, split=2.0)]) + assert rows[1].ret_raw == pytest.approx(0.0) + rows = pull("X", [rec("2024-01-02", 100), rec("2024-01-03", 99, div=1.0)]) + assert rows[1].ret_raw == pytest.approx(0.0) + assert rows[1].div_cash == 1.0 and rows[1].split_factor == 1.0 + + +def test_incremental_pull_rebased_adjclose_keeps_returns(): + d1, d2, d3 = date(2024, 1, 2), date(2024, 1, 3), date(2024, 1, 4) + first = pull("X", [rec(str(d1), 100, 100), rec(str(d2), 110, 110)]) + second = pull("X", [rec(str(d2), 110, 55), rec(str(d3), 121, 60.5)], overlap=d2) + assert [r.day for r in second] == [d3] + assert second[0].ret == pytest.approx(0.1) + con = duckdb.connect(":memory:") + T.ensure_tables(con) + TI.write_prices(con, "X", first) + TI.write_prices(con, "X", second) + stored = {r.day: r for r in TI.read_prices(con, "X")} + assert stored[d2].adj_close == 110 and stored[d2].ret == pytest.approx(0.1) + assert stored[d3].ret == pytest.approx(0.1) + + +def test_prices_clipped_to_security_window(): + rows = pull("AGL", [rec("2015-06-01", 1), rec("2021-05-03", 1)]) + kept = TI.clip(rows, date(2021, 4, 15), None) + assert [r.day for r in kept] == [date(2021, 5, 3)] + assert [r.day for r in TI.clip(rows, None, date(2016, 1, 1))] == [date(2015, 6, 1)] + assert len(TI.clip(rows, None, None)) == 2 + + +def test_window_gap(): + sec = T.SecurityRow( + "1", "AGL", "NYSE", date(2021, 4, 15), date(2023, 6, 30), "seed" + ) + ok = {"startDate": "2021-04-15", "endDate": "2023-06-30"} + assert TI.window_gap(ok, sec) is None + note = TI.window_gap({"startDate": "2021-04-15", "endDate": "2023-05-02"}, sec) + assert "2023-05-02" in note and "2023-06-30" in note + late = TI.window_gap({"startDate": "2021-06-01", "endDate": None}, sec) + assert "2021-06-01" in late and "2021-04-15" in late + assert TI.window_gap({}, sec) is None + assert TI.window_gap({"startDate": "2021-04-20"}, sec) is None # within tolerance + + +def test_budget_monthly_symbols_and_daily_cap(tmp_path): + today = date(2026, 11, 12) + b = TI.Budget("2026-11", {f"S{i}" for i in range(450)}, {}) + assert not b.allow("NEW", today) + assert b.allow("S7", today) + b2 = TI.Budget("2026-11", set(), {today.isoformat(): 950}) + assert not b2.allow("S7", today) + b3 = TI.Budget("2026-11", set(), {}) + b3.spend("A", today) + b3.spend("A", today) + assert b3.symbols == {"A"} and b3.day_counts == {"2026-11-12": 2} + + +def test_budget_persists_and_rolls_over(tmp_path, monkeypatch): + monkeypatch.setattr(TI, "state_dir", lambda: tmp_path) + today = date(2026, 11, 12) + b = TI.Budget.load(today) + b.spend("A", today) + again = TI.Budget.load(today) + assert again.symbols == {"A"} and again.remaining_symbols == 449 + nxt = TI.Budget.load(date(2026, 12, 1)) + assert nxt.symbols == set() and nxt.month == "2026-12" + assert (tmp_path / "tiingo-2026-11.json").is_file() + + +def test_monthly_tri_month_end(): + rows = pull( + "X", + [ + rec("2024-01-02", 1, 100), + rec("2024-01-31", 1, 110), + rec("2024-02-29", 1, 99), + ], + ) + out = TI.tri_monthly("X", rows, item_key="K") + assert [(m.year, m.month, m.value) for m in out] == [ + (2024, 1, pytest.approx(110.0)), + (2024, 2, pytest.approx(99.0)), + ] + assert out[0].series_id == "tiingo:X:tri" and out[0].item_key == "K" + assert out[0].symbol == "X" and out[0].source == "tiingo" + assert TI.tri_monthly("X", [], item_key="K") == [] + + +def _universe_db(): + con = duckdb.connect(":memory:") + T.ensure_tables(con) + + def sec(key, ticker, priced=True): + T.write_securities( + con, key, [T.SecurityRow(key, ticker, "NYSE", None, None, "s", priced)] + ) + + def link(key, basis, target, strength): + T.write_links( + con, [T.LinkRow(key, "program", target, basis, strength, "adsh", "a")] + ) + + def tenk(key, filed): + T.write_filings( + con, key, [T.FilingRow(key, f"A-{key}", "10-K", filed, "", None, "", "")] + ) + + sec("SEED", "SEEDCO") + sec("OLD", "OLDCO", priced=False) + sec("REV", "REVCO") + sec("MEN", "MENCO") + sec("MEN2", "MEN2CO") + sec("SIC", "SICCO") + sec("GONE", "GONECO") + link("SEED", "curated", "PFS", "curated") + link("REV", "fts", "PFS", "revenue_exposure") + link("MEN", "fts", "PFS", "mention") + link("MEN", "fts", "APM", "mention") + link("MEN2", "fts", "PFS", "mention") + link("SIC", "sic", "8082", "sector") + link("GONE", "fts", "PFS", "revenue_exposure") + tenk("MEN", date(2020, 1, 1)) + tenk("MEN2", date(2025, 1, 1)) + return con + + +SUPPORTED = { + t: (date(2000, 1, 1), None) + for t in ( + "SPY", + "XLV", + "IHF", + "SEEDCO", + "REVCO", + "MENCO", + "MEN2CO", + "SICCO", + "OLDCO", + ) +} + + +def test_universe_ranked_capped_excludes_sic_only(): + con = _universe_db() + kw = {"supported": SUPPORTED, "seed_order": ["SEED"]} + got = [t for t, _ in TI.universe(con, used=set(), **kw)] + assert got[:3] == ["SPY", "XLV", "IHF"] + assert got[3:] == ["SEEDCO", "REVCO", "MENCO", "MEN2CO"] + assert "SICCO" not in got and "OLDCO" not in got and "GONECO" not in got + keys = dict(TI.universe(con, used=set(), **kw)) + assert keys["REVCO"] == "REV" and keys["SPY"] == "index:SPY" + assert [t for t, _ in TI.universe(con, used=set(), cap=5, **kw)] == got[:5] + capped = [t for t, _ in TI.universe(con, used={"SPY"}, cap=5, **kw)] + assert capped[:2] == ["SPY", "XLV"] and len(capped) == 6 + assert "SEEDCO" in capped and "REVCO" in capped and "MEN2CO" not in capped + + +def test_universe_default_seed_order_reads_the_seed(): + con = duckdb.connect(":memory:") + T.ensure_tables(con) + assert [t for t, _ in TI.universe(con, used=set())] == ["SPY", "XLV", "IHF"] + + +def _zip(csv_bytes: bytes) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + z.writestr("supported_tickers.csv", csv_bytes) + return buf.getvalue() + + +def test_parse_supported(): + got = TI.parse_supported(_zip((FX / "supported_tickers_head.csv").read_bytes())) + assert got["OSH"] == (date(2020, 8, 6), date(2023, 5, 2)) + assert "000001" not in got + assert got["SPY"][0] == date(1993, 1, 29) + extra = b"ticker,exchange,assetType,priceCurrency,startDate,endDate\nZZ,NYSE,Stock,USD,,\nZZ,NYSE,ETF,USD,2001-01-01,2002-01-01\n" + assert TI.parse_supported(_zip(extra))["ZZ"] == (date(2001, 1, 1), date(2002, 1, 1)) + + +def _client(routes, seen): + def handler(request: httpx.Request): + seen.append(request) + for frag, make in routes: + if frag in str(request.url): + return make(request) + return httpx.Response(404) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +class _Pacer: + def wait(self): + pass + + +@pytest.fixture +def hoard(tmp_path, monkeypatch): + monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market") + monkeypatch.setattr(TI, "state_dir", lambda: tmp_path / "state") + return tmp_path / "market" + + +def test_benchmark_first_pull_starts_at_listing(hoard): + seen = [] + cl = _client( + [("/prices", lambda r: httpx.Response(200, json=[rec("2024-01-02", 1)]))], seen + ) + budget = TI.Budget("2026-11", set(), {}) + for t in ("XLV", "SPY"): + start = TI.BENCHMARKS[t][1] + TI.fetch_eod( + cl, + t, + start, + date(2026, 11, 12), + pacer=_Pacer(), + budget=budget, + today=date(2026, 11, 12), + ) + starts = {r.url.path.split("/")[3]: r.url.params["startDate"] for r in seen} + assert starts == {"XLV": "1998-12-22", "SPY": "1993-01-29"} + assert all(r.url.params["resampleFreq"] == "daily" for r in seen) + assert all("token" not in str(r.url).lower() for r in seen) + assert TI.BENCHMARKS["IHF"] == ("providers", date(2006, 5, 5)) + + +def test_fetch_eod_caches_and_never_overwrites(hoard): + seen = [] + body = [rec("2024-01-02", 1)] + cl = _client([("/prices", lambda r: httpx.Response(200, json=body))], seen) + budget = TI.Budget("2026-11", set(), {}) + args = (cl, "X", date(2024, 1, 1), date(2024, 1, 5)) + kw = {"pacer": _Pacer(), "budget": budget, "today": date(2026, 11, 12)} + recs, path = TI.fetch_eod(*args, **kw) + assert recs == body and path.parent.name == "X" + again, _ = TI.fetch_eod(*args, **kw) + assert again == body and len(seen) == 1 # cache hit costs nothing + body[:] = [rec("2024-01-02", 2)] + changed, p2 = TI.fetch_eod(*args, refresh=True, **kw) + assert changed == body and p2 != path and path.is_file() + assert budget.symbols == {"X"} and budget.day_counts["2026-11-12"] == 2 + + +def test_fetch_eod_unknown_ticker_and_budget(hoard): + cl = _client([], []) + budget = TI.Budget("2026-11", set(), {}) + kw = {"pacer": _Pacer(), "today": date(2026, 11, 12)} + assert TI.fetch_eod( + cl, "NOPE", date(2024, 1, 1), date(2024, 1, 5), budget=budget, **kw + ) == (None, None) + full = TI.Budget("2026-11", {f"S{i}" for i in range(450)}, {}) + with pytest.raises(TI.BudgetExhausted): + TI.fetch_eod(cl, "NEW", date(2024, 1, 1), date(2024, 1, 5), budget=full, **kw) + + +def test_fetch_supported_hoards_and_reads_cache(hoard): + seen = [] + z = _zip((FX / "supported_tickers_head.csv").read_bytes()) + cl = _client( + [("supported_tickers.zip", lambda r: httpx.Response(200, content=z))], seen + ) + got = TI.fetch_supported(cl, pacer=_Pacer()) + assert got["OSH"][1] == date(2023, 5, 2) + assert (hoard / "tiingo" / "supported_tickers.zip").is_file() + assert TI.fetch_supported(None, pacer=_Pacer(), offline=True)["SPY"] + assert TI.fetch_supported(cl, pacer=_Pacer())["SPY"] and len(seen) == 1 + assert len(TI.fetch_supported(cl, pacer=_Pacer(), refresh=True)) == len(got) + + +def test_fetch_supported_offline_without_cache(hoard): + assert TI.fetch_supported(None, pacer=_Pacer(), offline=True) == {} + + +class _Store: + def __init__(self): + self.items = [] + self.attached = [] + + def upsert(self, item, tags=()): + self.items.append(item) + return "KEY1" + + def attach_file(self, *a, **k): # pragma: no cover - must never run + self.attached.append(a) + + +def test_tiingo_cite_has_no_attachment(): + store = _Store() + assert TI.cite(store, "SPY") == "KEY1" + assert store.attached == [] + assert "token" not in store.items[0].url.lower() and "SPY" in store.items[0].url + + +def test_plan_targets_windows_and_start(): + con = duckdb.connect(":memory:") + T.ensure_tables(con) + T.write_securities( + con, "C", [T.SecurityRow("C", "CI", "NYSE", date(2018, 12, 21), None, "s")] + ) + T.write_securities(con, "E", [T.SecurityRow("E", "ELV", "NYSE", None, None, "s")]) + by = { + t.ticker: t + for t in TI.plan( + con, + tickers=["CI", "ELV", "ZZZ"], + universe_on=False, + benchmarks_on=True, + used=set(), + ) + } + assert by["CI"].start == date(2018, 12, 21) and by["CI"].windows == ( + (date(2018, 12, 21), None), + ) + assert by["ELV"].start == TI.FLOOR and by["ZZZ"].windows == () + assert by["SPY"].start == date(1993, 1, 29) and by["XLV"].start == date( + 1998, 12, 22 + ) + assert by["CI"].securities[0].ticker == "CI" + + +def test_roundtrip_helpers(): + con = duckdb.connect(":memory:") + T.ensure_tables(con) + spy = pull("SPY", [rec("2024-01-02", 1, 100), rec("2024-01-03", 1, 101)]) + aaa = pull("AAA", [rec("2024-01-02", 1, 100), rec("2024-01-03", 1, 90)]) + TI.write_prices(con, "SPY", spy) + TI.write_prices(con, "AAA", aaa) + assert TI.write_prices(con, "AAA", []) == 0 + got = TI.read_returns( + con, ["SPY", "AAA", "NONE"], date(2024, 1, 1), date(2024, 1, 31) + ) + assert got["SPY"] == {date(2024, 1, 3): pytest.approx(0.01)} + assert got["AAA"] == {date(2024, 1, 3): pytest.approx(-0.1)} and got["NONE"] == {} + assert ( + TI.read_returns(con, ["SPY"], date(2024, 1, 4), date(2024, 2, 1))["SPY"] == {} + ) + assert TI.trading_days(con) == [date(2024, 1, 2), date(2024, 1, 3)] + assert TI.read_returns(con, [], date(2024, 1, 1), date(2024, 1, 2)) == {} + m = TI.tri_monthly("SPY", spy, item_key="K") + TI.write_series(con, "tiingo:SPY:tri", m) + TI.write_series(con, "tiingo:SPY:tri", m) + assert TI.read_series(con, ["SPY"]) == m and TI.read_series(con, []) == [] + assert TI.write_series(con, "x", []) == 0 + assert TI.last_pulls(con) == { + "AAA": (date(2024, 1, 3), date(2026, 11, 12)), + "SPY": (date(2024, 1, 3), date(2026, 11, 12)), + } + + +def test_pacer_and_state_dir_defaults(): + assert TI.new_pacer().min_interval == 75.0 + assert TI.state_dir().parts[-2:] == (".state", "market") + assert TI.today() == date.today() + + +def test_plan_deduplicates_names(): + con = duckdb.connect(":memory:") + T.ensure_tables(con) + got = TI.plan( + con, + tickers=["spy", "SPY"], + universe_on=True, + benchmarks_on=True, + used=set(), + seed_order=[], + ) + assert [t.ticker for t in got] == ["SPY", "XLV", "IHF"] + assert TI.clip_windows([], []) == []