feat(market): Tiingo daily prices for a ranked, capped universe; benchmarks SPY/XLV/IHF; monthly series (#797)
This commit is contained in:
@@ -17,7 +17,7 @@ from typing import Any
|
||||
|
||||
import typer
|
||||
|
||||
from market import edgar, http
|
||||
from market import edgar, http, tiingo
|
||||
from market.seed import (
|
||||
SEAT_LABEL_KEYS,
|
||||
SEED_PATH,
|
||||
@@ -34,19 +34,23 @@ from market.tables import (
|
||||
clear_gap,
|
||||
ensure_tables,
|
||||
read_gaps,
|
||||
read_prices,
|
||||
retire_links,
|
||||
retire_organizations,
|
||||
write_entities,
|
||||
write_filings,
|
||||
write_gap,
|
||||
write_links,
|
||||
write_prices,
|
||||
write_securities,
|
||||
write_series,
|
||||
)
|
||||
from pfs.codetables import is_missing_table_error
|
||||
|
||||
app = typer.Typer(no_args_is_help=True)
|
||||
|
||||
ENTITIES_GAP = "sec-edgar:entities"
|
||||
PRICES_GAP = "tiingo:prices"
|
||||
|
||||
|
||||
# ── indirections the tests monkeypatch ───────────────────────────────
|
||||
@@ -326,6 +330,217 @@ def entities(
|
||||
)
|
||||
|
||||
|
||||
# ── prices ───────────────────────────────────────────────────────────
|
||||
def _or_empty(read: Any, con: Any, empty: Any) -> Any:
|
||||
"""``read(con)``, or ``empty`` when a market table does not exist yet."""
|
||||
try:
|
||||
return read(con)
|
||||
except Exception as exc:
|
||||
if not is_missing_table_error(exc):
|
||||
raise
|
||||
return empty
|
||||
|
||||
|
||||
def _merge_gap(existing: str, issues: dict[str, str], fetched: set[str]) -> str:
|
||||
"""Earlier gap entries (``TICKER: reason`` joined by `` | ``) stay unless
|
||||
that ticker was priced this run; this run's issues are added."""
|
||||
kept = {}
|
||||
for part in filter(None, existing.split(" | ")):
|
||||
kept[part.split(":", 1)[0]] = part
|
||||
for t in fetched:
|
||||
kept.pop(t, None)
|
||||
kept.update({t: f"{t}: {note}" for t, note in issues.items()})
|
||||
return " | ".join(kept[t] for t in sorted(kept))
|
||||
|
||||
|
||||
@app.command()
|
||||
def prices(
|
||||
ticker: list[str] = typer.Option(
|
||||
[], "--ticker", help="Price this ticker (repeatable)."
|
||||
),
|
||||
universe: bool = typer.Option(
|
||||
False, "--universe", help="The ranked, capped universe (seed + filing-linked)."
|
||||
),
|
||||
benchmarks: bool = typer.Option(False, "--benchmarks", help="SPY, XLV and IHF."),
|
||||
limit: int = typer.Option(
|
||||
0, "--limit", help="Download at most this many symbols (0 = no limit)."
|
||||
),
|
||||
refresh: bool = typer.Option(
|
||||
False, "--refresh", help="Download again even if priced or hoarded today."
|
||||
),
|
||||
dry_run: bool = typer.Option(
|
||||
False, "--dry-run", help="Print the ranked universe and the budget only."
|
||||
),
|
||||
write: bool = typer.Option(
|
||||
False, "--write", help="Store prices, returns and the monthly series."
|
||||
),
|
||||
) -> None:
|
||||
"""Tiingo daily prices and total returns (market.price_daily) and the
|
||||
month-end total-return series (market.series, #797). Prices are internal
|
||||
use only. The monthly symbol allowance is 450; a first pull of a symbol
|
||||
starts at its listing, later pulls at the last stored day."""
|
||||
if not (ticker or universe or benchmarks):
|
||||
typer.echo("nothing to price: pass --ticker, --universe or --benchmarks")
|
||||
raise typer.Exit(2)
|
||||
key = None
|
||||
if not dry_run:
|
||||
key = _require_key("tiingo", PRICES_GAP, write=write)
|
||||
if key is None:
|
||||
return
|
||||
day = tiingo.today()
|
||||
budget = tiingo.Budget.load(day)
|
||||
pacer = tiingo.new_pacer()
|
||||
try:
|
||||
with http.client("tiingo") as free:
|
||||
supported = tiingo.fetch_supported(free, pacer=pacer, refresh=refresh)
|
||||
except http.Refused as exc:
|
||||
_gap(PRICES_GAP, str(exc), write=write)
|
||||
return
|
||||
|
||||
import duckdb
|
||||
|
||||
con = _read()
|
||||
try:
|
||||
state = _or_empty(tiingo.last_pulls, con, {})
|
||||
targets = _or_empty(
|
||||
lambda c: tiingo.plan(
|
||||
c,
|
||||
tickers=ticker,
|
||||
universe_on=universe,
|
||||
benchmarks_on=benchmarks,
|
||||
used=budget.symbols,
|
||||
supported=supported,
|
||||
),
|
||||
con,
|
||||
None,
|
||||
)
|
||||
finally:
|
||||
con.close()
|
||||
if targets is None:
|
||||
mem = duckdb.connect(":memory:")
|
||||
ensure_tables(mem)
|
||||
targets = tiingo.plan(
|
||||
mem,
|
||||
tickers=ticker,
|
||||
universe_on=False,
|
||||
benchmarks_on=benchmarks or universe,
|
||||
used=budget.symbols,
|
||||
supported=supported,
|
||||
)
|
||||
mem.close()
|
||||
|
||||
typer.echo(
|
||||
f"budget {budget.month}: {len(budget.symbols)}/{tiingo.MONTHLY_SYMBOLS} "
|
||||
f"symbols used, {budget.remaining_symbols} remaining; requests today "
|
||||
f"{budget.day_counts.get(day.isoformat(), 0)}/{tiingo.DAILY_REQUESTS}"
|
||||
)
|
||||
if dry_run:
|
||||
for i, t in enumerate(targets, 1):
|
||||
seen = state.get(t.ticker)
|
||||
note = f"stored to {seen[0]}" if seen else f"first pull from {t.start}"
|
||||
used = "used" if t.ticker in budget.symbols else "new"
|
||||
typer.echo(f"{i:>4}. {t.ticker:<8} {t.entity_key:<14} {used:<4} {note}")
|
||||
typer.echo(f"{len(targets)} symbols ranked")
|
||||
return
|
||||
|
||||
fetched: dict[str, list[Any]] = {}
|
||||
issues: dict[str, str] = {}
|
||||
n_downloaded = 0
|
||||
cl = http.client("tiingo", key)
|
||||
try:
|
||||
for t in targets:
|
||||
if limit and n_downloaded >= limit:
|
||||
break
|
||||
last = state.get(t.ticker)
|
||||
if last and last[1] == day and not refresh:
|
||||
continue
|
||||
start = last[0] if last else t.start
|
||||
try:
|
||||
records, path = tiingo.fetch_eod(
|
||||
cl,
|
||||
t.ticker,
|
||||
start,
|
||||
day,
|
||||
pacer=pacer,
|
||||
budget=budget,
|
||||
today=day,
|
||||
refresh=refresh,
|
||||
)
|
||||
except tiingo.BudgetExhausted as exc:
|
||||
typer.echo(f"budget: stopped, {exc}")
|
||||
break
|
||||
n_downloaded += 1
|
||||
if records is None:
|
||||
issues[t.ticker] = "not found at Tiingo"
|
||||
typer.echo(f"{t.ticker}: not found at Tiingo")
|
||||
continue
|
||||
new = tiingo.clip_windows(
|
||||
tiingo.to_prices(
|
||||
t.ticker,
|
||||
records,
|
||||
pulled_on=day,
|
||||
source_file=str(path),
|
||||
overlap=last[0] if last else None,
|
||||
),
|
||||
t.windows,
|
||||
)
|
||||
meta = {
|
||||
k: v
|
||||
for k, v in zip(
|
||||
("startDate", "endDate"),
|
||||
(supported.get(t.ticker) or (None, None)),
|
||||
strict=True,
|
||||
)
|
||||
}
|
||||
notes = [n for s in t.securities if (n := tiingo.window_gap(meta, s))]
|
||||
if notes:
|
||||
issues[t.ticker] = "; ".join(notes)
|
||||
fetched[t.ticker] = new
|
||||
span = f"{new[0].day}..{new[-1].day}" if new else "no new rows"
|
||||
typer.echo(f"{t.ticker}: +{len(new)} rows {span}")
|
||||
except http.Refused as exc:
|
||||
_gap(PRICES_GAP, str(exc), write=write)
|
||||
return
|
||||
finally:
|
||||
cl.close()
|
||||
|
||||
n_rows = sum(len(r) for r in fetched.values())
|
||||
typer.echo(
|
||||
f"priced {len(fetched)} symbols, {n_rows} rows; budget "
|
||||
f"{len(budget.symbols)}/{tiingo.MONTHLY_SYMBOLS} symbols"
|
||||
)
|
||||
for t, note in sorted(issues.items()):
|
||||
typer.echo(f"gap: {PRICES_GAP}: {t}: {note}")
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write to store")
|
||||
return
|
||||
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
store = _store()
|
||||
item_keys = {t: tiingo.cite(store, t) for t in fetched}
|
||||
n_series = 0
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
for t, rows in fetched.items():
|
||||
write_prices(con, t, rows)
|
||||
series = tiingo.tri_monthly(t, read_prices(con, t), item_key=item_keys[t])
|
||||
n_series += write_series(con, f"tiingo:{t}:tri", series)
|
||||
old = {g.source: g.note for g in read_gaps(con)}.get(PRICES_GAP, "")
|
||||
merged = _merge_gap(old, issues, set(fetched) - set(issues))
|
||||
if merged:
|
||||
write_gap(con, PRICES_GAP, merged)
|
||||
else:
|
||||
clear_gap(con, PRICES_GAP)
|
||||
src = str(http.hoard(http.MANIFEST))
|
||||
for table, n in (("market.price_daily", n_rows), ("market.series", n_series)):
|
||||
log_ingest(
|
||||
con, module="market.tiingo", table_name=table, rows=n, source_file=src
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: {n_rows} price rows, {n_series} monthly rows")
|
||||
|
||||
|
||||
@app.command()
|
||||
def gaps() -> None:
|
||||
"""Sources that could not be read (market.source_gap)."""
|
||||
|
||||
@@ -35,6 +35,16 @@ CREATE TABLE IF NOT EXISTS market.filing (
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.source_gap (
|
||||
source VARCHAR, note VARCHAR, recorded_on DATE
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.price_daily (
|
||||
ticker VARCHAR, day DATE, open DOUBLE, high DOUBLE, low DOUBLE,
|
||||
close DOUBLE, volume DOUBLE, adj_close DOUBLE, div_cash DOUBLE,
|
||||
split_factor DOUBLE, ret DOUBLE, ret_raw DOUBLE, pulled_on DATE,
|
||||
source_file VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.series (
|
||||
series_id VARCHAR, symbol VARCHAR, source VARCHAR, year INTEGER,
|
||||
month INTEGER, value DOUBLE, item_key VARCHAR
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -125,6 +135,42 @@ class GapRow:
|
||||
recorded_on: date
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PriceRow:
|
||||
"""One trading day of one ticker (internal use only: Tiingo's licence)."""
|
||||
|
||||
ticker: str
|
||||
day: date
|
||||
open: float
|
||||
high: float
|
||||
low: float
|
||||
close: float
|
||||
volume: float
|
||||
adj_close: float
|
||||
div_cash: float
|
||||
split_factor: float
|
||||
#: adjClose_t / adjClose_{t-1} - 1, within one pull (``None`` on its
|
||||
#: first row)
|
||||
ret: float | None
|
||||
#: (close_t + divCash_t) * splitFactor_t / close_{t-1} - 1
|
||||
ret_raw: float | None
|
||||
pulled_on: date
|
||||
source_file: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MonthlyRow:
|
||||
"""A month-end value of a series (``market.series``)."""
|
||||
|
||||
series_id: str
|
||||
symbol: str
|
||||
source: str
|
||||
year: int
|
||||
month: int
|
||||
value: float
|
||||
item_key: str
|
||||
|
||||
|
||||
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
|
||||
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
|
||||
a statement per row, far too slow for filing lists)."""
|
||||
@@ -335,6 +381,50 @@ def read_gaps(con: Any) -> list[GapRow]:
|
||||
return sorted((GapRow(*r) for r in rows), key=lambda g: g.source)
|
||||
|
||||
|
||||
# ── price_daily / series ─────────────────────────────────────────────
|
||||
|
||||
|
||||
def write_prices(con: Any, ticker: str, rows: Sequence[PriceRow]) -> int:
|
||||
"""Delete *ticker*'s rows over the days of *rows*, then insert them."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
days = [r.day for r in rows]
|
||||
con.execute(
|
||||
"DELETE FROM market.price_daily WHERE ticker = ? AND day BETWEEN ? AND ?",
|
||||
[ticker, min(days), max(days)],
|
||||
)
|
||||
return _insert(con, "market.price_daily", rows)
|
||||
|
||||
|
||||
def read_prices(con: Any, ticker: str) -> list[PriceRow]:
|
||||
rows = _select(con, PriceRow, "market.price_daily", "ticker = ?", [ticker])
|
||||
return sorted((PriceRow(*r) for r in rows), key=lambda r: r.day)
|
||||
|
||||
|
||||
def write_series(con: Any, series_id: str, rows: Sequence[MonthlyRow]) -> int:
|
||||
"""Replace the whole series *series_id*."""
|
||||
ensure_tables(con)
|
||||
con.execute("DELETE FROM market.series WHERE series_id = ?", [series_id])
|
||||
return _insert(con, "market.series", rows)
|
||||
|
||||
|
||||
def read_series(con: Any, symbols: Sequence[str]) -> list[MonthlyRow]:
|
||||
if not symbols:
|
||||
return []
|
||||
rows = _select(
|
||||
con,
|
||||
MonthlyRow,
|
||||
"market.series",
|
||||
f"symbol IN ({','.join('?' * len(symbols))})",
|
||||
list(symbols),
|
||||
)
|
||||
return sorted(
|
||||
(MonthlyRow(*r) for r in rows),
|
||||
key=lambda m: (m.series_id, m.year, m.month),
|
||||
)
|
||||
|
||||
|
||||
# ── districts ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
529
src/market/tiingo.py
Normal file
529
src/market/tiingo.py
Normal file
@@ -0,0 +1,529 @@
|
||||
"""Tiingo daily prices and total returns: the index ETFs, the seed companies
|
||||
and the filing-linked companies, ranked and capped to the monthly symbol
|
||||
allowance, plus a month-end total-return series (#797).
|
||||
|
||||
Prices are internal use only (Tiingo's licence): they live in
|
||||
``data/market/tiingo/`` and ``market.price_daily`` / ``market.series`` and are
|
||||
never attached to bib, committed or shared. The key travels only in the
|
||||
``Authorization`` header (:func:`market.http.client`).
|
||||
|
||||
Tiingo keeps a company's full history under its current ticker, so a
|
||||
security is pulled by the provider's ticker and clipped only by a window in
|
||||
which that ticker belonged to a different company (:func:`clip`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import datetime as dt
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from market import http
|
||||
from market.tables import ( # noqa: F401 (re-exported)
|
||||
MonthlyRow,
|
||||
PriceRow,
|
||||
SecurityRow,
|
||||
read_links,
|
||||
read_prices,
|
||||
read_securities,
|
||||
read_series,
|
||||
write_prices,
|
||||
write_series,
|
||||
)
|
||||
|
||||
SOURCE = "tiingo"
|
||||
EOD = "https://api.tiingo.com/tiingo/daily/{ticker}/prices"
|
||||
META = "https://api.tiingo.com/tiingo/daily/{ticker}"
|
||||
SUPPORTED = "https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip"
|
||||
BENCHMARKS = {
|
||||
"SPY": ("market", date(1993, 1, 29)),
|
||||
"XLV": ("sector", date(1998, 12, 22)),
|
||||
"IHF": ("providers", date(2006, 5, 5)),
|
||||
}
|
||||
#: no history is pulled from before SPY's first day
|
||||
FLOOR = date(1993, 1, 29)
|
||||
MONTHLY_SYMBOLS = 450
|
||||
DAILY_REQUESTS = 950
|
||||
PACE_SECONDS = 75.0
|
||||
_STRENGTH = {"revenue_exposure": 0, "risk_factor": 1, "mention": 2}
|
||||
|
||||
|
||||
def new_pacer() -> http.Pacer:
|
||||
return http.Pacer(PACE_SECONDS)
|
||||
|
||||
|
||||
def today() -> date:
|
||||
return dt.date.today()
|
||||
|
||||
|
||||
def state_dir() -> Path:
|
||||
return Path(__file__).resolve().parents[2] / ".state" / "market"
|
||||
|
||||
|
||||
# ── budget ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class BudgetExhausted(Exception):
|
||||
"""The monthly symbol allowance or the daily request cap is spent."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Budget:
|
||||
"""Tiingo usage this month: distinct symbols and requests per day."""
|
||||
|
||||
month: str
|
||||
symbols: set[str] = field(default_factory=set)
|
||||
day_counts: dict[str, int] = field(default_factory=dict)
|
||||
path: Path | None = None
|
||||
|
||||
@classmethod
|
||||
def load(cls, day: date) -> Budget:
|
||||
month = f"{day:%Y-%m}"
|
||||
path = state_dir() / f"tiingo-{month}.json"
|
||||
if path.is_file():
|
||||
raw = json.loads(path.read_text())
|
||||
return cls(
|
||||
month,
|
||||
set(raw.get("symbols", [])),
|
||||
{k: int(v) for k, v in raw.get("day_counts", {}).items()},
|
||||
path,
|
||||
)
|
||||
return cls(month, path=path)
|
||||
|
||||
def save(self) -> None:
|
||||
if self.path is None:
|
||||
return
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"month": self.month,
|
||||
"symbols": sorted(self.symbols),
|
||||
"day_counts": self.day_counts,
|
||||
},
|
||||
indent=1,
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def remaining_symbols(self) -> int:
|
||||
return MONTHLY_SYMBOLS - len(self.symbols)
|
||||
|
||||
def allow(self, ticker: str, day: date) -> bool:
|
||||
if self.day_counts.get(day.isoformat(), 0) >= DAILY_REQUESTS:
|
||||
return False
|
||||
return ticker in self.symbols or len(self.symbols) < MONTHLY_SYMBOLS
|
||||
|
||||
def spend(self, ticker: str, day: date) -> None:
|
||||
self.symbols.add(ticker)
|
||||
key = day.isoformat()
|
||||
self.day_counts[key] = self.day_counts.get(key, 0) + 1
|
||||
self.save()
|
||||
|
||||
|
||||
# ── supported tickers ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _d(text: Any) -> date | None:
|
||||
text = str(text or "").strip()[:10]
|
||||
return date.fromisoformat(text) if text else None
|
||||
|
||||
|
||||
def parse_supported(zip_bytes: bytes) -> dict[str, tuple[date | None, date | None]]:
|
||||
"""``supported_tickers.csv`` USD rows as ``{ticker: (start, end)}``
|
||||
(a ticker listed twice keeps its widest span)."""
|
||||
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as z:
|
||||
text = z.read("supported_tickers.csv").decode("utf-8-sig")
|
||||
out: dict[str, tuple[date | None, date | None]] = {}
|
||||
for row in csv.DictReader(io.StringIO(text)):
|
||||
if row.get("priceCurrency") != "USD":
|
||||
continue
|
||||
ticker = row["ticker"].strip().upper()
|
||||
start, end = _d(row.get("startDate")), _d(row.get("endDate"))
|
||||
if ticker in out:
|
||||
s0, e0 = out[ticker]
|
||||
start = min((d for d in (s0, start) if d), default=None)
|
||||
end = max((d for d in (e0, end) if d), default=None)
|
||||
out[ticker] = (start, end)
|
||||
return out
|
||||
|
||||
|
||||
def fetch_supported(
|
||||
cl: httpx.Client | None,
|
||||
*,
|
||||
pacer: Any,
|
||||
refresh: bool = False,
|
||||
offline: bool = False,
|
||||
) -> dict[str, tuple[date | None, date | None]]:
|
||||
"""The supported-ticker list (keyless, outside the budget), hoarded at
|
||||
``data/market/tiingo/supported_tickers.zip``."""
|
||||
path = http.hoard("tiingo", "supported_tickers.zip")
|
||||
have = http.newest(path)
|
||||
if have.is_file() and (offline or not refresh):
|
||||
return parse_supported(have.read_bytes())
|
||||
if offline or cl is None:
|
||||
return {}
|
||||
r = http.get(cl, SUPPORTED, pacer=pacer, source=SOURCE)
|
||||
saved = http.save(path, r.content)
|
||||
http.manifest(saved, SUPPORTED)
|
||||
return parse_supported(saved.read_bytes())
|
||||
|
||||
|
||||
# ── fetching ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def fetch_eod(
|
||||
cl: httpx.Client,
|
||||
ticker: str,
|
||||
start: date,
|
||||
end: date,
|
||||
*,
|
||||
pacer: Any,
|
||||
budget: Budget,
|
||||
today: date,
|
||||
refresh: bool = False,
|
||||
) -> tuple[list[dict[str, Any]] | None, Path | None]:
|
||||
"""End-of-day records of *ticker* for ``start..end``, hoarded at
|
||||
``data/market/tiingo/eod/<T>/<start>_<end>.json`` (a changed body lands
|
||||
beside the old one). A hoarded file costs nothing; a download spends the
|
||||
budget first. An unknown ticker is ``(None, None)``."""
|
||||
path = http.hoard("tiingo", "eod", ticker.replace("/", "_"), f"{start}_{end}.json")
|
||||
have = http.newest(path)
|
||||
if have.is_file() and not refresh:
|
||||
return json.loads(have.read_bytes()), have
|
||||
if not budget.allow(ticker, today):
|
||||
raise BudgetExhausted(
|
||||
f"{ticker}: {len(budget.symbols)}/{MONTHLY_SYMBOLS} symbols this month, "
|
||||
f"{budget.day_counts.get(today.isoformat(), 0)}/{DAILY_REQUESTS} requests today"
|
||||
)
|
||||
budget.spend(ticker, today)
|
||||
url = EOD.format(ticker=ticker)
|
||||
r = http.get(
|
||||
cl,
|
||||
url,
|
||||
params={
|
||||
"startDate": start.isoformat(),
|
||||
"endDate": end.isoformat(),
|
||||
"format": "json",
|
||||
"resampleFreq": "daily",
|
||||
},
|
||||
pacer=pacer,
|
||||
source=SOURCE,
|
||||
ok_404=True,
|
||||
)
|
||||
if r.status_code == 404:
|
||||
return None, None
|
||||
saved = http.save(path, r.content)
|
||||
http.manifest(saved, url)
|
||||
return json.loads(saved.read_bytes()), saved
|
||||
|
||||
|
||||
# ── returns ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def to_prices(
|
||||
ticker: str,
|
||||
records: Sequence[Mapping[str, Any]],
|
||||
*,
|
||||
pulled_on: date,
|
||||
source_file: str,
|
||||
overlap: date | None,
|
||||
) -> list[PriceRow]:
|
||||
"""Price rows with returns computed within this pull only (adjusted
|
||||
prices are rebased by later splits and dividends, so levels never mix
|
||||
across pulls). The *overlap* day, already stored with its own return, is
|
||||
the base of the next day's return and is not returned."""
|
||||
rows: list[PriceRow] = []
|
||||
prev: Mapping[str, Any] | None = None
|
||||
for rec in sorted(records, key=lambda r: str(r["date"])):
|
||||
close = float(rec["close"])
|
||||
adj = float(rec["adjClose"])
|
||||
div = float(rec.get("divCash") or 0.0)
|
||||
split = float(rec.get("splitFactor") or 1.0)
|
||||
ret = ret_raw = None
|
||||
if prev is not None:
|
||||
ret = adj / float(prev["adjClose"]) - 1.0
|
||||
ret_raw = (close + div) * split / float(prev["close"]) - 1.0
|
||||
prev = rec
|
||||
day = date.fromisoformat(str(rec["date"])[:10])
|
||||
if overlap is not None and day == overlap:
|
||||
continue
|
||||
rows.append(
|
||||
PriceRow(
|
||||
ticker,
|
||||
day,
|
||||
float(rec["open"]),
|
||||
float(rec["high"]),
|
||||
float(rec["low"]),
|
||||
close,
|
||||
float(rec.get("volume") or 0.0),
|
||||
adj,
|
||||
div,
|
||||
split,
|
||||
ret,
|
||||
ret_raw,
|
||||
pulled_on,
|
||||
source_file,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def clip(
|
||||
rows: Sequence[PriceRow], valid_from: date | None, valid_to: date | None
|
||||
) -> list[PriceRow]:
|
||||
"""Rows inside ``valid_from..valid_to`` (either end open)."""
|
||||
return [
|
||||
r
|
||||
for r in rows
|
||||
if (valid_from is None or r.day >= valid_from)
|
||||
and (valid_to is None or r.day <= valid_to)
|
||||
]
|
||||
|
||||
|
||||
def window_gap(
|
||||
meta: Mapping[str, Any], sec: SecurityRow, tol_days: int = 7
|
||||
) -> str | None:
|
||||
"""A note when Tiingo's ``startDate``/``endDate`` fall short of the
|
||||
security's window by more than *tol_days*."""
|
||||
tol = dt.timedelta(days=tol_days)
|
||||
start, end = _d(meta.get("startDate")), _d(meta.get("endDate"))
|
||||
notes = []
|
||||
if sec.valid_from and start and start > sec.valid_from + tol:
|
||||
notes.append(f"Tiingo starts {start}, after valid_from {sec.valid_from}")
|
||||
if sec.valid_to and end and end < sec.valid_to - tol:
|
||||
notes.append(f"Tiingo ends {end}, before valid_to {sec.valid_to}")
|
||||
return "; ".join(notes) or None
|
||||
|
||||
|
||||
def tri_monthly(
|
||||
ticker: str, rows: Sequence[PriceRow], *, item_key: str
|
||||
) -> list[MonthlyRow]:
|
||||
"""Month-end total-return index, 100 on the first stored day, chained
|
||||
from the stored daily returns (so a later rebasing of adjusted prices
|
||||
cannot break it)."""
|
||||
level = 100.0
|
||||
last: dict[tuple[int, int], float] = {}
|
||||
for r in sorted(rows, key=lambda r: r.day):
|
||||
if r.ret is not None:
|
||||
level *= 1.0 + r.ret
|
||||
last[(r.day.year, r.day.month)] = level
|
||||
return [
|
||||
MonthlyRow(f"tiingo:{ticker}:tri", ticker, SOURCE, y, m, v, item_key)
|
||||
for (y, m), v in sorted(last.items())
|
||||
]
|
||||
|
||||
|
||||
def cite(store: Any, ticker: str) -> str:
|
||||
"""A bib ``source`` item for *ticker*'s Tiingo series: a citation only,
|
||||
never an attachment (the prices are internal-use)."""
|
||||
from bib.item import Source
|
||||
from bib.tag import Tag
|
||||
|
||||
item = Source(
|
||||
title=f"Tiingo end-of-day prices: {ticker}",
|
||||
url=EOD.format(ticker=ticker),
|
||||
doc_type="dataset",
|
||||
abstract="Internal use only (Tiingo licence); raw prices stay in "
|
||||
"data/market/tiingo and market.price_daily.",
|
||||
access_date=today().isoformat(),
|
||||
)
|
||||
return store.upsert(
|
||||
item,
|
||||
tags=[
|
||||
Tag.module("market"),
|
||||
Tag.table("market.price_daily"),
|
||||
Tag.source("tiingo"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# ── readers over market.price_daily ──────────────────────────────────
|
||||
|
||||
|
||||
def read_returns(
|
||||
con: Any, tickers: Sequence[str], start: date, end: date
|
||||
) -> dict[str, dict[date, float]]:
|
||||
"""``{ticker: {day: ret}}`` over ``start..end``; days without a return
|
||||
are dropped."""
|
||||
out: dict[str, dict[date, float]] = {t: {} for t in tickers}
|
||||
if not tickers:
|
||||
return out
|
||||
marks = ",".join("?" * len(tickers))
|
||||
for t, day, ret in con.execute(
|
||||
"SELECT ticker, day, ret FROM market.price_daily "
|
||||
f"WHERE ticker IN ({marks}) AND day BETWEEN ? AND ? AND ret IS NOT NULL "
|
||||
"ORDER BY day",
|
||||
[*tickers, start, end],
|
||||
).fetchall():
|
||||
out[t][day] = ret
|
||||
return out
|
||||
|
||||
|
||||
def trading_days(con: Any, ticker: str = "SPY") -> list[date]:
|
||||
return [
|
||||
d
|
||||
for (d,) in con.execute(
|
||||
"SELECT day FROM market.price_daily WHERE ticker = ? ORDER BY day",
|
||||
[ticker],
|
||||
).fetchall()
|
||||
]
|
||||
|
||||
|
||||
def last_pulls(con: Any) -> dict[str, tuple[date, date]]:
|
||||
"""``{ticker: (last stored day, last pulled_on)}``."""
|
||||
return {
|
||||
t: (d, p)
|
||||
for t, d, p in con.execute(
|
||||
"SELECT ticker, max(day), max(pulled_on) FROM market.price_daily "
|
||||
"GROUP BY ticker"
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
|
||||
# ── the ranked universe (M4) ─────────────────────────────────────────
|
||||
|
||||
|
||||
def _seed_ciks() -> list[str]:
|
||||
from market.seed import load_seed
|
||||
|
||||
return [c.cik for c in load_seed()[0]]
|
||||
|
||||
|
||||
def universe(
|
||||
con: Any,
|
||||
*,
|
||||
used: set[str],
|
||||
cap: int = MONTHLY_SYMBOLS,
|
||||
supported: Mapping[str, Any] | None = None,
|
||||
seed_order: Sequence[str] | None = None,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""``(ticker, entity_key)`` in rank order: the benchmarks, the seed
|
||||
securities in seed order, then the securities of entities with a
|
||||
filing-text (``fts``) link, ranked by strongest link, number of distinct
|
||||
link targets and newest 10-K. SIC-only entities and unsupported or
|
||||
unpriced tickers are excluded; at most *cap* symbols not already in
|
||||
*used* are returned."""
|
||||
by_entity: dict[str, list[str]] = defaultdict(list)
|
||||
for s in read_securities(con):
|
||||
if s.priced:
|
||||
by_entity[s.entity_key].append(s.ticker)
|
||||
|
||||
ranked: list[tuple[str, str]] = [(t, f"index:{t}") for t in BENCHMARKS]
|
||||
for key in seed_order if seed_order is not None else _seed_ciks():
|
||||
ranked += [(t, key) for t in by_entity.get(key, [])]
|
||||
|
||||
best: dict[str, int] = {}
|
||||
targets: dict[str, set[str]] = defaultdict(set)
|
||||
for link in read_links(con, basis="fts"):
|
||||
rank = _STRENGTH.get(link.strength, len(_STRENGTH))
|
||||
best[link.entity_key] = min(best.get(link.entity_key, rank), rank)
|
||||
targets[link.entity_key].add(link.target)
|
||||
newest = {
|
||||
cik: filed
|
||||
for cik, filed in con.execute(
|
||||
"SELECT cik, max(filed) FROM market.filing "
|
||||
"WHERE form IN ('10-K', '10-K/A') GROUP BY cik"
|
||||
).fetchall()
|
||||
}
|
||||
for key in sorted(
|
||||
best,
|
||||
key=lambda k: (
|
||||
best[k],
|
||||
-len(targets[k]),
|
||||
-(newest[k].toordinal() if newest.get(k) else 0),
|
||||
k,
|
||||
),
|
||||
):
|
||||
ranked += [(t, key) for t in by_entity.get(key, [])]
|
||||
|
||||
out: list[tuple[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
fresh = 0
|
||||
for ticker, key in ranked:
|
||||
if ticker in seen or (supported is not None and ticker not in supported):
|
||||
continue
|
||||
if ticker not in used:
|
||||
if fresh >= cap:
|
||||
continue
|
||||
fresh += 1
|
||||
seen.add(ticker)
|
||||
out.append((ticker, key))
|
||||
return out
|
||||
|
||||
|
||||
# ── what to pull ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Target:
|
||||
ticker: str
|
||||
entity_key: str
|
||||
#: ``(valid_from, valid_to)`` of the securities carrying this ticker;
|
||||
#: empty when none is known
|
||||
windows: tuple[tuple[date | None, date | None], ...]
|
||||
#: where a first pull starts
|
||||
start: date
|
||||
securities: tuple[SecurityRow, ...] = ()
|
||||
|
||||
|
||||
def plan(
|
||||
con: Any,
|
||||
*,
|
||||
tickers: Sequence[str],
|
||||
universe_on: bool,
|
||||
benchmarks_on: bool,
|
||||
used: set[str],
|
||||
supported: Mapping[str, Any] | None = None,
|
||||
cap: int = MONTHLY_SYMBOLS,
|
||||
seed_order: Sequence[str] | None = None,
|
||||
) -> list[Target]:
|
||||
"""Benchmarks, then the named tickers, then the ranked universe."""
|
||||
names: list[tuple[str, str]] = []
|
||||
if benchmarks_on:
|
||||
names += [(t, f"index:{t}") for t in BENCHMARKS]
|
||||
secs: dict[str, list[SecurityRow]] = defaultdict(list)
|
||||
for s in read_securities(con):
|
||||
secs[s.ticker].append(s)
|
||||
for t in tickers:
|
||||
t = t.upper()
|
||||
names.append((t, secs[t][0].entity_key if secs.get(t) else ""))
|
||||
if universe_on:
|
||||
names += universe(
|
||||
con, used=used, cap=cap, supported=supported, seed_order=seed_order
|
||||
)
|
||||
out: list[Target] = []
|
||||
seen: set[str] = set()
|
||||
for ticker, key in names:
|
||||
if ticker in seen:
|
||||
continue
|
||||
seen.add(ticker)
|
||||
rows = secs.get(ticker, []) if ticker not in BENCHMARKS else []
|
||||
windows = tuple((s.valid_from, s.valid_to) for s in rows)
|
||||
if ticker in BENCHMARKS:
|
||||
start = BENCHMARKS[ticker][1]
|
||||
elif windows and all(w[0] for w in windows):
|
||||
start = max(FLOOR, min(w[0] for w in windows))
|
||||
else:
|
||||
start = FLOOR
|
||||
out.append(Target(ticker, key, windows, start, tuple(rows)))
|
||||
return out
|
||||
|
||||
|
||||
def clip_windows(
|
||||
rows: Sequence[PriceRow], windows: Sequence[tuple[date | None, date | None]]
|
||||
) -> list[PriceRow]:
|
||||
"""Rows inside any of *windows* (all rows when there is none)."""
|
||||
if not windows:
|
||||
return list(rows)
|
||||
return [r for r in rows if any(clip([r], a, b) for a, b in windows)]
|
||||
@@ -587,3 +587,335 @@ def test_entities_stores_ticker_history_and_link_dates(env):
|
||||
if l.entity_key == "0001801170"
|
||||
]
|
||||
assert clov.valid_to == date(2023, 12, 31)
|
||||
|
||||
|
||||
# ── stack market prices ──────────────────────────────────────────────
|
||||
import io # noqa: E402
|
||||
import zipfile # noqa: E402
|
||||
|
||||
from market import tiingo # noqa: E402
|
||||
|
||||
DAY = date(2026, 11, 12)
|
||||
|
||||
|
||||
def _supported_zip(*rows: str) -> bytes:
|
||||
head = "ticker,exchange,assetType,priceCurrency,startDate,endDate\n"
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as z:
|
||||
z.writestr("supported_tickers.csv", head + "\n".join(rows) + "\n")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _eod(day: str, close: float, adj: float | None = None) -> dict:
|
||||
return {
|
||||
"date": f"{day}T00:00:00.000Z",
|
||||
"open": close,
|
||||
"high": close,
|
||||
"low": close,
|
||||
"close": close,
|
||||
"volume": 5,
|
||||
"adjClose": close if adj is None else adj,
|
||||
"divCash": 0.0,
|
||||
"splitFactor": 1.0,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def px(env, monkeypatch, tmp_path):
|
||||
c, store, published, hoard = env
|
||||
monkeypatch.setenv("TIINGO_API_KEY", "secret-key")
|
||||
monkeypatch.setattr(tiingo, "state_dir", lambda: tmp_path / "state")
|
||||
monkeypatch.setattr(tiingo, "new_pacer", lambda: _Pacer())
|
||||
monkeypatch.setattr(tiingo, "today", lambda: DAY)
|
||||
seen: list[httpx.Request] = []
|
||||
series = {
|
||||
"AAA": [
|
||||
_eod("2024-01-02", 100),
|
||||
_eod("2024-01-03", 110),
|
||||
_eod("2024-01-04", 99),
|
||||
],
|
||||
"BBB": [_eod("2024-01-02", 10), _eod("2024-01-03", 11)],
|
||||
"SPY": [_eod("2024-01-02", 400), _eod("2024-01-03", 404)],
|
||||
"XLV": [_eod("2024-01-02", 100)],
|
||||
"IHF": [_eod("2024-01-02", 50)],
|
||||
}
|
||||
z = _supported_zip(
|
||||
"AAA,NYSE,Stock,USD,2000-01-01,2026-11-12",
|
||||
"BBB,NYSE,Stock,USD,2000-01-01,2026-11-12",
|
||||
"SPY,NYSE,ETF,USD,1993-01-29,2026-11-12",
|
||||
"XLV,NYSE,ETF,USD,1998-12-22,2026-11-12",
|
||||
"IHF,NYSE,ETF,USD,2006-05-05,2026-11-12",
|
||||
)
|
||||
|
||||
def prices_for(m):
|
||||
if m.group(1) not in series:
|
||||
return httpx.Response(404, json={"detail": "Not found."})
|
||||
return httpx.Response(200, json=series[m.group(1)])
|
||||
|
||||
routes = [
|
||||
(r"supported_tickers\.zip", lambda m: httpx.Response(200, content=z)),
|
||||
(r"/daily/(\w+)/prices", prices_for),
|
||||
]
|
||||
|
||||
def handler(request):
|
||||
seen.append(request)
|
||||
return _handler(routes)(request)
|
||||
|
||||
monkeypatch.setattr(
|
||||
http,
|
||||
"client",
|
||||
lambda source, key=None: httpx.Client(
|
||||
transport=httpx.MockTransport(handler),
|
||||
headers={"Authorization": f"Token {key}"} if key else {},
|
||||
),
|
||||
)
|
||||
return c, store, published, hoard, seen, series, tmp_path / "state"
|
||||
|
||||
|
||||
def _price_requests(seen):
|
||||
return [r for r in seen if "/prices" in r.url.path]
|
||||
|
||||
|
||||
def test_prices_cli_fetch_write(px):
|
||||
c, _, published, hoard, seen, _, state = px
|
||||
res = runner.invoke(
|
||||
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert _count(c, "market.price_daily") == 5
|
||||
assert [r.url.params["startDate"] for r in _price_requests(seen)] == [
|
||||
"1993-01-29"
|
||||
] * 2
|
||||
assert all("secret-key" not in str(r.url) for r in seen)
|
||||
keyed = [r for r in seen if "/prices" in r.url.path]
|
||||
assert all(r.headers["authorization"] == "Token secret-key" for r in keyed)
|
||||
free = [r for r in seen if "supported" in r.url.path]
|
||||
assert free and "authorization" not in free[0].headers
|
||||
assert {
|
||||
(t, n)
|
||||
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||||
} == {("market.price_daily", 5), ("market.series", 2)}
|
||||
assert [s.value for s in T.read_series(c, ["AAA"])] == [pytest.approx(99.0)]
|
||||
assert published == [True]
|
||||
used = json.loads((state / "tiingo-2026-11.json").read_text())
|
||||
assert used["symbols"] == ["AAA", "BBB"] and used["day_counts"] == {"2026-11-12": 2}
|
||||
assert (hoard / "tiingo" / "eod" / "AAA" / "1993-01-29_2026-11-12.json").is_file()
|
||||
assert "priced 2 symbols, 5 rows" in res.output and "2/450" in res.output
|
||||
assert T.read_gaps(c) == []
|
||||
|
||||
|
||||
def test_prices_incremental_pull_and_same_day_skip(px, monkeypatch):
|
||||
c, _, _, _, seen, series, _ = px
|
||||
cmd = ["market", "prices", "--ticker", "AAA", "--write"]
|
||||
assert runner.invoke(app, cmd).exit_code == 0
|
||||
before = len(_price_requests(seen))
|
||||
again = runner.invoke(app, cmd)
|
||||
assert len(_price_requests(seen)) == before # priced today: nothing to do
|
||||
assert "priced 0 symbols" in again.output
|
||||
monkeypatch.setattr(tiingo, "today", lambda: date(2026, 11, 13))
|
||||
series["AAA"] = [_eod("2024-01-04", 99, 49.5), _eod("2024-01-05", 108.9, 54.45)]
|
||||
assert runner.invoke(app, cmd).exit_code == 0
|
||||
last = _price_requests(seen)[-1]
|
||||
assert last.url.params["startDate"] == "2024-01-04"
|
||||
rows = {r.day: r for r in T.read_prices(c, "AAA")}
|
||||
assert len(rows) == 4 and rows[date(2024, 1, 4)].adj_close == 99
|
||||
assert rows[date(2024, 1, 5)].ret == pytest.approx(0.1)
|
||||
assert [m.value for m in T.read_series(c, ["AAA"])] == [pytest.approx(108.9)]
|
||||
|
||||
|
||||
def test_prices_benchmarks_start_at_listing(px):
|
||||
c, _, _, _, seen, _, _ = px
|
||||
res = runner.invoke(app, ["market", "prices", "--benchmarks", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
starts = {
|
||||
r.url.path.split("/")[3]: r.url.params["startDate"]
|
||||
for r in _price_requests(seen)
|
||||
}
|
||||
assert starts == {"SPY": "1993-01-29", "XLV": "1998-12-22", "IHF": "2006-05-05"}
|
||||
|
||||
|
||||
def test_prices_dry_run_reports_budget(px, monkeypatch):
|
||||
c, _, published, _, seen, _, state = px
|
||||
T.ensure_tables(c)
|
||||
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
|
||||
T.write_links(c, [T.LinkRow("K1", "program", "PFS", "fts", "mention", "adsh", "a")])
|
||||
state.mkdir(parents=True)
|
||||
(state / "tiingo-2026-11.json").write_text(
|
||||
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
||||
)
|
||||
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "1/450 symbols used, 449 remaining" in res.output
|
||||
assert "1. SPY" in res.output and "used" in res.output
|
||||
assert "AAA" in res.output and "K1" in res.output and "first pull" in res.output
|
||||
assert _price_requests(seen) == [] and published == []
|
||||
assert not (state / "tiingo-2026-11.json").read_text().count("AAA")
|
||||
|
||||
|
||||
def test_prices_dry_run_without_market_tables_lists_benchmarks(px):
|
||||
res = runner.invoke(app, ["market", "prices", "--universe", "--dry-run"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "3 symbols ranked" in res.output
|
||||
|
||||
|
||||
def test_prices_dry_run_shows_stored_span(px):
|
||||
c, *_ = px
|
||||
T.ensure_tables(c)
|
||||
T.write_prices(
|
||||
c,
|
||||
"AAA",
|
||||
tiingo.to_prices(
|
||||
"AAA", [_eod("2024-01-02", 1)], pulled_on=DAY, source_file="f", overlap=None
|
||||
),
|
||||
)
|
||||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--dry-run"])
|
||||
assert "stored to 2024-01-02" in res.output
|
||||
|
||||
|
||||
def test_prices_without_key_records_gap(px, monkeypatch):
|
||||
c, _, published, _, seen, _, _ = px
|
||||
monkeypatch.delenv("TIINGO_API_KEY")
|
||||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: tiingo:prices: TIINGO_API_KEY not in .env" in res.output
|
||||
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"]
|
||||
assert _count(c, "market.price_daily") == 0 and seen == [] and published == [True]
|
||||
|
||||
|
||||
def test_prices_refusal_prints_gap(px, monkeypatch):
|
||||
c, _, published, _, _, _, _ = px
|
||||
monkeypatch.setattr(
|
||||
http,
|
||||
"client",
|
||||
lambda source, key=None: httpx.Client(
|
||||
transport=httpx.MockTransport(lambda r: httpx.Response(403))
|
||||
),
|
||||
)
|
||||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: tiingo:prices:" in res.output and "403" in res.output
|
||||
assert _count(c, "market.price_daily") == 0
|
||||
assert [g.source for g in T.read_gaps(c)] == ["tiingo:prices"] and published == [
|
||||
True
|
||||
]
|
||||
|
||||
|
||||
def test_prices_refusal_on_prices_after_supported_ok(px, monkeypatch):
|
||||
c, _, published, _, _, _, _ = px
|
||||
z = _supported_zip("AAA,NYSE,Stock,USD,2000-01-01,2026-11-12")
|
||||
routes = [
|
||||
(r"supported_tickers", lambda m: httpx.Response(200, content=z)),
|
||||
(r"/prices", lambda m: httpx.Response(401)),
|
||||
]
|
||||
_online(monkeypatch, routes)
|
||||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA"])
|
||||
assert (
|
||||
res.exit_code == 0
|
||||
and "gap: tiingo:prices:" in res.output
|
||||
and "401" in res.output
|
||||
)
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_prices_nothing_selected_exits(px):
|
||||
res = runner.invoke(app, ["market", "prices"])
|
||||
assert res.exit_code == 2 and "nothing to price" in res.output
|
||||
|
||||
|
||||
def test_prices_unknown_ticker_and_window_gap_become_gap_note(px):
|
||||
c, _, _, _, _, series, _ = px
|
||||
T.ensure_tables(c)
|
||||
T.write_securities(
|
||||
c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, date(2030, 1, 1), "s")]
|
||||
)
|
||||
res = runner.invoke(
|
||||
app, ["market", "prices", "--ticker", "BBB", "--ticker", "ZZZ", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "ZZZ: not found at Tiingo" in res.output
|
||||
(gap,) = T.read_gaps(c)
|
||||
assert (
|
||||
"ZZZ: not found at Tiingo" in gap.note
|
||||
and "BBB: Tiingo ends 2026-11-12" in gap.note
|
||||
)
|
||||
# BBB later prices cleanly: its entry goes, ZZZ's stays
|
||||
T.write_securities(c, "K1", [T.SecurityRow("K1", "BBB", "NYSE", None, None, "s")])
|
||||
runner.invoke(app, ["market", "prices", "--ticker", "BBB", "--refresh", "--write"])
|
||||
(gap,) = T.read_gaps(c)
|
||||
assert gap.note == "ZZZ: not found at Tiingo"
|
||||
|
||||
|
||||
def test_prices_clean_run_clears_gap_and_window_clips(px):
|
||||
c, _, _, _, _, _, _ = px
|
||||
T.ensure_tables(c)
|
||||
T.write_gap(c, "tiingo:prices", "AAA: not found at Tiingo")
|
||||
T.write_securities(
|
||||
c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", date(2024, 1, 3), None, "s")]
|
||||
)
|
||||
res = runner.invoke(app, ["market", "prices", "--ticker", "AAA", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert [r.day for r in T.read_prices(c, "AAA")] == [
|
||||
date(2024, 1, 3),
|
||||
date(2024, 1, 4),
|
||||
]
|
||||
assert T.read_gaps(c) == []
|
||||
|
||||
|
||||
def test_prices_limit_stops_downloads(px):
|
||||
c, _, _, _, seen, _, _ = px
|
||||
res = runner.invoke(
|
||||
app, ["market", "prices", "--benchmarks", "--limit", "1", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0 and len(_price_requests(seen)) == 1
|
||||
assert [r.ticker for r in [*T.read_prices(c, "SPY")][:1]] == ["SPY"]
|
||||
|
||||
|
||||
def test_prices_budget_exhausted_stops_and_writes_what_it_has(px):
|
||||
c, _, published, _, seen, _, state = px
|
||||
state.mkdir(parents=True)
|
||||
(state / "tiingo-2026-11.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"month": "2026-11",
|
||||
"symbols": [f"S{i}" for i in range(449)] + ["AAA"],
|
||||
"day_counts": {},
|
||||
}
|
||||
)
|
||||
)
|
||||
res = runner.invoke(
|
||||
app, ["market", "prices", "--ticker", "AAA", "--ticker", "BBB", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "budget: stopped, BBB" in res.output
|
||||
assert [r.ticker for r in T.read_prices(c, "AAA")][:1] == ["AAA"]
|
||||
assert T.read_prices(c, "BBB") == [] and published == [True]
|
||||
|
||||
|
||||
def test_prices_without_write_stores_nothing(px, monkeypatch):
|
||||
c, _, published, _, _, _, _ = px
|
||||
monkeypatch.setattr(
|
||||
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
||||
)
|
||||
res = runner.invoke(app, ["market", "prices", "--ticker", "ZZZ"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert (
|
||||
"dry run: pass --write" in res.output
|
||||
and "gap: tiingo:prices: ZZZ" in res.output
|
||||
)
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_merge_gap_keeps_other_tickers_and_orders():
|
||||
assert mk._merge_gap("", {}, set()) == ""
|
||||
assert mk._merge_gap("A: x | B: y", {"C": "z"}, {"A"}) == "B: y | C: z"
|
||||
|
||||
|
||||
def test_or_empty_reraises_other_errors():
|
||||
def boom(_):
|
||||
raise RuntimeError("disk")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
mk._or_empty(boom, None, {})
|
||||
|
||||
7
tests/market/fixtures/supported_tickers_head.csv
Normal file
7
tests/market/fixtures/supported_tickers_head.csv
Normal file
@@ -0,0 +1,7 @@
|
||||
ticker,exchange,assetType,priceCurrency,startDate,endDate
|
||||
000001,SHE,Stock,CNY,2007-01-04,2026-11-12
|
||||
ELV,NYSE,Stock,USD,2001-10-30,2026-11-12
|
||||
IHF,NYSE,ETF,USD,2006-05-05,2026-11-12
|
||||
OSH,NYSE,Stock,USD,2020-08-06,2023-05-02
|
||||
SPY,NYSE,ETF,USD,1993-01-29,2026-11-12
|
||||
XLV,NYSE,ETF,USD,1998-12-22,2026-11-12
|
||||
|
@@ -6,7 +6,13 @@ from __future__ import annotations
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
MODULES = ["market.http", "market.tables", "market.seed", "market.edgar"]
|
||||
MODULES = [
|
||||
"market.http",
|
||||
"market.tables",
|
||||
"market.seed",
|
||||
"market.edgar",
|
||||
"market.tiingo",
|
||||
]
|
||||
|
||||
_PROBE = """
|
||||
import builtins, importlib, sys
|
||||
|
||||
@@ -9,6 +9,7 @@ import duckdb
|
||||
import pytest
|
||||
|
||||
from market import tables as T
|
||||
from market import tiingo as TI
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -30,7 +31,15 @@ def test_ensure_tables_idempotent():
|
||||
"WHERE table_schema = 'market'"
|
||||
).fetchall()
|
||||
}
|
||||
assert names == {"entity", "security", "entity_link", "filing", "source_gap"}
|
||||
assert names == {
|
||||
"entity",
|
||||
"security",
|
||||
"entity_link",
|
||||
"filing",
|
||||
"source_gap",
|
||||
"price_daily",
|
||||
"series",
|
||||
}
|
||||
|
||||
|
||||
ENTITY = T.EntityRow(
|
||||
@@ -67,6 +76,24 @@ FILING = T.FilingRow(
|
||||
)
|
||||
GAP = T.GapRow("sec-edgar:entities", "403", dt.date.today())
|
||||
|
||||
PRICE = TI.PriceRow(
|
||||
"OSH",
|
||||
date(2021, 1, 4),
|
||||
10.0,
|
||||
11.0,
|
||||
9.5,
|
||||
10.5,
|
||||
1000.0,
|
||||
10.4,
|
||||
0.0,
|
||||
1.0,
|
||||
0.01,
|
||||
0.012,
|
||||
date(2026, 11, 12),
|
||||
"eod/OSH/x.json",
|
||||
)
|
||||
SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY")
|
||||
|
||||
ROUNDTRIP = {
|
||||
"entity": (
|
||||
ENTITY,
|
||||
@@ -84,6 +111,16 @@ ROUNDTRIP = {
|
||||
lambda c, r: T.write_filings(c, r.cik, [r]),
|
||||
lambda c: T.read_filings(c, "0001564406"),
|
||||
),
|
||||
"price_daily": (
|
||||
PRICE,
|
||||
lambda c, r: TI.write_prices(c, r.ticker, [r]),
|
||||
lambda c: TI.read_prices(c, "OSH"),
|
||||
),
|
||||
"series": (
|
||||
SERIES,
|
||||
lambda c, r: TI.write_series(c, r.series_id, [r]),
|
||||
lambda c: TI.read_series(c, ["OSH"]),
|
||||
),
|
||||
"gap": (
|
||||
GAP,
|
||||
lambda c, r: T.write_gap(c, r.source, r.note),
|
||||
|
||||
427
tests/market/test_tiingo.py
Normal file
427
tests/market/test_tiingo.py
Normal file
@@ -0,0 +1,427 @@
|
||||
"""market.tiingo: returns, windows, budget, ranked universe, monthly series.
|
||||
Prices here are synthetic (Tiingo's licence forbids sharing real ones)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from market import http
|
||||
from market import tables as T
|
||||
from market import tiingo as TI
|
||||
|
||||
FX = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def rec(day, close, adj=None, div=0.0, split=1.0):
|
||||
return {
|
||||
"date": f"{day}T00:00:00.000Z",
|
||||
"open": close,
|
||||
"high": close,
|
||||
"low": close,
|
||||
"close": close,
|
||||
"volume": 10,
|
||||
"adjClose": close if adj is None else adj,
|
||||
"divCash": div,
|
||||
"splitFactor": split,
|
||||
}
|
||||
|
||||
|
||||
def pull(ticker, records, overlap=None):
|
||||
return TI.to_prices(
|
||||
ticker,
|
||||
records,
|
||||
pulled_on=date(2026, 11, 12),
|
||||
source_file="f.json",
|
||||
overlap=overlap,
|
||||
)
|
||||
|
||||
|
||||
def test_to_prices_returns_within_pull():
|
||||
rows = pull(
|
||||
"X",
|
||||
[
|
||||
rec("2024-01-02", 1, 100),
|
||||
rec("2024-01-03", 1, 110),
|
||||
rec("2024-01-04", 1, 99),
|
||||
],
|
||||
)
|
||||
assert [r.ret for r in rows] == [None, pytest.approx(0.1), pytest.approx(-0.1)]
|
||||
assert rows[0].day == date(2024, 1, 2) and rows[0].source_file == "f.json"
|
||||
|
||||
|
||||
def test_ret_raw_with_split_and_dividend():
|
||||
rows = pull("X", [rec("2024-01-02", 100), rec("2024-01-03", 50, split=2.0)])
|
||||
assert rows[1].ret_raw == pytest.approx(0.0)
|
||||
rows = pull("X", [rec("2024-01-02", 100), rec("2024-01-03", 99, div=1.0)])
|
||||
assert rows[1].ret_raw == pytest.approx(0.0)
|
||||
assert rows[1].div_cash == 1.0 and rows[1].split_factor == 1.0
|
||||
|
||||
|
||||
def test_incremental_pull_rebased_adjclose_keeps_returns():
|
||||
d1, d2, d3 = date(2024, 1, 2), date(2024, 1, 3), date(2024, 1, 4)
|
||||
first = pull("X", [rec(str(d1), 100, 100), rec(str(d2), 110, 110)])
|
||||
second = pull("X", [rec(str(d2), 110, 55), rec(str(d3), 121, 60.5)], overlap=d2)
|
||||
assert [r.day for r in second] == [d3]
|
||||
assert second[0].ret == pytest.approx(0.1)
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
TI.write_prices(con, "X", first)
|
||||
TI.write_prices(con, "X", second)
|
||||
stored = {r.day: r for r in TI.read_prices(con, "X")}
|
||||
assert stored[d2].adj_close == 110 and stored[d2].ret == pytest.approx(0.1)
|
||||
assert stored[d3].ret == pytest.approx(0.1)
|
||||
|
||||
|
||||
def test_prices_clipped_to_security_window():
|
||||
rows = pull("AGL", [rec("2015-06-01", 1), rec("2021-05-03", 1)])
|
||||
kept = TI.clip(rows, date(2021, 4, 15), None)
|
||||
assert [r.day for r in kept] == [date(2021, 5, 3)]
|
||||
assert [r.day for r in TI.clip(rows, None, date(2016, 1, 1))] == [date(2015, 6, 1)]
|
||||
assert len(TI.clip(rows, None, None)) == 2
|
||||
|
||||
|
||||
def test_window_gap():
|
||||
sec = T.SecurityRow(
|
||||
"1", "AGL", "NYSE", date(2021, 4, 15), date(2023, 6, 30), "seed"
|
||||
)
|
||||
ok = {"startDate": "2021-04-15", "endDate": "2023-06-30"}
|
||||
assert TI.window_gap(ok, sec) is None
|
||||
note = TI.window_gap({"startDate": "2021-04-15", "endDate": "2023-05-02"}, sec)
|
||||
assert "2023-05-02" in note and "2023-06-30" in note
|
||||
late = TI.window_gap({"startDate": "2021-06-01", "endDate": None}, sec)
|
||||
assert "2021-06-01" in late and "2021-04-15" in late
|
||||
assert TI.window_gap({}, sec) is None
|
||||
assert TI.window_gap({"startDate": "2021-04-20"}, sec) is None # within tolerance
|
||||
|
||||
|
||||
def test_budget_monthly_symbols_and_daily_cap(tmp_path):
|
||||
today = date(2026, 11, 12)
|
||||
b = TI.Budget("2026-11", {f"S{i}" for i in range(450)}, {})
|
||||
assert not b.allow("NEW", today)
|
||||
assert b.allow("S7", today)
|
||||
b2 = TI.Budget("2026-11", set(), {today.isoformat(): 950})
|
||||
assert not b2.allow("S7", today)
|
||||
b3 = TI.Budget("2026-11", set(), {})
|
||||
b3.spend("A", today)
|
||||
b3.spend("A", today)
|
||||
assert b3.symbols == {"A"} and b3.day_counts == {"2026-11-12": 2}
|
||||
|
||||
|
||||
def test_budget_persists_and_rolls_over(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(TI, "state_dir", lambda: tmp_path)
|
||||
today = date(2026, 11, 12)
|
||||
b = TI.Budget.load(today)
|
||||
b.spend("A", today)
|
||||
again = TI.Budget.load(today)
|
||||
assert again.symbols == {"A"} and again.remaining_symbols == 449
|
||||
nxt = TI.Budget.load(date(2026, 12, 1))
|
||||
assert nxt.symbols == set() and nxt.month == "2026-12"
|
||||
assert (tmp_path / "tiingo-2026-11.json").is_file()
|
||||
|
||||
|
||||
def test_monthly_tri_month_end():
|
||||
rows = pull(
|
||||
"X",
|
||||
[
|
||||
rec("2024-01-02", 1, 100),
|
||||
rec("2024-01-31", 1, 110),
|
||||
rec("2024-02-29", 1, 99),
|
||||
],
|
||||
)
|
||||
out = TI.tri_monthly("X", rows, item_key="K")
|
||||
assert [(m.year, m.month, m.value) for m in out] == [
|
||||
(2024, 1, pytest.approx(110.0)),
|
||||
(2024, 2, pytest.approx(99.0)),
|
||||
]
|
||||
assert out[0].series_id == "tiingo:X:tri" and out[0].item_key == "K"
|
||||
assert out[0].symbol == "X" and out[0].source == "tiingo"
|
||||
assert TI.tri_monthly("X", [], item_key="K") == []
|
||||
|
||||
|
||||
def _universe_db():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
|
||||
def sec(key, ticker, priced=True):
|
||||
T.write_securities(
|
||||
con, key, [T.SecurityRow(key, ticker, "NYSE", None, None, "s", priced)]
|
||||
)
|
||||
|
||||
def link(key, basis, target, strength):
|
||||
T.write_links(
|
||||
con, [T.LinkRow(key, "program", target, basis, strength, "adsh", "a")]
|
||||
)
|
||||
|
||||
def tenk(key, filed):
|
||||
T.write_filings(
|
||||
con, key, [T.FilingRow(key, f"A-{key}", "10-K", filed, "", None, "", "")]
|
||||
)
|
||||
|
||||
sec("SEED", "SEEDCO")
|
||||
sec("OLD", "OLDCO", priced=False)
|
||||
sec("REV", "REVCO")
|
||||
sec("MEN", "MENCO")
|
||||
sec("MEN2", "MEN2CO")
|
||||
sec("SIC", "SICCO")
|
||||
sec("GONE", "GONECO")
|
||||
link("SEED", "curated", "PFS", "curated")
|
||||
link("REV", "fts", "PFS", "revenue_exposure")
|
||||
link("MEN", "fts", "PFS", "mention")
|
||||
link("MEN", "fts", "APM", "mention")
|
||||
link("MEN2", "fts", "PFS", "mention")
|
||||
link("SIC", "sic", "8082", "sector")
|
||||
link("GONE", "fts", "PFS", "revenue_exposure")
|
||||
tenk("MEN", date(2020, 1, 1))
|
||||
tenk("MEN2", date(2025, 1, 1))
|
||||
return con
|
||||
|
||||
|
||||
SUPPORTED = {
|
||||
t: (date(2000, 1, 1), None)
|
||||
for t in (
|
||||
"SPY",
|
||||
"XLV",
|
||||
"IHF",
|
||||
"SEEDCO",
|
||||
"REVCO",
|
||||
"MENCO",
|
||||
"MEN2CO",
|
||||
"SICCO",
|
||||
"OLDCO",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def test_universe_ranked_capped_excludes_sic_only():
|
||||
con = _universe_db()
|
||||
kw = {"supported": SUPPORTED, "seed_order": ["SEED"]}
|
||||
got = [t for t, _ in TI.universe(con, used=set(), **kw)]
|
||||
assert got[:3] == ["SPY", "XLV", "IHF"]
|
||||
assert got[3:] == ["SEEDCO", "REVCO", "MENCO", "MEN2CO"]
|
||||
assert "SICCO" not in got and "OLDCO" not in got and "GONECO" not in got
|
||||
keys = dict(TI.universe(con, used=set(), **kw))
|
||||
assert keys["REVCO"] == "REV" and keys["SPY"] == "index:SPY"
|
||||
assert [t for t, _ in TI.universe(con, used=set(), cap=5, **kw)] == got[:5]
|
||||
capped = [t for t, _ in TI.universe(con, used={"SPY"}, cap=5, **kw)]
|
||||
assert capped[:2] == ["SPY", "XLV"] and len(capped) == 6
|
||||
assert "SEEDCO" in capped and "REVCO" in capped and "MEN2CO" not in capped
|
||||
|
||||
|
||||
def test_universe_default_seed_order_reads_the_seed():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
assert [t for t, _ in TI.universe(con, used=set())] == ["SPY", "XLV", "IHF"]
|
||||
|
||||
|
||||
def _zip(csv_bytes: bytes) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as z:
|
||||
z.writestr("supported_tickers.csv", csv_bytes)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def test_parse_supported():
|
||||
got = TI.parse_supported(_zip((FX / "supported_tickers_head.csv").read_bytes()))
|
||||
assert got["OSH"] == (date(2020, 8, 6), date(2023, 5, 2))
|
||||
assert "000001" not in got
|
||||
assert got["SPY"][0] == date(1993, 1, 29)
|
||||
extra = b"ticker,exchange,assetType,priceCurrency,startDate,endDate\nZZ,NYSE,Stock,USD,,\nZZ,NYSE,ETF,USD,2001-01-01,2002-01-01\n"
|
||||
assert TI.parse_supported(_zip(extra))["ZZ"] == (date(2001, 1, 1), date(2002, 1, 1))
|
||||
|
||||
|
||||
def _client(routes, seen):
|
||||
def handler(request: httpx.Request):
|
||||
seen.append(request)
|
||||
for frag, make in routes:
|
||||
if frag in str(request.url):
|
||||
return make(request)
|
||||
return httpx.Response(404)
|
||||
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
class _Pacer:
|
||||
def wait(self):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hoard(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
|
||||
monkeypatch.setattr(TI, "state_dir", lambda: tmp_path / "state")
|
||||
return tmp_path / "market"
|
||||
|
||||
|
||||
def test_benchmark_first_pull_starts_at_listing(hoard):
|
||||
seen = []
|
||||
cl = _client(
|
||||
[("/prices", lambda r: httpx.Response(200, json=[rec("2024-01-02", 1)]))], seen
|
||||
)
|
||||
budget = TI.Budget("2026-11", set(), {})
|
||||
for t in ("XLV", "SPY"):
|
||||
start = TI.BENCHMARKS[t][1]
|
||||
TI.fetch_eod(
|
||||
cl,
|
||||
t,
|
||||
start,
|
||||
date(2026, 11, 12),
|
||||
pacer=_Pacer(),
|
||||
budget=budget,
|
||||
today=date(2026, 11, 12),
|
||||
)
|
||||
starts = {r.url.path.split("/")[3]: r.url.params["startDate"] for r in seen}
|
||||
assert starts == {"XLV": "1998-12-22", "SPY": "1993-01-29"}
|
||||
assert all(r.url.params["resampleFreq"] == "daily" for r in seen)
|
||||
assert all("token" not in str(r.url).lower() for r in seen)
|
||||
assert TI.BENCHMARKS["IHF"] == ("providers", date(2006, 5, 5))
|
||||
|
||||
|
||||
def test_fetch_eod_caches_and_never_overwrites(hoard):
|
||||
seen = []
|
||||
body = [rec("2024-01-02", 1)]
|
||||
cl = _client([("/prices", lambda r: httpx.Response(200, json=body))], seen)
|
||||
budget = TI.Budget("2026-11", set(), {})
|
||||
args = (cl, "X", date(2024, 1, 1), date(2024, 1, 5))
|
||||
kw = {"pacer": _Pacer(), "budget": budget, "today": date(2026, 11, 12)}
|
||||
recs, path = TI.fetch_eod(*args, **kw)
|
||||
assert recs == body and path.parent.name == "X"
|
||||
again, _ = TI.fetch_eod(*args, **kw)
|
||||
assert again == body and len(seen) == 1 # cache hit costs nothing
|
||||
body[:] = [rec("2024-01-02", 2)]
|
||||
changed, p2 = TI.fetch_eod(*args, refresh=True, **kw)
|
||||
assert changed == body and p2 != path and path.is_file()
|
||||
assert budget.symbols == {"X"} and budget.day_counts["2026-11-12"] == 2
|
||||
|
||||
|
||||
def test_fetch_eod_unknown_ticker_and_budget(hoard):
|
||||
cl = _client([], [])
|
||||
budget = TI.Budget("2026-11", set(), {})
|
||||
kw = {"pacer": _Pacer(), "today": date(2026, 11, 12)}
|
||||
assert TI.fetch_eod(
|
||||
cl, "NOPE", date(2024, 1, 1), date(2024, 1, 5), budget=budget, **kw
|
||||
) == (None, None)
|
||||
full = TI.Budget("2026-11", {f"S{i}" for i in range(450)}, {})
|
||||
with pytest.raises(TI.BudgetExhausted):
|
||||
TI.fetch_eod(cl, "NEW", date(2024, 1, 1), date(2024, 1, 5), budget=full, **kw)
|
||||
|
||||
|
||||
def test_fetch_supported_hoards_and_reads_cache(hoard):
|
||||
seen = []
|
||||
z = _zip((FX / "supported_tickers_head.csv").read_bytes())
|
||||
cl = _client(
|
||||
[("supported_tickers.zip", lambda r: httpx.Response(200, content=z))], seen
|
||||
)
|
||||
got = TI.fetch_supported(cl, pacer=_Pacer())
|
||||
assert got["OSH"][1] == date(2023, 5, 2)
|
||||
assert (hoard / "tiingo" / "supported_tickers.zip").is_file()
|
||||
assert TI.fetch_supported(None, pacer=_Pacer(), offline=True)["SPY"]
|
||||
assert TI.fetch_supported(cl, pacer=_Pacer())["SPY"] and len(seen) == 1
|
||||
assert len(TI.fetch_supported(cl, pacer=_Pacer(), refresh=True)) == len(got)
|
||||
|
||||
|
||||
def test_fetch_supported_offline_without_cache(hoard):
|
||||
assert TI.fetch_supported(None, pacer=_Pacer(), offline=True) == {}
|
||||
|
||||
|
||||
class _Store:
|
||||
def __init__(self):
|
||||
self.items = []
|
||||
self.attached = []
|
||||
|
||||
def upsert(self, item, tags=()):
|
||||
self.items.append(item)
|
||||
return "KEY1"
|
||||
|
||||
def attach_file(self, *a, **k): # pragma: no cover - must never run
|
||||
self.attached.append(a)
|
||||
|
||||
|
||||
def test_tiingo_cite_has_no_attachment():
|
||||
store = _Store()
|
||||
assert TI.cite(store, "SPY") == "KEY1"
|
||||
assert store.attached == []
|
||||
assert "token" not in store.items[0].url.lower() and "SPY" in store.items[0].url
|
||||
|
||||
|
||||
def test_plan_targets_windows_and_start():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
T.write_securities(
|
||||
con, "C", [T.SecurityRow("C", "CI", "NYSE", date(2018, 12, 21), None, "s")]
|
||||
)
|
||||
T.write_securities(con, "E", [T.SecurityRow("E", "ELV", "NYSE", None, None, "s")])
|
||||
by = {
|
||||
t.ticker: t
|
||||
for t in TI.plan(
|
||||
con,
|
||||
tickers=["CI", "ELV", "ZZZ"],
|
||||
universe_on=False,
|
||||
benchmarks_on=True,
|
||||
used=set(),
|
||||
)
|
||||
}
|
||||
assert by["CI"].start == date(2018, 12, 21) and by["CI"].windows == (
|
||||
(date(2018, 12, 21), None),
|
||||
)
|
||||
assert by["ELV"].start == TI.FLOOR and by["ZZZ"].windows == ()
|
||||
assert by["SPY"].start == date(1993, 1, 29) and by["XLV"].start == date(
|
||||
1998, 12, 22
|
||||
)
|
||||
assert by["CI"].securities[0].ticker == "CI"
|
||||
|
||||
|
||||
def test_roundtrip_helpers():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
spy = pull("SPY", [rec("2024-01-02", 1, 100), rec("2024-01-03", 1, 101)])
|
||||
aaa = pull("AAA", [rec("2024-01-02", 1, 100), rec("2024-01-03", 1, 90)])
|
||||
TI.write_prices(con, "SPY", spy)
|
||||
TI.write_prices(con, "AAA", aaa)
|
||||
assert TI.write_prices(con, "AAA", []) == 0
|
||||
got = TI.read_returns(
|
||||
con, ["SPY", "AAA", "NONE"], date(2024, 1, 1), date(2024, 1, 31)
|
||||
)
|
||||
assert got["SPY"] == {date(2024, 1, 3): pytest.approx(0.01)}
|
||||
assert got["AAA"] == {date(2024, 1, 3): pytest.approx(-0.1)} and got["NONE"] == {}
|
||||
assert (
|
||||
TI.read_returns(con, ["SPY"], date(2024, 1, 4), date(2024, 2, 1))["SPY"] == {}
|
||||
)
|
||||
assert TI.trading_days(con) == [date(2024, 1, 2), date(2024, 1, 3)]
|
||||
assert TI.read_returns(con, [], date(2024, 1, 1), date(2024, 1, 2)) == {}
|
||||
m = TI.tri_monthly("SPY", spy, item_key="K")
|
||||
TI.write_series(con, "tiingo:SPY:tri", m)
|
||||
TI.write_series(con, "tiingo:SPY:tri", m)
|
||||
assert TI.read_series(con, ["SPY"]) == m and TI.read_series(con, []) == []
|
||||
assert TI.write_series(con, "x", []) == 0
|
||||
assert TI.last_pulls(con) == {
|
||||
"AAA": (date(2024, 1, 3), date(2026, 11, 12)),
|
||||
"SPY": (date(2024, 1, 3), date(2026, 11, 12)),
|
||||
}
|
||||
|
||||
|
||||
def test_pacer_and_state_dir_defaults():
|
||||
assert TI.new_pacer().min_interval == 75.0
|
||||
assert TI.state_dir().parts[-2:] == (".state", "market")
|
||||
assert TI.today() == date.today()
|
||||
|
||||
|
||||
def test_plan_deduplicates_names():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
got = TI.plan(
|
||||
con,
|
||||
tickers=["spy", "SPY"],
|
||||
universe_on=True,
|
||||
benchmarks_on=True,
|
||||
used=set(),
|
||||
seed_order=[],
|
||||
)
|
||||
assert [t.ticker for t in got] == ["SPY", "XLV", "IHF"]
|
||||
assert TI.clip_windows([], []) == []
|
||||
Reference in New Issue
Block a user