diff --git a/src/cli/market.py b/src/cli/market.py index bf394a6..85586ca 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -33,6 +33,7 @@ from market import events as events_api from market import fec as fec_api from market import footprint as footprint_api from market import lda as lda_api +from market import portfolio as portfolio_api from market.seed import ( SEAT_LABEL_KEYS, SEED_PATH, @@ -56,6 +57,7 @@ from market.tables import ( clear_gap, ensure_tables, read_entities, + read_events, read_gaps, read_hits, read_links, @@ -68,6 +70,7 @@ from market.tables import ( write_bill_actions, write_bill_members, write_bills, + write_cars, write_county_districts, write_entities, write_entity_sites, @@ -85,6 +88,7 @@ from market.tables import ( write_prices, write_securities, write_series, + write_tenure_returns, write_zip_districts, ) from pfs.codetables import is_missing_table_error @@ -2092,6 +2096,143 @@ def events( typer.echo(f"stored: {n} events") +def _parse_windows(raw: list[str]) -> list[tuple[int, int]]: + if not raw: + from market.abnormal import WINDOWS + + return list(WINDOWS) + parsed: list[tuple[int, int]] = [] + for item in raw: + lo, hi = item.split(",") + parsed.append((int(lo), int(hi))) + return parsed + + +def _echo_cars(rows: list) -> None: + import json + from collections import Counter + + counts: Counter[str] = Counter() + for row in rows: + if row.unit == "firm": + typer.echo(f"{row.event_id} {row.ticker}") + elif row.event_id == "pooled": + typer.echo( + f"pooled {row.win_lo},{row.win_hi} CAR {row.car} " + f"CI {row.ci_low} {row.ci_high}" + ) + else: + typer.echo( + f"{row.event_id} {row.unit} members {row.n_members} " + f"gap {row.n_gap} CAR {row.car} CI {row.ci_low} {row.ci_high}" + ) + for note in json.loads(row.caveats): + counts[note] += 1 + for note, n in sorted(counts.items()): + typer.echo(f"caveat {n} {note}") + + +@app.command() +def car( + family: str = typer.Option("", "--family", help="Family key."), + specialty: str = typer.Option("", "--specialty", help="Specialty key."), + program: str = typer.Option( + "", "--program", help="Program key. Events are the PFS rules." + ), + kind: list[str] = typer.Option( + [], "--kind", help="Event kind. Repeatable. Default: every kind." + ), + model: str = typer.Option( + "market", "--model", help="market, sector, or two-factor." + ), + window: list[str] = typer.Option( + [], + "--window", + help="lo,hi trading-day offsets. Default: the five study windows.", + ), + strict: bool = typer.Option( + False, "--strict", help="Keep only curated and revenue-exposure links." + ), + write: bool = typer.Option(False, "--write", help="Store market.abnormal_return."), +) -> None: + """CARs for one family, specialty, or program portfolio.""" + picked = [ + (value, scope) + for value, scope in ( + (family, "family"), + (specialty, "specialty"), + (program, "program"), + ) + if value + ] + if len(picked) != 1: + typer.echo("pass exactly one of --family, --specialty or --program") + raise typer.Exit(2) + target, scope = picked[0] + windows = _parse_windows(window) + con = _read() + try: + if not tiingo.trading_days(con, "SPY"): + typer.echo("no prices stored — run `stack market prices`") + return + found = read_events(con) + if kind: + found = [event for event in found if event.kind in set(kind)] + rows = portfolio_api.run_events( + con, + found, + scope=scope, + target=target, + model=model, + windows=windows, + strict=strict, + ) + finally: + con.close() + _echo_cars(rows) + if not write: + typer.echo("dry run: pass --write") + return + with _batch() as batch: + n = write_cars(batch, scope, target, model, rows) + _publish() + typer.echo(f"stored: {n} abnormal returns") + + +@app.command() +def tenure( + key: str = typer.Option("", "--key", help="One tenure key."), + all_keys: bool = typer.Option(False, "--all", help="Every tenure key."), + write: bool = typer.Option(False, "--write", help="Store market.tenure_return."), +) -> None: + """Annual BHAR against SPY, seat years against the other years.""" + if bool(key) == all_keys: + typer.echo("pass exactly one of --key or --all") + raise typer.Exit(2) + con = _read() + try: + rows = portfolio_api.tenure_rows(con, None if all_keys else [key]) + finally: + con.close() + show = rows if all_keys else [row for row in rows if row.key == key] + for name in sorted({row.key for row in show}): + got = portfolio_api.contrast([row for row in show if row.key == name]) + diff = "" if got.diff is None else f"{got.diff:.4f}" + typer.echo( + f"{got.key} on {got.mean_on} off {got.mean_off} diff {diff} " + f"n_on {got.n_on} n_off {got.n_off}" + ) + for note in got.caveats: + typer.echo(note) + if not write: + typer.echo("dry run: pass --write") + return + with _batch() as batch: + n = write_tenure_returns(batch, show) + _publish() + typer.echo(f"stored: {n} tenure returns") + + @app.command() def gaps() -> None: """Sources that could not be read (market.source_gap).""" diff --git a/src/market/portfolio.py b/src/market/portfolio.py new file mode 100644 index 0000000..f45d524 --- /dev/null +++ b/src/market/portfolio.py @@ -0,0 +1,373 @@ +"""Equal-weight portfolios, stored CARs, and the RUC-tenure contrast. + +No numpy at import time: the market model imports it inside its functions. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from zoneinfo import ZoneInfo + +from market.abnormal import MODELS, event_car, pooled +from market.events import day0, tenures +from market.tables import ( + CarRow, + FilingRow, + MarketEvent, + TenureRow, + read_entities, + read_filings, + read_links, + read_securities, +) +from market.tiingo import read_returns, trading_days +from pfs import event_study +from pfs.codetables import read_seats + +_ASSOC = "association, not a causal estimate" +_EARNINGS = "earnings release (8-K Item 2.02) in window" +_ROSTER = "roster years 2019\u20132026 only" +_KEEP = frozenset({"curated", "revenue_exposure"}) +_YEARS = range(2019, 2027) +_ET = ZoneInfo("America/New_York") + + +@dataclass(frozen=True) +class Member: + entity_key: str + ticker: str + + +@dataclass(frozen=True) +class TenureContrast: + key: str + mean_on: float | None + mean_off: float | None + diff: float | None + ci_low: float | None + ci_high: float | None + n_on: int + n_off: int + caveats: tuple[str, ...] + + +def _open_on(start: date | None, end: date | None, on: date) -> bool: + """``None`` bounds are open. The end date itself still counts.""" + started = start is None or start <= on + return started and (end is None or end >= on) + + +def members( + con, + *, + scope: str, + target: str, + on: date, + strict: bool = False, +) -> tuple[list[Member], list[str]]: + """Linked names with a security valid on *on*, and keys with no series. + + A link whose only basis is ``sic`` is neither. ``strict`` keeps + ``curated`` and ``revenue_exposure`` and ignores the rest. An entity + that ended before *on* is omitted. + """ + grouped: dict[str, list] = defaultdict(list) + for link in read_links(con, scope=scope, target=target): + grouped[link.entity_key].append(link) + ended = {row.key: row.ended_on for row in read_entities(con)} + series: dict[str, list] = defaultdict(list) + for row in read_securities(con): + series[row.entity_key].append(row) + chosen: list[Member] = [] + gaps: list[str] = [] + for key in sorted(grouped): + group = grouped[key] + if all(link.basis == "sic" for link in group): + continue + if strict: + group = [link for link in group if link.strength in _KEEP] + if not group: + continue + end = ended.get(key) + if end is not None and end < on: + continue + usable = [ + row + for row in series.get(key, []) + if _open_on(row.valid_from, row.valid_to, on) + ] + if not usable: + gaps.append(key) + continue + chosen.append(Member(key, usable[0].ticker)) + return chosen, gaps + + +def portfolio_returns( + returns: Mapping[str, Mapping[date, float]], members: Sequence[Member] +) -> dict[date, float]: + """Equal weight across the members that have a return that day.""" + tickers = [member.ticker for member in members] + days: set[date] = set() + for ticker in tickers: + days.update(returns.get(ticker, {})) + out: dict[date, float] = {} + for day in days: + vals = [ + returns[ticker][day] for ticker in tickers if day in returns.get(ticker, {}) + ] + if vals: + out[day] = sum(vals) / len(vals) + return out + + +def earnings_days(filings: Sequence[FilingRow]) -> set[date]: + """8-K item 2.02 dates. ``accepted`` is UTC; 16:00 ET rolls to the next day.""" + found: set[date] = set() + for filing in filings: + if filing.form != "8-K" or "2.02" not in filing.items: + continue + local = datetime.fromisoformat( + filing.accepted.replace("Z", "+00:00") + ).astimezone(_ET) + day = local.date() + if local.hour >= 16: + day += timedelta(days=1) + found.add(day) + return found + + +def _stored( + scope: str, + target: str, + car, + caveats: list[str], + n_members: int, + n_gap: int, + today: date, +) -> CarRow: + return CarRow( + car.event_id, + scope, + target, + car.unit, + car.ticker, + car.model, + car.window[0], + car.window[1], + car.car, + car.se, + car.ci_low, + car.ci_high, + car.n_est, + n_members, + n_gap, + json.dumps(caveats), + today, + ) + + +def _window_days( + calendar: Sequence[date], d0: date, window: tuple[int, int] +) -> list[date]: + i0 = list(calendar).index(d0) + lo, hi = window + return [calendar[i] for i in range(i0 + lo, i0 + hi + 1) if 0 <= i < len(calendar)] + + +def run_events( + con, + events: Sequence[MarketEvent], + *, + scope: str, + target: str, + model: str, + windows: Sequence[tuple[int, int]], + strict: bool, +) -> list[CarRow]: + """Firm, portfolio and pooled CARs for the day-precision events in *events*. + + A program run keeps events aimed at ``PFS`` and members linked to *target*. + """ + calendar = trading_days(con, "SPY") + event_target = "PFS" if scope == "program" else target + plans: list[tuple[MarketEvent, date, list[Member], list[str]]] = [] + for event in events: + if ( + event.scope != scope + or event.target != event_target + or event.precision != "day" + ): + continue + d0 = day0(event, calendar) + if d0 is None: + continue + mem, gap = members(con, scope=scope, target=target, on=d0, strict=strict) + plans.append((event, d0, mem, gap)) + if not plans: + return [] + + tickers = {member.ticker for _, _, mem, _ in plans for member in mem} + series = read_returns( + con, sorted({*tickers, *MODELS[model]}), calendar[0], calendar[-1] + ) + factors = {name: series.get(name, {}) for name in MODELS[model]} + entities = {row.key: row for row in read_entities(con)} + other = [d0 for _, d0, _, _ in plans] + today = date.today() + out: list[CarRow] = [] + by_window: dict[tuple[int, int], list] = defaultdict(list) + for event, d0, mem, gap in plans: + earnings = {} + for member in mem: + entity = entities.get(member.entity_key) + cik = entity.cik if entity is not None and entity.cik else member.entity_key + earnings[member.entity_key] = earnings_days(read_filings(con, cik)) + for window in windows: + win = set(_window_days(calendar, d0, window)) + hit_any = False + for member in mem: + hit = bool(earnings[member.entity_key] & win) + hit_any = hit_any or hit + car = event_car( + series.get(member.ticker, {}), + factors, + calendar, + d0, + model=model, + window=window, + event_id=event.event_id, + unit="firm", + ticker=member.ticker, + other_events=[day for day in other if day != d0], + ) + notes = list(car.caveats) + if hit: + notes.insert(0, _EARNINGS) + out.append(_stored(scope, target, car, notes, 1, len(gap), today)) + port = event_car( + portfolio_returns(series, mem), + factors, + calendar, + d0, + model=model, + window=window, + event_id=event.event_id, + unit=f"portfolio:{scope}:{target}", + ticker="", + other_events=[day for day in other if day != d0], + ) + notes = list(port.caveats) + if hit_any: + notes.insert(0, _EARNINGS) + out.append(_stored(scope, target, port, notes, len(mem), len(gap), today)) + by_window[tuple(window)].append(port) + for window, cars in by_window.items(): + agg = pooled(cars) + out.append( + CarRow( + "pooled", + scope, + target, + f"portfolio:{scope}:{target}", + "", + model, + window[0], + window[1], + agg.mean_car, + None, + agg.ci_low, + agg.ci_high, + 0, + 0, + 0, + json.dumps(list(agg.caveats)), + today, + ) + ) + return out + + +def year_bhar( + port: Mapping[date, float], + mkt: Mapping[date, float], + year: int, + *, + min_days: int = 200, +) -> float | None: + """Π(1+r_p) − Π(1+r_m) over the year's common trading days.""" + days = sorted(day for day in port if day.year == year and day in mkt) + if len(days) < min_days: + return None + grown_p = 1.0 + grown_m = 1.0 + for day in days: + grown_p *= 1.0 + port[day] + grown_m *= 1.0 + mkt[day] + return grown_p - grown_m + + +def contrast(rows: Sequence[TenureRow]) -> TenureContrast: + """Seat years against the other years. Fewer than two in either group has no CI.""" + key = rows[0].key if rows else "" + on = [row.bhar for row in rows if row.on_ruc and row.bhar is not None] + off = [row.bhar for row in rows if not row.on_ruc and row.bhar is not None] + mean_on = sum(on) / len(on) if on else None + mean_off = sum(off) / len(off) if off else None + diff = None if mean_on is None or mean_off is None else mean_on - mean_off + lo = hi = None + if len(on) >= 2 and len(off) >= 2: + left = event_study.bootstrap_draws(on, seed=event_study._SEED) + right = event_study.bootstrap_draws(off, seed=event_study._SEED + 1) + lo, hi = event_study.percentile_interval( + [a - b for a, b in zip(left, right, strict=True)] + ) + return TenureContrast( + key, mean_on, mean_off, diff, lo, hi, len(on), len(off), (_ASSOC, _ROSTER) + ) + + +def tenure_rows( + con, keys: Sequence[str] | None = None, *, min_days: int = 200 +) -> list[TenureRow]: + """One row per key per year, 2019–2026. On RUC when that year's roster holds the key. + + Members are the specialty's links as of 31 December. BHAR is the equal-weight + portfolio against SPY. + """ + held = tenures(read_seats(con, "RUC")) + wanted = sorted({row.key for row in held}) if keys is None else list(keys) + on_years: dict[str, set[int]] = defaultdict(set) + for tenure in held: + on_years[tenure.key].update(range(tenure.first_year, tenure.last_year + 1)) + out: list[TenureRow] = [] + for key in wanted: + per_year: list[tuple[int, list[Member]]] = [] + tickers: set[str] = set() + for year in _YEARS: + mem, _gap = members( + con, scope="specialty", target=key, on=date(year, 12, 31) + ) + per_year.append((year, mem)) + tickers.update(member.ticker for member in mem) + series = read_returns( + con, sorted({"SPY", *tickers}), date(2019, 1, 1), date(2026, 12, 31) + ) + spy = series.get("SPY", {}) + for year, mem in per_year: + out.append( + TenureRow( + key, + year, + year in on_years.get(key, ()), + year_bhar( + portfolio_returns(series, mem), spy, year, min_days=min_days + ), + len(mem), + ) + ) + return out diff --git a/src/market/tables.py b/src/market/tables.py index 715934f..9292f31 100644 --- a/src/market/tables.py +++ b/src/market/tables.py @@ -129,6 +129,15 @@ CREATE TABLE IF NOT EXISTS market.event ( event_id VARCHAR, kind VARCHAR, event_date DATE, after_close BOOLEAN, precision VARCHAR, scope VARCHAR, target VARCHAR, item_key VARCHAR, p_id INTEGER, document_number VARCHAR, filed_at VARCHAR, note VARCHAR +); +CREATE TABLE IF NOT EXISTS market.abnormal_return ( + event_id VARCHAR, scope VARCHAR, target VARCHAR, unit VARCHAR, ticker VARCHAR, + model VARCHAR, win_lo INTEGER, win_hi INTEGER, car DOUBLE, se DOUBLE, + ci_low DOUBLE, ci_high DOUBLE, n_est INTEGER, n_members INTEGER, n_gap INTEGER, + caveats VARCHAR, computed_on DATE +); +CREATE TABLE IF NOT EXISTS market.tenure_return ( + key VARCHAR, year INTEGER, on_ruc BOOLEAN, bhar DOUBLE, n_members INTEGER ) """ @@ -543,6 +552,40 @@ class MarketEvent: note: str +@dataclass(frozen=True) +class CarRow: + """One stored CAR (``market.abnormal_return``). ``caveats`` is a JSON list.""" + + event_id: str + scope: str + target: str + unit: str + ticker: str + model: str + win_lo: int + win_hi: int + car: float | None + se: float | None + ci_low: float | None + ci_high: float | None + n_est: int + n_members: int + n_gap: int + caveats: str + computed_on: date + + +@dataclass(frozen=True) +class TenureRow: + """One specialty-year buy-and-hold contrast input (``market.tenure_return``).""" + + key: str + year: int + on_ruc: bool + bhar: float | None + n_members: int + + def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int: """Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is a statement per row, far too slow for filing lists).""" @@ -1182,6 +1225,60 @@ def read_events(con: Any) -> list[MarketEvent]: ) +def write_cars( + con: Any, scope: str, target: str, model: str, rows: Sequence[CarRow] +) -> int: + """Replace the stored CARs for one scope, target and model. + + An empty batch deletes nothing. + """ + ensure_tables(con) + if not rows: + return 0 + con.execute( + "DELETE FROM market.abnormal_return WHERE scope = ? AND target = ? AND model = ?", + [scope, target, model], + ) + return _insert(con, "market.abnormal_return", rows) + + +def read_cars( + con: Any, scope: str, target: str, model: str | None = None +) -> list[CarRow]: + where, params = ["scope = ?", "target = ?"], [scope, target] + if model is not None: + where.append("model = ?") + params.append(model) + rows = _select(con, CarRow, "market.abnormal_return", " AND ".join(where), params) + return sorted( + (CarRow(*r) for r in rows), + key=lambda r: (r.event_id, r.unit, r.ticker, r.win_lo, r.win_hi), + ) + + +def write_tenure_returns(con: Any, rows: Sequence[TenureRow]) -> int: + """Replace stored years for the keys in *rows*. An empty batch deletes nothing.""" + ensure_tables(con) + if not rows: + return 0 + keys = sorted({r.key for r in rows}) + con.execute( + "DELETE FROM market.tenure_return WHERE key IN (" + + ", ".join("?" * len(keys)) + + ")", + keys, + ) + return _insert(con, "market.tenure_return", rows) + + +def read_tenure_returns(con: Any, key: str | None = None) -> list[TenureRow]: + where, params = "", [] + if key is not None: + where, params = "key = ?", [key] + rows = _select(con, TenureRow, "market.tenure_return", where, params) + return sorted((TenureRow(*r) for r in rows), key=lambda r: (r.key, r.year)) + + # ── districts ──────────────────────────────────────────────────────── diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index 29972ff..3668e16 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -3244,3 +3244,180 @@ def test_committees_cli_from_after_to(congress): res = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "113"]) assert res.exit_code == 2 assert "--from 119 is after --to 113" in res.output + + +def _car_company(key: str, ended: date | None = None) -> T.EntityRow: + return T.EntityRow(key, key, "company", key, ended_on=ended) + + +def _car_days(d0: date, before: int, after: int) -> list[date]: + from datetime import timedelta + + days: list[date] = [] + day = d0 + while len(days) < before: + day -= timedelta(days=1) + if day.weekday() < 5: + days.append(day) + days.reverse() + days.append(d0) + day = d0 + extra = 0 + while extra < after: + day += timedelta(days=1) + if day.weekday() < 5: + days.append(day) + extra += 1 + return days + + +def _seed_car(c, *, prices: bool) -> None: + d0 = date(2022, 11, 3) + T.ensure_tables(c) + T.write_entities(c, [_car_company("OSH")]) + T.write_securities( + c, "OSH", [T.SecurityRow("OSH", "OSH", "NYSE", date(2020, 8, 6), None, "seed")] + ) + T.write_links( + c, [T.LinkRow("OSH", "family", "HOME-VISITS", "curated", "curated", "url", "u")] + ) + T.write_events( + c, + [ + T.MarketEvent( + "rule-final:2022-23873", + "rule-final", + d0, + False, + "day", + "family", + "HOME-VISITS", + "", + 0, + "", + "", + "", + ) + ], + ) + if not prices: + return + for ticker in ("OSH", "SPY"): + T.write_prices( + c, + ticker, + [ + T.PriceRow( + ticker, + day, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + ((i % 5) - 2) * 0.001, + ((i % 5) - 2) * 0.001, + date(2026, 11, 17), + "t", + ) + for i, day in enumerate(_car_days(d0, 250, 5)) + ], + ) + + +def _fake_tenure(con, keys, **_kwargs): + from market import portfolio as P + + rows = [ + P.TenureRow("FAMILY PRACTICE", 2019, True, 0.1, 2), + P.TenureRow("FAMILY PRACTICE", 2020, True, 0.3, 2), + P.TenureRow("FAMILY PRACTICE", 2021, False, 0.0, 2), + P.TenureRow("FAMILY PRACTICE", 2022, False, 0.2, 2), + P.TenureRow("ONCOLOGY", 2019, True, 0.5, 1), + P.TenureRow("ONCOLOGY", 2020, False, 0.4, 1), + ] + if keys: + want = set(keys) + return [row for row in rows if row.key in want] + return rows + + +def test_car_cli_writes_rows(env): + c, _, published, _ = env + _seed_car(c, prices=True) + res = runner.invoke( + app, + [ + "market", + "car", + "--family", + "HOME-VISITS", + "--kind", + "rule-final", + "--window", + "-1,1", + "--write", + ], + ) + assert res.exit_code == 0, res.output + rows = T.read_cars(c, "family", "HOME-VISITS") + assert len(rows) == 3 + assert any(r.unit == "firm" and r.ticker == "OSH" for r in rows) + assert any(r.unit == "portfolio:family:HOME-VISITS" for r in rows) + assert any(r.event_id == "pooled" for r in rows) + assert published == [True] + + +def test_car_cli_no_write_prints(env): + c, _, published, _ = env + _seed_car(c, prices=True) + res = runner.invoke( + app, ["market", "car", "--family", "HOME-VISITS", "--window", "-1,1"] + ) + assert res.exit_code == 0, res.output + assert "OSH" in res.output + assert "dry run: pass --write" in res.output + assert _count(c, "market.abnormal_return") == 0 + assert published == [] + + +def test_car_cli_no_prices_says_run_prices(env): + c, _, published, _ = env + _seed_car(c, prices=False) + res = runner.invoke(app, ["market", "car", "--family", "HOME-VISITS"]) + assert res.exit_code == 0, res.output + assert "stack market prices" in res.output + assert published == [] + + +def test_tenure_cli_prints_contrast(env, monkeypatch): + _, _, published, _ = env + monkeypatch.setattr("market.portfolio.tenure_rows", _fake_tenure) + res = runner.invoke(app, ["market", "tenure", "--key", "FAMILY PRACTICE"]) + assert res.exit_code == 0, res.output + assert "FAMILY PRACTICE" in res.output + assert "diff 0.1" in res.output + assert "association, not a causal estimate" in res.output + assert "roster years 2019\u20132026 only" in res.output + assert "ONCOLOGY" not in res.output + assert "dry run: pass --write" in res.output + assert published == [] + + +def test_tenure_cli_write(env, monkeypatch): + c, _, published, _ = env + T.ensure_tables(c) + monkeypatch.setattr("market.portfolio.tenure_rows", _fake_tenure) + T.write_tenure_returns(c, [T.TenureRow("CARDIOLOGY", 2019, True, 0.2, 3)]) + res = runner.invoke( + app, ["market", "tenure", "--key", "FAMILY PRACTICE", "--write"] + ) + assert res.exit_code == 0, res.output + rows = T.read_tenure_returns(c) + assert sum(r.key == "FAMILY PRACTICE" for r in rows) == 4 + assert any(r.key == "CARDIOLOGY" and r.bhar == 0.2 for r in rows) + assert all(r.key != "ONCOLOGY" for r in rows) + assert published == [True] diff --git a/tests/market/test_imports.py b/tests/market/test_imports.py index 1278652..3edf6a2 100644 --- a/tests/market/test_imports.py +++ b/tests/market/test_imports.py @@ -19,6 +19,7 @@ MODULES = [ "market.footprint", "market.events", "market.abnormal", + "market.portfolio", ] _PROBE = """ diff --git a/tests/market/test_portfolio.py b/tests/market/test_portfolio.py new file mode 100644 index 0000000..b9bfd06 --- /dev/null +++ b/tests/market/test_portfolio.py @@ -0,0 +1,395 @@ +"""Portfolios, stored CARs, and the RUC-tenure contrast.""" + +from __future__ import annotations + +from datetime import date, timedelta + +import duckdb +import pytest + +from market import portfolio as P +from market import tables as T +from pfs.codetables import SeatRow, write_seats + +D0 = date(2022, 11, 3) +_EARNINGS = "earnings release (8-K Item 2.02) in window" +_ASSOC = "association, not a causal estimate" +_ROSTER = "roster years 2019\u20132026 only" + + +@pytest.fixture +def con(): + c = duckdb.connect(":memory:") + T.ensure_tables(c) + yield c + c.close() + + +def _company(key: str, ended: date | None = None) -> T.EntityRow: + return T.EntityRow(key, key, "company", key, ended_on=ended) + + +def _security(key: str, ticker: str, start: date, end: date | None) -> T.SecurityRow: + return T.SecurityRow(key, ticker, "NYSE", start, end, "seed") + + +def _link( + key: str, + scope: str, + target: str, + basis: str = "curated", + strength: str = "curated", +) -> T.LinkRow: + return T.LinkRow(key, scope, target, basis, strength, "url", "u") + + +def _event( + event_id: str, + day: date, + scope: str, + target: str, + precision: str = "day", + kind: str = "rule-final", +) -> T.MarketEvent: + return T.MarketEvent( + event_id, kind, day, False, precision, scope, target, "", 0, "", "", "" + ) + + +def _weekdays(d0: date, before: int, after: int) -> list[date]: + days: list[date] = [] + day = d0 + while len(days) < before: + day -= timedelta(days=1) + if day.weekday() < 5: + days.append(day) + days.reverse() + days.append(d0) + day = d0 + extra = 0 + while extra < after: + day += timedelta(days=1) + if day.weekday() < 5: + days.append(day) + extra += 1 + return days + + +def _load_prices(con, tickers: list[str], days: list[date]) -> None: + for ticker in tickers: + rows = [] + for i, day in enumerate(days): + ret = ((i % 5) - 2) * 0.001 + rows.append( + T.PriceRow( + ticker, + day, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + ret, + ret, + date(2026, 11, 17), + "t", + ) + ) + T.write_prices(con, ticker, rows) + + +def _listed(con, key: str, ticker: str, scope: str, target: str) -> None: + T.write_entities(con, [_company(key)]) + T.write_securities(con, key, [_security(key, ticker, date(2020, 1, 2), None)]) + T.write_links(con, [_link(key, scope, target)]) + + +def _run(con, events, scope: str, target: str): + return P.run_events( + con, + events, + scope=scope, + target=target, + model="market", + windows=[(-1, 1)], + strict=False, + ) + + +def test_members_as_of_day0_include_later_delisted(con): + T.write_entities(con, [_company("OSH", date(2023, 5, 2))]) + T.write_securities( + con, "OSH", [_security("OSH", "OSH", date(2020, 8, 6), date(2023, 5, 2))] + ) + T.write_links(con, [_link("OSH", "family", "HOME-VISITS")]) + got, _gaps = P.members(con, scope="family", target="HOME-VISITS", on=D0) + assert got == [P.Member("OSH", "OSH")] + later, _later_gaps = P.members( + con, scope="family", target="HOME-VISITS", on=date(2024, 11, 5) + ) + assert later == [] + + +def test_no_price_series_counted_as_gap(con): + T.write_entities(con, [_company("GAP")]) + T.write_links(con, [_link("GAP", "family", "HOME-VISITS")]) + got, gaps = P.members(con, scope="family", target="HOME-VISITS", on=D0) + assert got == [] + assert gaps == ["GAP"] + + +def test_sic_only_entities_never_members(con): + T.write_entities(con, [_company("SICONLY")]) + T.write_securities( + con, "SICONLY", [_security("SICONLY", "SIC", date(2020, 1, 2), None)] + ) + T.write_links( + con, [_link("SICONLY", "sector", "8000", basis="sic", strength="sic")] + ) + got, gaps = P.members(con, scope="sector", target="8000", on=D0) + assert got == [] + assert gaps == [] + + +def test_equal_weight_daily_rebalanced(): + day = D0 + nxt = date(2022, 11, 4) + members = [P.Member("A", "AAA"), P.Member("B", "BBB")] + out = P.portfolio_returns({"AAA": {day: 0.1, nxt: 0.2}, "BBB": {day: 0.0}}, members) + assert out[day] == pytest.approx(0.05) + assert out[nxt] == pytest.approx(0.2) + + +def test_strict_members_only_curated_or_revenue_exposure(con): + strengths = { + "CUR": "curated", + "REV": "revenue_exposure", + "MENTION": "mention", + "RISK": "risk_factor", + } + T.write_entities(con, [_company(key) for key in strengths]) + for key, ticker in strengths.items(): + T.write_securities(con, key, [_security(key, ticker, date(2020, 1, 2), None)]) + T.write_links( + con, + [ + _link(key, "family", "HOME-VISITS", strength=strength) + for key, strength in strengths.items() + ], + ) + strict, gaps = P.members( + con, scope="family", target="HOME-VISITS", on=D0, strict=True + ) + assert {m.entity_key for m in strict} == {"CUR", "REV"} + assert gaps == [] + loose, _loose_gaps = P.members( + con, scope="family", target="HOME-VISITS", on=D0, strict=False + ) + assert {m.entity_key for m in loose} == set(strengths) + + +def test_earnings_in_window_caveat(con): + kept = T.FilingRow( + "OSH", "0001-22-1", "8-K", D0, "2022-11-03T13:00:00.000Z", None, "2.02", "a.htm" + ) + ignored = [ + T.FilingRow( + "OSH", + "0001-22-2", + "10-K", + D0, + "2022-11-03T13:00:00.000Z", + None, + "2.02", + "b.htm", + ), + T.FilingRow( + "OSH", + "0001-22-3", + "8-K", + D0, + "2022-11-03T13:00:00.000Z", + None, + "5.02", + "c.htm", + ), + ] + after_close = T.FilingRow( + "OSH", "0001-22-4", "8-K", D0, "2022-11-03T20:00:00.000Z", None, "2.02", "d.htm" + ) + assert P.earnings_days([kept, *ignored]) == {D0} + assert P.earnings_days([after_close]) == {date(2022, 11, 4)} + + days = _weekdays(D0, 250, 5) + _listed(con, "OSH", "OSH", "family", "HOME-VISITS") + _load_prices(con, ["OSH", "SPY"], days) + T.write_filings(con, "OSH", [kept]) + rows = _run( + con, + [_event("rule-final:2022-23873", D0, "family", "HOME-VISITS")], + "family", + "HOME-VISITS", + ) + firm = next(r for r in rows if r.unit == "firm") + port = next(r for r in rows if r.unit == "portfolio:family:HOME-VISITS") + assert firm.ticker == "OSH" and firm.car is not None and firm.n_members == 1 + assert port.ticker == "" and port.car is not None + assert _EARNINGS in firm.caveats and _EARNINGS in port.caveats + assert _ASSOC in firm.caveats + assert any(r.event_id == "pooled" for r in rows) + + +def test_month_precision_event_skipped(con): + days = _weekdays(D0, 250, 5) + _listed(con, "OSH", "OSH", "family", "HOME-VISITS") + _load_prices(con, ["OSH", "SPY"], days) + rows = _run( + con, + [ + _event("rule-final:day", D0, "family", "HOME-VISITS"), + _event("rule-final:month", D0, "family", "HOME-VISITS", precision="month"), + _event("rule-final:later", date(2099, 1, 4), "family", "HOME-VISITS"), + ], + "family", + "HOME-VISITS", + ) + assert rows + assert all(r.event_id not in {"rule-final:month", "rule-final:later"} for r in rows) + assert ( + _run( + con, + [ + _event( + "rule-final:only-month", + D0, + "family", + "HOME-VISITS", + precision="month", + ) + ], + "family", + "HOME-VISITS", + ) + == [] + ) + + +def test_specialty_run_uses_specialty_events(con): + days = _weekdays(D0, 250, 5) + _listed(con, "FAM", "FAM", "specialty", "FAMILY PRACTICE") + _listed(con, "OSH", "OSH", "family", "HOME-VISITS") + _load_prices(con, ["FAM", "OSH", "SPY"], days) + rows = _run( + con, + [ + _event("rule-final:spec", D0, "specialty", "FAMILY PRACTICE"), + _event("rule-final:fam", D0, "family", "HOME-VISITS"), + ], + "specialty", + "FAMILY PRACTICE", + ) + assert {r.event_id for r in rows if r.event_id != "pooled"} == {"rule-final:spec"} + assert {r.ticker for r in rows if r.unit == "firm"} == {"FAM"} + assert all(r.scope == "specialty" and r.target == "FAMILY PRACTICE" for r in rows) + + +def test_program_run_uses_pfs_events_and_program_members(con): + days = _weekdays(D0, 250, 5) + _listed(con, "MSSP", "MSSP", "program", "MSSP") + _listed(con, "ONLYPFS", "ONLYPFS", "program", "PFS") + _load_prices(con, ["MSSP", "ONLYPFS", "SPY"], days) + rows = _run( + con, + [ + _event("rule-final:pfs", D0, "program", "PFS"), + _event("rule-final:mssp", D0, "program", "MSSP"), + ], + "program", + "MSSP", + ) + assert {r.event_id for r in rows if r.event_id != "pooled"} == {"rule-final:pfs"} + assert {r.ticker for r in rows if r.unit == "firm"} == {"MSSP"} + assert all(r.scope == "program" and r.target == "MSSP" for r in rows) + + +def test_year_bhar(): + days = [date(2024, 1, 2), date(2024, 1, 3)] + port = {days[0]: 0.1, days[1]: -0.1} + mkt = {days[0]: 0.0, days[1]: 0.0} + assert P.year_bhar(port, mkt, 2024, min_days=2) == pytest.approx(-0.01) + assert P.year_bhar(port, mkt, 2024, min_days=3) is None + assert P.year_bhar(port, mkt, 2023, min_days=1) is None + + +def test_tenure_contrast(con): + rows = [ + T.TenureRow("FAMILY PRACTICE", 2019, True, 0.1, 1), + T.TenureRow("FAMILY PRACTICE", 2020, True, 0.3, 1), + T.TenureRow("FAMILY PRACTICE", 2021, True, None, 0), + T.TenureRow("FAMILY PRACTICE", 2022, False, 0.0, 1), + T.TenureRow("FAMILY PRACTICE", 2023, False, 0.2, 1), + ] + got = P.contrast(rows) + assert got.key == "FAMILY PRACTICE" + assert got.mean_on == pytest.approx(0.2) + assert got.mean_off == pytest.approx(0.1) + assert got.diff == pytest.approx(0.1) + assert got.n_on == 2 and got.n_off == 2 + assert got.ci_low is not None and got.ci_high is not None + assert _ASSOC in got.caveats + assert _ROSTER in got.caveats + short = P.contrast( + [ + T.TenureRow("FAMILY PRACTICE", 2019, True, 0.1, 1), + T.TenureRow("FAMILY PRACTICE", 2020, True, 0.3, 1), + T.TenureRow("FAMILY PRACTICE", 2021, False, 0.0, 1), + ] + ) + assert short.ci_low is None and short.ci_high is None + assert short.n_off == 1 + + write_seats( + con, + "RUC", + [ + SeatRow( + "RUC", + 2024, + "Family Medicine", + "FAMILY PRACTICE", + "permanent", + "u", + "K", + "s", + "specialty", + ) + ], + ) + _listed(con, "FAM", "FAM", "specialty", "FAMILY PRACTICE") + held = [date(2024, 6, 3), date(2024, 6, 4)] + T.write_prices( + con, + "FAM", + [ + T.PriceRow("FAM", held[0], 1, 1, 1, 1, 1, 1, 0, 1, 0.1, 0.1, D0, "t"), + T.PriceRow("FAM", held[1], 1, 1, 1, 1, 1, 1, 0, 1, -0.1, -0.1, D0, "t"), + ], + ) + T.write_prices( + con, + "SPY", + [ + T.PriceRow("SPY", held[0], 1, 1, 1, 1, 1, 1, 0, 1, 0.0, 0.0, D0, "t"), + T.PriceRow("SPY", held[1], 1, 1, 1, 1, 1, 1, 0, 1, 0.0, 0.0, D0, "t"), + ], + ) + roster = P.tenure_rows(con, ["FAMILY PRACTICE"], min_days=2) + assert [r.year for r in roster] == list(range(2019, 2027)) + year = next(r for r in roster if r.year == 2024) + assert year.on_ruc is True and year.n_members == 1 + assert year.bhar == pytest.approx(-0.01) + off = next(r for r in roster if r.year == 2023) + assert off.on_ruc is False and off.bhar is None diff --git a/tests/market/test_tables.py b/tests/market/test_tables.py index 72dd43c..34fff51 100644 --- a/tests/market/test_tables.py +++ b/tests/market/test_tables.py @@ -56,6 +56,8 @@ def test_ensure_tables_idempotent(): "entity_site", "footprint", "event", + "abnormal_return", + "tenure_return", } @@ -274,6 +276,26 @@ FOOTPRINT = T.FootprintRow( "prov-svc", ) SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY") +CAR = T.CarRow( + "rule-final:2022-23873", + "family", + "HOME-VISITS", + "firm", + "OSH", + "market", + -1, + 1, + 0.02, + 0.01, + 0.0004, + 0.0396, + 200, + 1, + 0, + '["association, not a causal estimate"]', + date(2026, 11, 17), +) +TENURE = T.TenureRow("FAMILY PRACTICE", 2024, True, 0.1, 4) ROUNDTRIP = { "lda_filing": ( @@ -388,6 +410,16 @@ ROUNDTRIP = { lambda c, r: T.write_gap(c, r.source, r.note), lambda c: T.read_gaps(c), ), + "abnormal_return": ( + CAR, + lambda c, r: T.write_cars(c, r.scope, r.target, r.model, [r]), + lambda c: T.read_cars(c, "family", "HOME-VISITS", "market"), + ), + "tenure_return": ( + TENURE, + lambda c, r: T.write_tenure_returns(c, [r]), + lambda c: T.read_tenure_returns(c, "FAMILY PRACTICE"), + ), } @@ -418,6 +450,14 @@ def test_write_empty_is_noop(con): assert T.write_entity_sites(con, []) == 0 assert T.write_footprints(con, []) == 0 assert T.write_events(con, []) == 0 + assert T.write_cars(con, "family", "HOME-VISITS", "market", []) == 0 + assert T.write_tenure_returns(con, []) == 0 + T.write_cars(con, CAR.scope, CAR.target, CAR.model, [CAR]) + assert T.write_cars(con, CAR.scope, CAR.target, CAR.model, []) == 0 + assert T.read_cars(con, "family", "HOME-VISITS") == [CAR] + T.write_tenure_returns(con, [TENURE]) + assert T.write_tenure_returns(con, []) == 0 + assert T.read_tenure_returns(con) == [TENURE] def test_lda_readers_filter_by_entity(con):