diff --git a/src/cli/market.py b/src/cli/market.py index 53b4472..3a073d2 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -21,6 +21,7 @@ import httpx import typer from market import edgar, http, tiingo +from market import fec as fec_api from market import lda as lda_api from market.seed import ( SEAT_LABEL_KEYS, @@ -32,6 +33,9 @@ from market.seed import ( ) from market.tables import ( EntityRow, + FecCommitteeRow, + FecContributionRow, + FecTotalsRow, FilingRow, HitRow, LdaFilingRow, @@ -50,6 +54,9 @@ from market.tables import ( retire_links, retire_organizations, write_entities, + write_fec_committees, + write_fec_contributions, + write_fec_totals, write_filings, write_gap, write_hits, @@ -67,6 +74,7 @@ ENTITIES_GAP = "sec-edgar:entities" PRICES_GAP = "tiingo:prices" FTS_GAP = "sec-edgar:fts" LDA_GAP = "lda:lda" +FEC_GAP = "fec:fec" # ── indirections the tests monkeypatch ─────────────────────────────── @@ -1258,6 +1266,152 @@ def lda( typer.echo(f"stored: {n} filings, {len(issues)} issue rows") +def _parse_cycles(text: str) -> tuple[int, ...]: + if not text.strip(): + return fec_api.CYCLES + try: + return tuple(int(part.strip()) for part in text.split(",") if part.strip()) + except ValueError: + typer.echo(f"--cycles {text} is not a list of years") + raise typer.Exit(2) from None + + +@app.command() +def fec( + entity: list[str] = typer.Option( + [], "--entity", help="A tracked entity key (repeatable; all when none given)." + ), + cycles: str = typer.Option( + "", + "--cycles", + help="Comma-separated cycles (default 2008 through 2026, even years).", + ), + limit: int = typer.Option( + 0, "--limit", help="At most N network requests (hoarded pages are free)." + ), + write: bool = typer.Option( + False, "--write", help="Store committees, totals and Schedule B." + ), +) -> None: + """FEC committees, cycle totals and Schedule B disbursements + (market.fec_committee, market.fec_totals, market.fec_contribution). A + curated committee id is fetched directly; otherwise a search hit is kept + only when its name equals the entity's name or an alias. Other hits are + printed as candidates and not stored. A rerun resumes from the hoard + (data/market/fec).""" + chosen_cycles = _parse_cycles(cycles) + if not chosen_cycles: + typer.echo("--cycles is empty") + raise typer.Exit(2) + key = _require_key("fec", FEC_GAP, write=write) + if key is None: + return + con = _read() + try: + tracked = _lda_tracked(con) + finally: + con.close() + if not tracked: + typer.echo("no tracked entities — run `stack market entities --write`") + raise typer.Exit(1) + by_key = {e.key: e for e in tracked} + for k in entity: + if k not in by_key: + typer.echo(f"unknown entity: {k}") + raise typer.Exit(2) + chosen = [by_key[k] for k in dict.fromkeys(entity)] or tracked + ids = fec_api.committee_ids() + pacer = fec_api.new_pacer() + budget = fec_api.Budget(limit) + root = http.hoard("fec") + committees: list[FecCommitteeRow] = [] + candidates: list[dict[str, str]] = [] + totals: list[FecTotalsRow] = [] + contribs: list[FecContributionRow] = [] + cl = http.client("fec", key) + try: + for e in chosen: + if budget.exhausted: + break + rows, cands = fec_api.resolve( + cl, + e, + ids.get(e.key, ()), + pacer=pacer, + cache=root, + budget=budget, + ) + committees.extend(rows) + candidates.extend(cands) + for row in rows: + for cycle in chosen_cycles: + if budget.exhausted: + break + got = fec_api.totals( + cl, + row.committee_id, + cycle, + pacer=pacer, + cache=root, + budget=budget, + ) + if got is not None: + totals.append(got) + contribs.extend( + fec_api.contributions( + cl, + row.committee_id, + cycle, + pacer=pacer, + cache=root, + budget=budget, + ) + ) + except http.Refused as exc: + _gap(FEC_GAP, str(exc), write=write) + return + finally: + cl.close() + + curated = sum(c.basis == "curated" for c in committees) + exact = sum(c.basis == "exact-name" for c in committees) + typer.echo( + f"{len(committees)} committees ({curated} curated, {exact} exact-name), " + f"{len(candidates)} candidates, {len(totals)} totals, " + f"{len(contribs)} schedule B rows; {budget.used} requests" + ) + for cand in candidates: + typer.echo( + f"candidate: {cand['entity_key']} {cand['committee_id']} {cand['name']}" + ) + if budget.exhausted: + typer.echo(f"stopped: {budget.why} (rerun resumes from the hoard)") + if not write: + typer.echo("dry run: pass --write to store") + return + + from cms.ingest_log import log_ingest + + with _batch() as con: + ensure_tables(con) + n_c = write_fec_committees(con, committees) + n_t = write_fec_totals(con, totals) + n_b = write_fec_contributions(con, contribs) + if not budget.exhausted: + clear_gap(con, FEC_GAP) + src = str(http.hoard(http.MANIFEST)) + for table, count in ( + ("market.fec_committee", n_c), + ("market.fec_totals", n_t), + ("market.fec_contribution", n_b), + ): + log_ingest( + con, module="market.fec", table_name=table, rows=count, source_file=src + ) + _publish() + typer.echo(f"stored: {n_c} committees, {n_t} totals, {n_b} schedule B rows") + + @app.command() def gaps() -> None: """Sources that could not be read (market.source_gap).""" diff --git a/src/market/fec.py b/src/market/fec.py new file mode 100644 index 0000000..dfb2ada --- /dev/null +++ b/src/market/fec.py @@ -0,0 +1,391 @@ +"""FEC committees, cycle totals and Schedule B disbursements. + +The key (``FEC_API_KEY``, else ``API_DATA_GOV_KEY``) travels only in the +``X-Api-Key`` header. A committee is stored from a curated +``fec_committee_ids`` entry, or, failing that, when its normalised name +equals an alias (``alias_match(prefix=False)``). Every other search hit is +a candidate: printed, not stored. Schedule B rows are stored as filed, +including an individual recipient's name; a memo row (``memo_code`` ``X``) +is stored and left out of the cycle sum. +""" + +from __future__ import annotations + +import datetime as dt +import json +import os +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Any + +import httpx + +from market import http +from market.seed import alias_match, load_seed, norm_org +from market.tables import ( + EntityRow, + FecCommitteeRow, + FecContributionRow, + FecTotalsRow, + ensure_tables, +) + +SOURCE = "fec" +FEC_API = "https://api.open.fec.gov/v1/" +PACER_INTERVAL = 3.7 # the published limit is 1,000 requests an hour +CYCLES = tuple(range(2008, 2028, 2)) +PER_PAGE = 100 + + +def new_pacer() -> http.Pacer: + return http.Pacer(PACER_INTERVAL) + + +def committee_ids() -> dict[str, tuple[str, ...]]: + """Curated FEC committee ids by entity key (an org key, or a company's + 10-digit CIK). Pseudo seats are skipped.""" + companies, orgs = load_seed() + out: dict[str, tuple[str, ...]] = {} + for org in orgs: + if org.fec_committee_ids and not org.pseudo: + out[org.key] = org.fec_committee_ids + for company in companies: + if company.fec_committee_ids: + out[company.cik] = company.fec_committee_ids + return out + + +def search_names(name: str, aliases: str) -> list[str]: + """The distinct (by normalised form) names to search: the entity's own + name first, then its ``|``-joined aliases.""" + seen: dict[str, str] = {} + for n in (name, *aliases.split("|")): + k = norm_org(n) + if k and k not in seen: + seen[k] = n.strip() + return list(seen.values()) + + +class Budget: + """At most *limit* network requests (0 = unlimited). A hoarded page + costs nothing.""" + + def __init__(self, limit: int = 0, stop: Callable[[], bool] | None = None) -> None: + self.limit = limit + self.stop = stop + self.used = 0 + self.exhausted = False + self.why = "" + + def take(self) -> bool: + if self.limit and self.used >= self.limit: + self.exhausted, self.why = True, "request limit" + return False + if self.stop is not None and self.stop(): + self.exhausted, self.why = True, "time limit" + return False + self.used += 1 + return True + + +def _date(value: Any) -> dt.date | None: + if not value: + return None + return dt.date.fromisoformat(str(value)[:10]) + + +def _money(value: Any) -> float: + return 0.0 if value in (None, "") else float(value) + + +def _text(value: Any) -> str: + return "" if value is None else str(value) + + +def cite_url(committee_id: str) -> str: + return f"https://www.fec.gov/data/committee/{committee_id}/" + + +def to_committee( + rec: Mapping[str, Any], *, entity_key: str, basis: str, source_file: str +) -> FecCommitteeRow: + cid = _text(rec.get("committee_id")) + return FecCommitteeRow( + committee_id=cid, + name=_text(rec.get("name")), + committee_type=_text(rec.get("committee_type")), + designation=_text(rec.get("designation")), + entity_key=entity_key, + basis=basis, + cite_url=cite_url(cid), + first_file_date=_date(rec.get("first_file_date")), + last_file_date=_date(rec.get("last_file_date")), + source_file=source_file, + ) + + +def to_totals(rec: Mapping[str, Any], *, source_file: str) -> FecTotalsRow: + return FecTotalsRow( + committee_id=_text(rec.get("committee_id")), + cycle=int(rec.get("cycle") or 0), + receipts=_money(rec.get("receipts")), + disbursements=_money(rec.get("disbursements")), + individual_contributions=_money(rec.get("individual_contributions")), + fed_candidate_committee_contributions=_money( + rec.get("fed_candidate_committee_contributions") + ), + independent_expenditures=_money(rec.get("independent_expenditures")), + coverage_start=_date( + rec.get("coverage_start_date") or rec.get("coverage_start") + ), + coverage_end=_date(rec.get("coverage_end_date") or rec.get("coverage_end")), + source_file=source_file, + ) + + +def to_contribution( + rec: Mapping[str, Any], *, source_file: str, cycle: int | None = None +) -> FecContributionRow: + raw_cycle = rec.get("two_year_transaction_period") + if raw_cycle in (None, ""): + raw_cycle = rec.get("cycle", cycle or 0) + amount = rec.get("disbursement_amount", rec.get("amount")) + return FecContributionRow( + sub_id=_text(rec.get("sub_id")), + committee_id=_text(rec.get("committee_id")), + cycle=int(raw_cycle or 0), + day=_date(rec.get("disbursement_date") or rec.get("day")), + amount=_money(amount), + recipient_committee_id=_text(rec.get("recipient_committee_id")), + recipient_name=_text(rec.get("recipient_name")), + candidate_id=_text(rec.get("candidate_id")), + candidate_name=_text(rec.get("candidate_name")), + candidate_office=_text(rec.get("candidate_office")), + candidate_state=_text( + rec.get("candidate_office_state") or rec.get("candidate_state") + ), + candidate_district=_text( + rec.get("candidate_office_district") or rec.get("candidate_district") + ), + memo_code=_text(rec.get("memo_code")), + source_file=source_file, + ) + + +def _page( + cl: httpx.Client, + url: str, + params: Mapping[str, Any] | None, + *, + pacer: Any, + path: Path | None, + refresh: bool, + budget: Budget | None, +) -> tuple[dict[str, Any], str] | None: + """One document: the hoarded copy when present, else a download (never + overwriting). ``None`` when the budget forbids the download.""" + if path is not None: + have = http.newest(path) + if have.is_file() and not refresh: + return json.loads(have.read_bytes()), str(have) + if budget is not None and not budget.take(): + return None + r = http.get(cl, url, params=params, pacer=pacer, source=SOURCE, ok_404=True) + if r.status_code == 404: + return {"results": []}, "" + try: + payload = r.json() + except ValueError: + payload = None + if not isinstance(payload, dict): + raise http.Refused( + SOURCE, r.status_code, "response is not JSON (a bot or challenge page?)" + ) + where = path + if path is not None: + where = http.save(path, r.content) + os.utime(where) + http.manifest(where, str(r.request.url)) + return payload, str(where or "") + + +def resolve( + cl: httpx.Client, + entity: EntityRow, + seed_ids: Sequence[str], + *, + pacer: Any, + cache: Path | None = None, + refresh: bool = False, + budget: Budget | None = None, +) -> tuple[list[FecCommitteeRow], list[dict[str, str]]]: + """Curated ids win (no name search). Otherwise each alias is searched + and a hit is stored only when its name equals an alias; every other hit + is a candidate. Returns ``(rows, candidates)``.""" + ids = [i for i in seed_ids if i] + if ids: + rows: list[FecCommitteeRow] = [] + for cid in ids: + path = None if cache is None else cache / "committee" / f"{cid}.json" + got = _page( + cl, + f"{FEC_API}committee/{cid}/", + None, + pacer=pacer, + path=path, + refresh=refresh, + budget=budget, + ) + if got is None: + break + payload, src = got + for rec in payload.get("results") or (): + rows.append( + to_committee( + rec, entity_key=entity.key, basis="curated", source_file=src + ) + ) + return rows, [] + + aliases = search_names(entity.name, entity.aliases) + found: dict[str, FecCommitteeRow] = {} + pending: list[dict[str, str]] = [] + for alias in aliases: + path = ( + None if cache is None else cache / "committees" / f"{http.slug(alias)}.json" + ) + got = _page( + cl, + f"{FEC_API}committees/", + {"q": alias, "per_page": 20}, + pacer=pacer, + path=path, + refresh=refresh, + budget=budget, + ) + if got is None: + break + payload, src = got + for rec in payload.get("results") or (): + cid = _text(rec.get("committee_id")) + name = _text(rec.get("name")) + if not cid or cid in found: + continue + if alias_match(name, aliases, prefix=False): + found[cid] = to_committee( + rec, entity_key=entity.key, basis="exact-name", source_file=src + ) + else: + pending.append( + { + "committee_id": cid, + "name": name, + "entity_key": entity.key, + "query": alias, + } + ) + matched = set(found) + candidates = [c for c in pending if c["committee_id"] not in matched] + return list(found.values()), candidates + + +def totals( + cl: httpx.Client, + cid: str, + cycle: int, + *, + pacer: Any, + cache: Path | None = None, + refresh: bool = False, + budget: Budget | None = None, +) -> FecTotalsRow | None: + """``/committee/{cid}/totals/?cycle=``, or ``None`` when that cycle has + no totals (or the budget stopped the download).""" + path = None if cache is None else cache / "totals" / cid / f"{cycle}.json" + got = _page( + cl, + f"{FEC_API}committee/{cid}/totals/", + {"cycle": cycle}, + pacer=pacer, + path=path, + refresh=refresh, + budget=budget, + ) + if got is None: + return None + payload, src = got + results = payload.get("results") or () + if not results: + return None + return to_totals(results[0], source_file=src) + + +def contributions( + cl: httpx.Client, + cid: str, + cycle: int, + *, + pacer: Any, + cache: Path | None = None, + refresh: bool = False, + budget: Budget | None = None, +) -> list[FecContributionRow]: + """Every Schedule B row for *cid* in *cycle*, keyset-paged on + ``pagination.last_indexes``. Memo rows are kept.""" + base = { + "committee_id": cid, + "two_year_transaction_period": cycle, + "per_page": PER_PAGE, + "sort": "-disbursement_date", + } + extra: dict[str, Any] = {} + seen: set[str] = set() + out: list[FecContributionRow] = [] + n = 0 + while True: + n += 1 + path = ( + None + if cache is None + else cache / "schedule_b" / cid / str(cycle) / f"{n}.json" + ) + got = _page( + cl, + f"{FEC_API}schedules/schedule_b/", + {**base, **extra}, + pacer=pacer, + path=path, + refresh=refresh, + budget=budget, + ) + if got is None: + break + payload, src = got + recs = payload.get("results") or () + for rec in recs: + out.append(to_contribution(rec, source_file=src, cycle=cycle)) + last = (payload.get("pagination") or {}).get("last_indexes") or {} + idx = _text(last.get("last_index")) + if not recs or not idx or idx in seen: + break + seen.add(idx) + extra = { + "last_index": last.get("last_index"), + "last_disbursement_date": last.get("last_disbursement_date") or "", + } + return out + + +def contributions_by_cycle(con: Any, entity_key: str) -> dict[int, float]: + """Schedule B amounts by cycle for *entity_key*. A memo row + (``memo_code`` ``X``) is not summed with the disbursement it restates.""" + ensure_tables(con) + got = con.execute( + "SELECT c.cycle, coalesce(sum(c.amount), 0) " + "FROM market.fec_contribution c " + "JOIN market.fec_committee m ON m.committee_id = c.committee_id " + "WHERE m.entity_key = ? AND c.memo_code <> 'X' " + "GROUP BY c.cycle ORDER BY c.cycle", + [entity_key], + ).fetchall() + return {int(cycle): float(total) for cycle, total in got} diff --git a/src/market/tables.py b/src/market/tables.py index fb84a88..84d9cb4 100644 --- a/src/market/tables.py +++ b/src/market/tables.py @@ -66,6 +66,25 @@ CREATE TABLE IF NOT EXISTS market.lda_filing ( ); CREATE TABLE IF NOT EXISTS market.lda_issue ( filing_uuid VARCHAR, general_issue_code VARCHAR, description VARCHAR +); +CREATE TABLE IF NOT EXISTS market.fec_committee ( + committee_id VARCHAR, name VARCHAR, committee_type VARCHAR, + designation VARCHAR, entity_key VARCHAR, basis VARCHAR, cite_url VARCHAR, + first_file_date DATE, last_file_date DATE, source_file VARCHAR +); +CREATE TABLE IF NOT EXISTS market.fec_totals ( + committee_id VARCHAR, cycle INTEGER, receipts DOUBLE, + disbursements DOUBLE, individual_contributions DOUBLE, + fed_candidate_committee_contributions DOUBLE, + independent_expenditures DOUBLE, coverage_start DATE, + coverage_end DATE, source_file VARCHAR +); +CREATE TABLE IF NOT EXISTS market.fec_contribution ( + sub_id VARCHAR, committee_id VARCHAR, cycle INTEGER, day DATE, + amount DOUBLE, recipient_committee_id VARCHAR, recipient_name VARCHAR, + candidate_id VARCHAR, candidate_name VARCHAR, candidate_office VARCHAR, + candidate_state VARCHAR, candidate_district VARCHAR, memo_code VARCHAR, + source_file VARCHAR ) """ @@ -265,6 +284,60 @@ class LdaIssueRow: description: str +@dataclass(frozen=True) +class FecCommitteeRow: + """One FEC committee tied to an entity. ``basis`` is ``curated`` (a seed + id) or ``exact-name`` (the normalised name equals an alias).""" + + committee_id: str + name: str + committee_type: str + designation: str + entity_key: str + basis: str + cite_url: str + first_file_date: date | None + last_file_date: date | None + source_file: str + + +@dataclass(frozen=True) +class FecTotalsRow: + """One committee's FEC totals for one two-year cycle.""" + + committee_id: str + cycle: int + receipts: float + disbursements: float + individual_contributions: float + fed_candidate_committee_contributions: float + independent_expenditures: float + coverage_start: date | None + coverage_end: date | None + source_file: str + + +@dataclass(frozen=True) +class FecContributionRow: + """One Schedule B disbursement, stored as filed. ``memo_code`` ``X`` + restates another row and is left out of cycle sums.""" + + sub_id: str + committee_id: str + cycle: int + day: date | None + amount: float + recipient_committee_id: str + recipient_name: str + candidate_id: str + candidate_name: str + candidate_office: str + candidate_state: str + candidate_district: str + memo_code: str + source_file: str + + def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int: """Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is a statement per row, far too slow for filing lists).""" @@ -610,6 +683,96 @@ def read_lda_issues(con: Any) -> list[LdaIssueRow]: ) +# ── fec ────────────────────────────────────────────────────────────── + + +def write_fec_committees(con: Any, rows: Sequence[FecCommitteeRow]) -> int: + """Replace each committee by ``committee_id``.""" + ensure_tables(con) + if not rows: + return 0 + for chunk in _chunks(sorted({r.committee_id for r in rows})): + con.execute( + "DELETE FROM market.fec_committee WHERE committee_id IN " + f"({','.join('?' * len(chunk))})", + list(chunk), + ) + return _insert(con, "market.fec_committee", rows) + + +def read_fec_committees( + con: Any, entity_key: str | None = None +) -> list[FecCommitteeRow]: + rows = _select( + con, + FecCommitteeRow, + "market.fec_committee", + "entity_key = ?" if entity_key is not None else "", + [entity_key] if entity_key is not None else [], + ) + return sorted((FecCommitteeRow(*r) for r in rows), key=lambda r: r.committee_id) + + +def write_fec_totals(con: Any, rows: Sequence[FecTotalsRow]) -> int: + """Replace the totals of each ``(committee_id, cycle)`` in *rows*.""" + ensure_tables(con) + if not rows: + return 0 + keys = sorted({(r.committee_id, r.cycle) for r in rows}) + for part in _chunks(keys, 400): + con.execute( + "DELETE FROM market.fec_totals WHERE (committee_id, cycle) IN " + "(SELECT * FROM (VALUES " + ", ".join(["(?, ?)"] * len(part)) + "))", + [v for k in part for v in k], + ) + return _insert(con, "market.fec_totals", rows) + + +def read_fec_totals(con: Any, committee_id: str | None = None) -> list[FecTotalsRow]: + rows = _select( + con, + FecTotalsRow, + "market.fec_totals", + "committee_id = ?" if committee_id is not None else "", + [committee_id] if committee_id is not None else [], + ) + return sorted( + (FecTotalsRow(*r) for r in rows), key=lambda r: (r.committee_id, r.cycle) + ) + + +def write_fec_contributions(con: Any, rows: Sequence[FecContributionRow]) -> int: + """Replace every Schedule B row of each ``(committee_id, cycle)`` in + *rows* (a rerun drops disbursements the source no longer returns).""" + ensure_tables(con) + if not rows: + return 0 + keys = sorted({(r.committee_id, r.cycle) for r in rows}) + for part in _chunks(keys, 400): + con.execute( + "DELETE FROM market.fec_contribution WHERE (committee_id, cycle) IN " + "(SELECT * FROM (VALUES " + ", ".join(["(?, ?)"] * len(part)) + "))", + [v for k in part for v in k], + ) + return _insert(con, "market.fec_contribution", rows) + + +def read_fec_contributions( + con: Any, committee_id: str | None = None +) -> list[FecContributionRow]: + rows = _select( + con, + FecContributionRow, + "market.fec_contribution", + "committee_id = ?" if committee_id is not None else "", + [committee_id] if committee_id is not None else [], + ) + return sorted( + (FecContributionRow(*r) for r in rows), + key=lambda r: (r.committee_id, r.cycle, r.sub_id), + ) + + # ── districts ──────────────────────────────────────────────────────── diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index 2ffcfa3..3782c81 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -1912,3 +1912,197 @@ def test_lda_cli_tracks_seed_companies_only(lda): assert res.exit_code == 0, res.output assert {r.url.params["client_name"] for r in seen} >= {seed.name} assert "0 filings" in res.output + + +# ── fec ────────────────────────────────────────────────────────────── + +FEC_SECRET = "fec-cli-key-do-not-leak" + + +def _fec_entities(c): + T.write_entities( + c, + [ + T.EntityRow( + "AAFP", + "American Academy of Family Physicians", + "organization", + aliases="AMERICAN ACADEMY OF FAMILY PHYSICIANS", + ), + T.EntityRow("AMA", "AMA", "organization", aliases="AMA"), + T.EntityRow( + "PRIMARY-CARE-SEAT", + "Primary care rotating-seat societies", + "organization", + aliases="PRIMARY CARE ROTATING SEAT", + pseudo=True, + ), + ], + ) + + +@pytest.fixture +def fec(env, monkeypatch): + from market import fec as fec_api + + c, store, published, hoard = env + _fec_entities(c) + seen: list[httpx.Request] = [] + page1 = json.loads((FX / "fec_schedule_b_page1.json").read_text()) + totals = json.loads((FX / "fec_totals_C00411553_2024.json").read_text()) + committee = json.loads((FX / "fec_committees_aafp.json").read_text()) + monkeypatch.setattr(fec_api, "new_pacer", lambda: _Pacer()) + state = {"status": 200} + + def handler(request): + seen.append(request) + if state["status"] != 200: + return httpx.Response(state["status"], text="no") + path = request.url.path + if path.endswith("/totals/"): + return httpx.Response(200, json=totals) + if path.endswith("/schedule_b/"): + if "last_index" in request.url.params: + return httpx.Response(200, json={"results": [], "pagination": {}}) + return httpx.Response(200, json=page1) + if path.startswith("/v1/committees"): + return httpx.Response( + 200, + json={ + "results": [ + { + "committee_id": "C00999999", + "name": "AMALGAMATED TRANSIT UNION PAC", + "committee_type": "Q", + "designation": "U", + } + ] + }, + ) + return httpx.Response(200, json=committee) + + monkeypatch.setattr( + http, + "client", + lambda source, key=None: httpx.Client( + transport=httpx.MockTransport(handler), + headers={"X-Api-Key": key} if key else {}, + ), + ) + monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET) + return c, store, published, hoard, seen, state + + +def test_fec_cli_write_from_cache(fec): + c, _, published, hoard, seen, state = fec + T.write_gap(c, "fec:fec", "old") + dry = runner.invoke(app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024"]) + assert dry.exit_code == 0, dry.output + assert "dry run: pass --write" in dry.output + assert "1 committees (1 curated, 0 exact-name)" in dry.output + assert _count(c, "market.fec_committee") == 0 + assert published == [] + assert seen and all("/committees" not in r.url.path for r in seen) + assert all(FEC_SECRET not in str(r.url) for r in seen) + n = len(seen) + state["status"] = 500 + res = runner.invoke( + app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"] + ) + assert res.exit_code == 0, res.output + assert len(seen) == n # the hoard answered; the 500 was never requested + assert "stored: 1 committees, 1 totals, 3 schedule B rows" in res.output + (committee,) = T.read_fec_committees(c) + assert committee.basis == "curated" and committee.committee_id == "C00411553" + assert committee.source_file.endswith("fec/committee/C00411553.json") + (totals,) = T.read_fec_totals(c) + assert totals.receipts == 513798.01 + assert totals.fed_candidate_committee_contributions == 683300.0 + names = {r.recipient_name for r in T.read_fec_contributions(c)} + assert "JANE Q PUBLIC" in names and "MEMO OF THE 100" in names + from market.fec import contributions_by_cycle + + assert contributions_by_cycle(c, "AAFP") == {2024: 125.0} + assert T.read_gaps(c) == [] + assert published == [True] + assert { + (t, n) + for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall() + } == { + ("market.fec_committee", 1), + ("market.fec_totals", 1), + ("market.fec_contribution", 3), + } + manifest = (hoard / "MANIFEST.jsonl").read_text() + assert FEC_SECRET not in manifest + assert (hoard / "fec" / "schedule_b" / "C00411553" / "2024" / "1.json").is_file() + + +def test_fec_cli_no_write_prints_candidates(fec): + c, _, published, _, seen, _ = fec + res = runner.invoke(app, ["market", "fec", "--entity", "AMA", "--cycles", "2024"]) + assert res.exit_code == 0, res.output + assert "candidate: AMA C00999999 AMALGAMATED TRANSIT UNION PAC" in res.output + assert "0 committees" in res.output and "1 candidates" in res.output + assert "dry run: pass --write" in res.output + assert _count(c, "market.fec_committee") == 0 + assert _count(c, "market.fec_contribution") == 0 + assert published == [] + assert seen[0].url.params["q"] == "AMA" + assert all( + "/committee/" not in r.url.path or r.url.path.startswith("/v1/committees") + for r in seen + ) + default = runner.invoke(app, ["market", "fec", "--entity", "AMA"]) + assert default.exit_code == 0, default.output + assert "1 candidates" in default.output + bad = runner.invoke(app, ["market", "fec", "--entity", "NOPE", "--cycles", "2024"]) + assert bad.exit_code == 2 and "unknown entity: NOPE" in bad.output + cycles = runner.invoke(app, ["market", "fec", "--cycles", "nope"]) + assert cycles.exit_code == 2 and "not a list of years" in cycles.output + + +def test_fec_cli_refusal_prints_gap(fec, monkeypatch): + c, _, published, _, seen, state = fec + monkeypatch.setattr(http, "api_key", lambda source, env=None: None) + missing = runner.invoke( + app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"] + ) + assert missing.exit_code == 0, missing.output + assert "gap: fec:fec:" in missing.output + assert "FEC_API_KEY" in missing.output + assert seen == [] + assert [g.source for g in T.read_gaps(c)] == ["fec:fec"] + assert published == [True] + monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET) + state["status"] = 403 + res = runner.invoke( + app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"] + ) + assert res.exit_code == 0, res.output + assert "gap: fec:fec: fec: HTTP 403" in res.output + assert FEC_SECRET not in res.output + assert _count(c, "market.fec_committee") == 0 + assert len(seen) == 1 + assert all(FEC_SECRET not in str(r.url) for r in seen) + assert seen[0].headers["x-api-key"] == FEC_SECRET + + +def test_fec_cli_limit_keeps_the_gap_and_empty_registry(fec, monkeypatch): + c, _, _, _, seen, _ = fec + T.write_gap(c, "fec:fec", "old") + res = runner.invoke( + app, + ["market", "fec", "--cycles", "2024,2022", "--limit", "1", "--write"], + ) + assert res.exit_code == 0, res.output + assert "stopped: request limit" in res.output + assert [g.note for g in T.read_gaps(c)] == ["old"] + assert _count(c, "market.fec_committee") == 1 + assert _count(c, "market.fec_totals") == 0 + assert len(seen) == 1 + monkeypatch.setattr(mk, "_lda_tracked", lambda con: []) + empty = runner.invoke(app, ["market", "fec", "--cycles", "2024"]) + assert empty.exit_code == 1 and "no tracked entities" in empty.output + blank = runner.invoke(app, ["market", "fec", "--cycles", ","]) + assert blank.exit_code == 2 and "--cycles is empty" in blank.output diff --git a/tests/market/fixtures/fec_committees_aafp.json b/tests/market/fixtures/fec_committees_aafp.json new file mode 100644 index 0000000..c6ad7be --- /dev/null +++ b/tests/market/fixtures/fec_committees_aafp.json @@ -0,0 +1,13 @@ +{ + "api_version": "1.0", + "results": [ + { + "committee_id": "C00411553", + "name": "AMERICAN ACADEMY OF FAMILY PHYSICIANS POLITICAL ACTION COMMITTEE (FAMMEDPAC)", + "committee_type": "Q", + "designation": "U", + "first_file_date": "2003-07-15", + "last_file_date": "2024-10-15" + } + ] +} diff --git a/tests/market/fixtures/fec_schedule_b_page1.json b/tests/market/fixtures/fec_schedule_b_page1.json new file mode 100644 index 0000000..48710b4 --- /dev/null +++ b/tests/market/fixtures/fec_schedule_b_page1.json @@ -0,0 +1,59 @@ +{ + "pagination": { + "count": 3, + "page": 1, + "pages": 2, + "per_page": 100, + "last_indexes": { + "last_index": 123, + "last_disbursement_date": "2024-03-01" + } + }, + "results": [ + { + "sub_id": "4010120240001", + "committee_id": "C00411553", + "two_year_transaction_period": 2024, + "disbursement_date": "2024-06-15", + "disbursement_amount": 100.0, + "recipient_committee_id": "C00111111", + "recipient_name": "FRIENDS OF SOMEONE", + "candidate_id": "H0KS00001", + "candidate_name": "SOMEONE, A", + "candidate_office": "H", + "candidate_office_state": "KS", + "candidate_office_district": "02", + "memo_code": null + }, + { + "sub_id": "4010120240002", + "committee_id": "C00411553", + "two_year_transaction_period": 2024, + "disbursement_date": "2024-05-01", + "disbursement_amount": 25.0, + "recipient_committee_id": null, + "recipient_name": "JANE Q PUBLIC", + "candidate_id": null, + "candidate_name": null, + "candidate_office": null, + "candidate_office_state": null, + "candidate_office_district": null, + "memo_code": null + }, + { + "sub_id": "4010120240003", + "committee_id": "C00411553", + "two_year_transaction_period": 2024, + "disbursement_date": "2024-04-01", + "disbursement_amount": 100.0, + "recipient_committee_id": null, + "recipient_name": "MEMO OF THE 100", + "candidate_id": null, + "candidate_name": null, + "candidate_office": null, + "candidate_office_state": null, + "candidate_office_district": null, + "memo_code": "X" + } + ] +} diff --git a/tests/market/fixtures/fec_totals_C00411553_2024.json b/tests/market/fixtures/fec_totals_C00411553_2024.json new file mode 100644 index 0000000..10c99d9 --- /dev/null +++ b/tests/market/fixtures/fec_totals_C00411553_2024.json @@ -0,0 +1,16 @@ +{ + "api_version": "1.0", + "results": [ + { + "committee_id": "C00411553", + "cycle": 2024, + "receipts": 513798.01, + "disbursements": 696685.91, + "individual_contributions": 480000.0, + "fed_candidate_committee_contributions": 683300.0, + "independent_expenditures": 0.0, + "coverage_start_date": "2023-01-01", + "coverage_end_date": "2024-12-31" + } + ] +} diff --git a/tests/market/test_fec.py b/tests/market/test_fec.py new file mode 100644 index 0000000..080f601 --- /dev/null +++ b/tests/market/test_fec.py @@ -0,0 +1,399 @@ +"""market.fec: curated committee ids, exact-name fallback, totals, Schedule B.""" + +from __future__ import annotations + +import json +from datetime import date +from pathlib import Path + +import httpx +import pytest + +from market import fec as F +from market import http +from market import tables as T + +FX = Path(__file__).parent / "fixtures" +SECRET = "fec-test-key-do-not-leak" +PAC = "AMERICAN ACADEMY OF FAMILY PHYSICIANS POLITICAL ACTION COMMITTEE (FAMMEDPAC)" +PAC_ALIAS = "AMERICAN ACADEMY OF FAMILY PHYSICIANS POLITICAL ACTION COMMITTEE FAMMEDPAC" +SHORT = "AMERICAN ACADEMY OF FAMILY PHYSICIANS" + + +def _load(name: str) -> dict: + return json.loads((FX / name).read_text()) + + +@pytest.fixture +def con(): + c = __import__("duckdb").connect(":memory:") + T.ensure_tables(c) + yield c + c.close() + + +class Pacer: + def __init__(self): + self.n = 0 + + def wait(self): + self.n += 1 + + +def entity(key, name, aliases=""): + return T.EntityRow(key, name, "organization", aliases=aliases or name) + + +def _client(handler, *, key: str | None = None) -> httpx.Client: + headers = {"X-Api-Key": key} if key else {} + return httpx.Client(transport=httpx.MockTransport(handler), headers=headers) + + +def test_committee_curated_ids_win(): + assert F.FEC_API == "https://api.open.fec.gov/v1/" + assert F.new_pacer().min_interval == 3.7 + seen = [] + body = _load("fec_committees_aafp.json") + + def handler(request): + seen.append(request) + return httpx.Response(200, json=body) + + cl = _client(handler) + rows, cands = F.resolve(cl, entity("AAFP", SHORT), ["C00411553"], pacer=Pacer()) + assert cands == [] + assert [(r.committee_id, r.basis, r.entity_key) for r in rows] == [ + ("C00411553", "curated", "AAFP") + ] + assert rows[0].cite_url == "https://www.fec.gov/data/committee/C00411553/" + assert [r.url.host for r in seen] == ["api.open.fec.gov"] + assert seen[0].url.path == "/v1/committee/C00411553/" + assert all("/committees" not in r.url.path for r in seen) + assert "api_key" not in str(seen[0].url) + + +def test_committee_exact_name_fallback_cited(): + body = {"results": _load("fec_committees_aafp.json")["results"]} + + def handler(request): + return httpx.Response(200, json=body) + + cl = _client(handler) + rows, cands = F.resolve(cl, entity("AAFP", SHORT), [], pacer=Pacer()) + assert rows == [] + assert cands[0]["name"] == PAC + assert cands[0]["committee_id"] == "C00411553" + + rows, cands = F.resolve(cl, entity("AAFP", PAC_ALIAS), [], pacer=Pacer()) + assert cands == [] + assert rows[0].basis == "exact-name" + assert rows[0].cite_url == "https://www.fec.gov/data/committee/C00411553/" + assert rows[0].name == PAC + + +def test_short_alias_never_matches_inside_names(): + name = "AMALGAMATED TRANSIT UNION PAC" + + def handler(request): + assert request.url.params["q"] == "AMA" + return httpx.Response( + 200, + json={"results": [{"committee_id": "C00999999", "name": name}]}, + ) + + rows, cands = F.resolve(_client(handler), entity("AMA", "AMA"), [], pacer=Pacer()) + assert rows == [] + assert [(c["committee_id"], c["name"]) for c in cands] == [("C00999999", name)] + + +def test_totals_row(): + body = _load("fec_totals_C00411553_2024.json") + + def handler(request): + assert request.url.path == "/v1/committee/C00411553/totals/" + assert request.url.params["cycle"] == "2024" + return httpx.Response(200, json=body) + + row = F.totals(_client(handler), "C00411553", 2024, pacer=Pacer()) + assert row is not None + assert row.receipts == 513798.01 + assert row.disbursements == 696685.91 + assert row.fed_candidate_committee_contributions == 683300.0 + assert (row.coverage_start, row.coverage_end) == ( + date(2023, 1, 1), + date(2024, 12, 31), + ) + + +def test_schedule_b_keyset_paging(): + page1 = _load("fec_schedule_b_page1.json") + page2 = { + "results": [ + { + "sub_id": "4010120240004", + "committee_id": "C00411553", + "two_year_transaction_period": 2024, + "disbursement_date": "2024-02-01", + "disbursement_amount": 1.0, + "recipient_name": "SECOND PAGE", + "memo_code": None, + } + ], + "pagination": {}, + } + seen = [] + + def handler(request): + seen.append(request) + if "last_index" in request.url.params: + return httpx.Response(200, json=page2) + return httpx.Response(200, json=page1) + + rows = F.contributions(_client(handler), "C00411553", 2024, pacer=Pacer()) + assert len(seen) == 2 + assert seen[0].url.path == "/v1/schedules/schedule_b/" + assert seen[0].url.params["per_page"] == "100" + assert seen[0].url.params["sort"] == "-disbursement_date" + assert seen[0].url.params["two_year_transaction_period"] == "2024" + assert "last_index" not in seen[0].url.params + assert seen[1].url.params["last_index"] == "123" + assert seen[1].url.params["last_disbursement_date"] == "2024-03-01" + assert [r.sub_id for r in rows] == [ + "4010120240001", + "4010120240002", + "4010120240003", + "4010120240004", + ] + + +def test_memo_rows_excluded_from_totals(con): + T.write_fec_committees( + con, + [ + T.FecCommitteeRow( + "C00411553", + PAC, + "Q", + "U", + "AAFP", + "curated", + "https://www.fec.gov/data/committee/C00411553/", + None, + None, + "", + ) + ], + ) + rows = [ + T.FecContributionRow( + "a", + "C00411553", + 2024, + date(2024, 6, 1), + 100.0, + "C1", + "PAC", + "", + "", + "", + "", + "", + "", + "", + ), + T.FecContributionRow( + "b", + "C00411553", + 2024, + date(2024, 5, 1), + 25.0, + "", + "JANE Q PUBLIC", + "", + "", + "", + "", + "", + "", + "", + ), + T.FecContributionRow( + "c", + "C00411553", + 2024, + date(2024, 4, 1), + 100.0, + "", + "MEMO", + "", + "", + "", + "", + "", + "X", + "", + ), + ] + T.write_fec_contributions(con, rows) + assert F.contributions_by_cycle(con, "AAFP") == {2024: 125.0} + + +def test_schedule_b_rows_stored_as_filed(con): + page = _load("fec_schedule_b_page1.json") + + def handler(request): + body = dict(page) + body["pagination"] = {} + return httpx.Response(200, json=body) + + rows = F.contributions(_client(handler), "C00411553", 2024, pacer=Pacer()) + person = next(r for r in rows if r.recipient_name == "JANE Q PUBLIC") + assert person.recipient_committee_id == "" + assert person.amount == 25.0 + memo = next(r for r in rows if r.memo_code == "X") + assert memo.recipient_name == "MEMO OF THE 100" + T.write_fec_contributions(con, rows) + stored = T.read_fec_contributions(con) + assert [r.recipient_name for r in stored if r.sub_id == "4010120240002"] == [ + "JANE Q PUBLIC" + ] + assert any(r.memo_code == "X" for r in stored) + + +def test_fec_error_message_has_no_key(): + seen = [] + + def handler(request): + seen.append(request) + return httpx.Response(403, text="no") + + cl = _client(handler, key=SECRET) + with pytest.raises(http.Refused) as exc: + F.totals(cl, "C00411553", 2024, pacer=Pacer()) + assert exc.value.source == "fec" + assert SECRET not in str(exc.value) + assert SECRET not in str(seen[0].url) + assert "api_key" not in str(seen[0].url) + assert seen[0].headers["x-api-key"] == SECRET + + def bot(request): + return httpx.Response(200, text="Just a moment...") + + with pytest.raises(http.Refused) as exc2: + F.totals(_client(bot, key=SECRET), "C00411553", 2024, pacer=Pacer()) + assert "not JSON" in str(exc2.value) + assert SECRET not in str(exc2.value) + + +def test_budget_edges_and_blank_fields(monkeypatch): + company = type("C", (), {"cik": "0000000001", "fec_committee_ids": ("C1",)})() + org = type( + "O", (), {"key": "AAFP", "fec_committee_ids": ("C2",), "pseudo": False} + )() + pseudo = type( + "O", (), {"key": "SEAT", "fec_committee_ids": ("C9",), "pseudo": True} + )() + monkeypatch.setattr(F, "load_seed", lambda: ([company], [org, pseudo])) + assert F.committee_ids() == {"AAFP": ("C2",), "0000000001": ("C1",)} + + limited = F.Budget(1) + assert limited.take() is True + assert limited.take() is False and limited.why == "request limit" + stopped = F.Budget(0, stop=lambda: True) + assert stopped.take() is False and stopped.why == "time limit" + assert F._date(None) is None and F._date("") is None + assert F._money("") == 0.0 + + row = F.to_contribution( + { + "sub_id": "z", + "amount": "", + "candidate_state": "KS", + "candidate_district": "01", + }, + source_file="s", + cycle=2024, + ) + assert (row.cycle, row.amount, row.day, row.candidate_state) == ( + 2024, + 0.0, + None, + "KS", + ) + + seen = [] + + def handler(request): + seen.append(request.url.path) + return httpx.Response( + 200, + json={ + "results": [ + {"committee_id": "", "name": "NO ID"}, + {"committee_id": "C1", "name": PAC}, + {"committee_id": "C1", "name": PAC}, + ] + }, + ) + + cl = _client(handler) + org_row = entity("AAFP", SHORT, aliases=f"{SHORT}|{PAC_ALIAS}") + rows, cands = F.resolve(cl, org_row, [], pacer=Pacer(), budget=F.Budget(1)) + assert [r.committee_id for r in rows] == ["C1"] + assert rows[0].basis == "exact-name" + assert cands == [] + assert seen == ["/v1/committees/"] # the second alias stops on the budget + + def curated(request): + return httpx.Response(200, json=_load("fec_committees_aafp.json")) + + one, _ = F.resolve( + _client(curated), + entity("AAFP", SHORT), + ["C00411553", "C00000001"], + pacer=Pacer(), + budget=F.Budget(1), + ) + assert [r.committee_id for r in one] == ["C00411553"] + + spent = F.Budget(1) + spent.take() + + def boom(request): + raise AssertionError("budget should have stopped the download") + + assert ( + F.totals(_client(boom), "C00411553", 2024, pacer=Pacer(), budget=spent) is None + ) + assert ( + F.contributions(_client(boom), "C00411553", 2024, pacer=Pacer(), budget=spent) + == [] + ) + + def empty(request): + return httpx.Response(200, json={"results": []}) + + assert F.totals(_client(empty), "C00411553", 2022, pacer=Pacer()) is None + + +def test_fec_hoard_reuses_and_skips_a_missing_committee(tmp_path, monkeypatch): + monkeypatch.setattr(http, "hoard_root", lambda: tmp_path) + calls = [] + body = _load("fec_committees_aafp.json") + + def handler(request): + calls.append(request.url.path) + if "C00000000" in request.url.path: + return httpx.Response(404, json={"results": []}) + return httpx.Response(200, json=body) + + cl = _client(handler) + cache = tmp_path / "fec" + kw = dict(pacer=Pacer(), cache=cache) + first, _ = F.resolve(cl, entity("AAFP", SHORT), ["C00411553"], **kw) + again, _ = F.resolve(cl, entity("AAFP", SHORT), ["C00411553"], **kw) + assert calls == ["/v1/committee/C00411553/"] + assert first[0].source_file.endswith("fec/committee/C00411553.json") + assert again[0].committee_id == "C00411553" + empty, cands = F.resolve(cl, entity("AAFP", SHORT), ["C00000000"], **kw) + assert empty == [] and cands == [] diff --git a/tests/market/test_imports.py b/tests/market/test_imports.py index 05cdf0c..797a007 100644 --- a/tests/market/test_imports.py +++ b/tests/market/test_imports.py @@ -13,6 +13,7 @@ MODULES = [ "market.edgar", "market.tiingo", "market.lda", + "market.fec", ] _PROBE = """ diff --git a/tests/market/test_tables.py b/tests/market/test_tables.py index 7063580..2f1c371 100644 --- a/tests/market/test_tables.py +++ b/tests/market/test_tables.py @@ -43,6 +43,9 @@ def test_ensure_tables_idempotent(): "series", "lda_filing", "lda_issue", + "fec_committee", + "fec_totals", + "fec_contribution", } @@ -141,6 +144,46 @@ LDA_FILING = T.LdaFilingRow( "f.json", ) LDA_ISSUE = T.LdaIssueRow(LDA_FILING.filing_uuid, "BUD", "No Bill No.") +FEC_COMMITTEE = T.FecCommitteeRow( + "C00411553", + "FAMMEDPAC", + "Q", + "U", + "AAFP", + "curated", + "https://www.fec.gov/data/committee/C00411553/", + date(2003, 7, 15), + date(2024, 10, 15), + "committee/C00411553.json", +) +FEC_TOTALS = T.FecTotalsRow( + "C00411553", + 2024, + 513798.01, + 696685.91, + 480000.0, + 683300.0, + 0.0, + date(2023, 1, 1), + date(2024, 12, 31), + "totals/C00411553/2024.json", +) +FEC_CONTRIBUTION = T.FecContributionRow( + "4010120240002", + "C00411553", + 2024, + date(2024, 5, 1), + 25.0, + "", + "JANE Q PUBLIC", + "", + "", + "", + "", + "", + "", + "schedule_b/C00411553/2024/1.json", +) SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY") ROUNDTRIP = { @@ -154,6 +197,21 @@ ROUNDTRIP = { lambda c, r: T.write_lda_issues(c, [r]), lambda c: T.read_lda_issues(c), ), + "fec_committee": ( + FEC_COMMITTEE, + lambda c, r: T.write_fec_committees(c, [r]), + lambda c: T.read_fec_committees(c), + ), + "fec_totals": ( + FEC_TOTALS, + lambda c, r: T.write_fec_totals(c, [r]), + lambda c: T.read_fec_totals(c), + ), + "fec_contribution": ( + FEC_CONTRIBUTION, + lambda c, r: T.write_fec_contributions(c, [r]), + lambda c: T.read_fec_contributions(c), + ), "entity": ( ENTITY, lambda c, r: T.write_entities(c, [r]), @@ -212,6 +270,9 @@ def test_write_empty_is_noop(con): assert T.write_securities(con, "1", []) == 0 assert T.write_lda_filings(con, []) == 0 assert T.write_lda_issues(con, []) == 0 + assert T.write_fec_committees(con, []) == 0 + assert T.write_fec_totals(con, []) == 0 + assert T.write_fec_contributions(con, []) == 0 def test_lda_readers_filter_by_entity(con):