feat(market): FEC PAC committees (curated ids, exact-name fallback), cycle totals and Schedule B

A curated committee id is fetched directly. A search hit is stored only when its normalised name equals an alias, and every other hit stays a candidate. Schedule B rows are kept as filed, with memo rows left out of the cycle sum.
This commit is contained in:
kert
2026-11-14 13:37:35 -05:00
parent 1f2431f457
commit 56918b3174
10 changed files with 1451 additions and 0 deletions

View File

@@ -21,6 +21,7 @@ import httpx
import typer
from market import edgar, http, tiingo
from market import fec as fec_api
from market import lda as lda_api
from market.seed import (
SEAT_LABEL_KEYS,
@@ -32,6 +33,9 @@ from market.seed import (
)
from market.tables import (
EntityRow,
FecCommitteeRow,
FecContributionRow,
FecTotalsRow,
FilingRow,
HitRow,
LdaFilingRow,
@@ -50,6 +54,9 @@ from market.tables import (
retire_links,
retire_organizations,
write_entities,
write_fec_committees,
write_fec_contributions,
write_fec_totals,
write_filings,
write_gap,
write_hits,
@@ -67,6 +74,7 @@ ENTITIES_GAP = "sec-edgar:entities"
PRICES_GAP = "tiingo:prices"
FTS_GAP = "sec-edgar:fts"
LDA_GAP = "lda:lda"
FEC_GAP = "fec:fec"
# ── indirections the tests monkeypatch ───────────────────────────────
@@ -1258,6 +1266,152 @@ def lda(
typer.echo(f"stored: {n} filings, {len(issues)} issue rows")
def _parse_cycles(text: str) -> tuple[int, ...]:
if not text.strip():
return fec_api.CYCLES
try:
return tuple(int(part.strip()) for part in text.split(",") if part.strip())
except ValueError:
typer.echo(f"--cycles {text} is not a list of years")
raise typer.Exit(2) from None
@app.command()
def fec(
entity: list[str] = typer.Option(
[], "--entity", help="A tracked entity key (repeatable; all when none given)."
),
cycles: str = typer.Option(
"",
"--cycles",
help="Comma-separated cycles (default 2008 through 2026, even years).",
),
limit: int = typer.Option(
0, "--limit", help="At most N network requests (hoarded pages are free)."
),
write: bool = typer.Option(
False, "--write", help="Store committees, totals and Schedule B."
),
) -> None:
"""FEC committees, cycle totals and Schedule B disbursements
(market.fec_committee, market.fec_totals, market.fec_contribution). A
curated committee id is fetched directly; otherwise a search hit is kept
only when its name equals the entity's name or an alias. Other hits are
printed as candidates and not stored. A rerun resumes from the hoard
(data/market/fec)."""
chosen_cycles = _parse_cycles(cycles)
if not chosen_cycles:
typer.echo("--cycles is empty")
raise typer.Exit(2)
key = _require_key("fec", FEC_GAP, write=write)
if key is None:
return
con = _read()
try:
tracked = _lda_tracked(con)
finally:
con.close()
if not tracked:
typer.echo("no tracked entities — run `stack market entities --write`")
raise typer.Exit(1)
by_key = {e.key: e for e in tracked}
for k in entity:
if k not in by_key:
typer.echo(f"unknown entity: {k}")
raise typer.Exit(2)
chosen = [by_key[k] for k in dict.fromkeys(entity)] or tracked
ids = fec_api.committee_ids()
pacer = fec_api.new_pacer()
budget = fec_api.Budget(limit)
root = http.hoard("fec")
committees: list[FecCommitteeRow] = []
candidates: list[dict[str, str]] = []
totals: list[FecTotalsRow] = []
contribs: list[FecContributionRow] = []
cl = http.client("fec", key)
try:
for e in chosen:
if budget.exhausted:
break
rows, cands = fec_api.resolve(
cl,
e,
ids.get(e.key, ()),
pacer=pacer,
cache=root,
budget=budget,
)
committees.extend(rows)
candidates.extend(cands)
for row in rows:
for cycle in chosen_cycles:
if budget.exhausted:
break
got = fec_api.totals(
cl,
row.committee_id,
cycle,
pacer=pacer,
cache=root,
budget=budget,
)
if got is not None:
totals.append(got)
contribs.extend(
fec_api.contributions(
cl,
row.committee_id,
cycle,
pacer=pacer,
cache=root,
budget=budget,
)
)
except http.Refused as exc:
_gap(FEC_GAP, str(exc), write=write)
return
finally:
cl.close()
curated = sum(c.basis == "curated" for c in committees)
exact = sum(c.basis == "exact-name" for c in committees)
typer.echo(
f"{len(committees)} committees ({curated} curated, {exact} exact-name), "
f"{len(candidates)} candidates, {len(totals)} totals, "
f"{len(contribs)} schedule B rows; {budget.used} requests"
)
for cand in candidates:
typer.echo(
f"candidate: {cand['entity_key']} {cand['committee_id']} {cand['name']}"
)
if budget.exhausted:
typer.echo(f"stopped: {budget.why} (rerun resumes from the hoard)")
if not write:
typer.echo("dry run: pass --write to store")
return
from cms.ingest_log import log_ingest
with _batch() as con:
ensure_tables(con)
n_c = write_fec_committees(con, committees)
n_t = write_fec_totals(con, totals)
n_b = write_fec_contributions(con, contribs)
if not budget.exhausted:
clear_gap(con, FEC_GAP)
src = str(http.hoard(http.MANIFEST))
for table, count in (
("market.fec_committee", n_c),
("market.fec_totals", n_t),
("market.fec_contribution", n_b),
):
log_ingest(
con, module="market.fec", table_name=table, rows=count, source_file=src
)
_publish()
typer.echo(f"stored: {n_c} committees, {n_t} totals, {n_b} schedule B rows")
@app.command()
def gaps() -> None:
"""Sources that could not be read (market.source_gap)."""

391
src/market/fec.py Normal file
View File

@@ -0,0 +1,391 @@
"""FEC committees, cycle totals and Schedule B disbursements.
The key (``FEC_API_KEY``, else ``API_DATA_GOV_KEY``) travels only in the
``X-Api-Key`` header. A committee is stored from a curated
``fec_committee_ids`` entry, or, failing that, when its normalised name
equals an alias (``alias_match(prefix=False)``). Every other search hit is
a candidate: printed, not stored. Schedule B rows are stored as filed,
including an individual recipient's name; a memo row (``memo_code`` ``X``)
is stored and left out of the cycle sum.
"""
from __future__ import annotations
import datetime as dt
import json
import os
from collections.abc import Callable, Mapping, Sequence
from pathlib import Path
from typing import Any
import httpx
from market import http
from market.seed import alias_match, load_seed, norm_org
from market.tables import (
EntityRow,
FecCommitteeRow,
FecContributionRow,
FecTotalsRow,
ensure_tables,
)
SOURCE = "fec"
FEC_API = "https://api.open.fec.gov/v1/"
PACER_INTERVAL = 3.7 # the published limit is 1,000 requests an hour
CYCLES = tuple(range(2008, 2028, 2))
PER_PAGE = 100
def new_pacer() -> http.Pacer:
return http.Pacer(PACER_INTERVAL)
def committee_ids() -> dict[str, tuple[str, ...]]:
"""Curated FEC committee ids by entity key (an org key, or a company's
10-digit CIK). Pseudo seats are skipped."""
companies, orgs = load_seed()
out: dict[str, tuple[str, ...]] = {}
for org in orgs:
if org.fec_committee_ids and not org.pseudo:
out[org.key] = org.fec_committee_ids
for company in companies:
if company.fec_committee_ids:
out[company.cik] = company.fec_committee_ids
return out
def search_names(name: str, aliases: str) -> list[str]:
"""The distinct (by normalised form) names to search: the entity's own
name first, then its ``|``-joined aliases."""
seen: dict[str, str] = {}
for n in (name, *aliases.split("|")):
k = norm_org(n)
if k and k not in seen:
seen[k] = n.strip()
return list(seen.values())
class Budget:
"""At most *limit* network requests (0 = unlimited). A hoarded page
costs nothing."""
def __init__(self, limit: int = 0, stop: Callable[[], bool] | None = None) -> None:
self.limit = limit
self.stop = stop
self.used = 0
self.exhausted = False
self.why = ""
def take(self) -> bool:
if self.limit and self.used >= self.limit:
self.exhausted, self.why = True, "request limit"
return False
if self.stop is not None and self.stop():
self.exhausted, self.why = True, "time limit"
return False
self.used += 1
return True
def _date(value: Any) -> dt.date | None:
if not value:
return None
return dt.date.fromisoformat(str(value)[:10])
def _money(value: Any) -> float:
return 0.0 if value in (None, "") else float(value)
def _text(value: Any) -> str:
return "" if value is None else str(value)
def cite_url(committee_id: str) -> str:
return f"https://www.fec.gov/data/committee/{committee_id}/"
def to_committee(
rec: Mapping[str, Any], *, entity_key: str, basis: str, source_file: str
) -> FecCommitteeRow:
cid = _text(rec.get("committee_id"))
return FecCommitteeRow(
committee_id=cid,
name=_text(rec.get("name")),
committee_type=_text(rec.get("committee_type")),
designation=_text(rec.get("designation")),
entity_key=entity_key,
basis=basis,
cite_url=cite_url(cid),
first_file_date=_date(rec.get("first_file_date")),
last_file_date=_date(rec.get("last_file_date")),
source_file=source_file,
)
def to_totals(rec: Mapping[str, Any], *, source_file: str) -> FecTotalsRow:
return FecTotalsRow(
committee_id=_text(rec.get("committee_id")),
cycle=int(rec.get("cycle") or 0),
receipts=_money(rec.get("receipts")),
disbursements=_money(rec.get("disbursements")),
individual_contributions=_money(rec.get("individual_contributions")),
fed_candidate_committee_contributions=_money(
rec.get("fed_candidate_committee_contributions")
),
independent_expenditures=_money(rec.get("independent_expenditures")),
coverage_start=_date(
rec.get("coverage_start_date") or rec.get("coverage_start")
),
coverage_end=_date(rec.get("coverage_end_date") or rec.get("coverage_end")),
source_file=source_file,
)
def to_contribution(
rec: Mapping[str, Any], *, source_file: str, cycle: int | None = None
) -> FecContributionRow:
raw_cycle = rec.get("two_year_transaction_period")
if raw_cycle in (None, ""):
raw_cycle = rec.get("cycle", cycle or 0)
amount = rec.get("disbursement_amount", rec.get("amount"))
return FecContributionRow(
sub_id=_text(rec.get("sub_id")),
committee_id=_text(rec.get("committee_id")),
cycle=int(raw_cycle or 0),
day=_date(rec.get("disbursement_date") or rec.get("day")),
amount=_money(amount),
recipient_committee_id=_text(rec.get("recipient_committee_id")),
recipient_name=_text(rec.get("recipient_name")),
candidate_id=_text(rec.get("candidate_id")),
candidate_name=_text(rec.get("candidate_name")),
candidate_office=_text(rec.get("candidate_office")),
candidate_state=_text(
rec.get("candidate_office_state") or rec.get("candidate_state")
),
candidate_district=_text(
rec.get("candidate_office_district") or rec.get("candidate_district")
),
memo_code=_text(rec.get("memo_code")),
source_file=source_file,
)
def _page(
cl: httpx.Client,
url: str,
params: Mapping[str, Any] | None,
*,
pacer: Any,
path: Path | None,
refresh: bool,
budget: Budget | None,
) -> tuple[dict[str, Any], str] | None:
"""One document: the hoarded copy when present, else a download (never
overwriting). ``None`` when the budget forbids the download."""
if path is not None:
have = http.newest(path)
if have.is_file() and not refresh:
return json.loads(have.read_bytes()), str(have)
if budget is not None and not budget.take():
return None
r = http.get(cl, url, params=params, pacer=pacer, source=SOURCE, ok_404=True)
if r.status_code == 404:
return {"results": []}, ""
try:
payload = r.json()
except ValueError:
payload = None
if not isinstance(payload, dict):
raise http.Refused(
SOURCE, r.status_code, "response is not JSON (a bot or challenge page?)"
)
where = path
if path is not None:
where = http.save(path, r.content)
os.utime(where)
http.manifest(where, str(r.request.url))
return payload, str(where or "")
def resolve(
cl: httpx.Client,
entity: EntityRow,
seed_ids: Sequence[str],
*,
pacer: Any,
cache: Path | None = None,
refresh: bool = False,
budget: Budget | None = None,
) -> tuple[list[FecCommitteeRow], list[dict[str, str]]]:
"""Curated ids win (no name search). Otherwise each alias is searched
and a hit is stored only when its name equals an alias; every other hit
is a candidate. Returns ``(rows, candidates)``."""
ids = [i for i in seed_ids if i]
if ids:
rows: list[FecCommitteeRow] = []
for cid in ids:
path = None if cache is None else cache / "committee" / f"{cid}.json"
got = _page(
cl,
f"{FEC_API}committee/{cid}/",
None,
pacer=pacer,
path=path,
refresh=refresh,
budget=budget,
)
if got is None:
break
payload, src = got
for rec in payload.get("results") or ():
rows.append(
to_committee(
rec, entity_key=entity.key, basis="curated", source_file=src
)
)
return rows, []
aliases = search_names(entity.name, entity.aliases)
found: dict[str, FecCommitteeRow] = {}
pending: list[dict[str, str]] = []
for alias in aliases:
path = (
None if cache is None else cache / "committees" / f"{http.slug(alias)}.json"
)
got = _page(
cl,
f"{FEC_API}committees/",
{"q": alias, "per_page": 20},
pacer=pacer,
path=path,
refresh=refresh,
budget=budget,
)
if got is None:
break
payload, src = got
for rec in payload.get("results") or ():
cid = _text(rec.get("committee_id"))
name = _text(rec.get("name"))
if not cid or cid in found:
continue
if alias_match(name, aliases, prefix=False):
found[cid] = to_committee(
rec, entity_key=entity.key, basis="exact-name", source_file=src
)
else:
pending.append(
{
"committee_id": cid,
"name": name,
"entity_key": entity.key,
"query": alias,
}
)
matched = set(found)
candidates = [c for c in pending if c["committee_id"] not in matched]
return list(found.values()), candidates
def totals(
cl: httpx.Client,
cid: str,
cycle: int,
*,
pacer: Any,
cache: Path | None = None,
refresh: bool = False,
budget: Budget | None = None,
) -> FecTotalsRow | None:
"""``/committee/{cid}/totals/?cycle=``, or ``None`` when that cycle has
no totals (or the budget stopped the download)."""
path = None if cache is None else cache / "totals" / cid / f"{cycle}.json"
got = _page(
cl,
f"{FEC_API}committee/{cid}/totals/",
{"cycle": cycle},
pacer=pacer,
path=path,
refresh=refresh,
budget=budget,
)
if got is None:
return None
payload, src = got
results = payload.get("results") or ()
if not results:
return None
return to_totals(results[0], source_file=src)
def contributions(
cl: httpx.Client,
cid: str,
cycle: int,
*,
pacer: Any,
cache: Path | None = None,
refresh: bool = False,
budget: Budget | None = None,
) -> list[FecContributionRow]:
"""Every Schedule B row for *cid* in *cycle*, keyset-paged on
``pagination.last_indexes``. Memo rows are kept."""
base = {
"committee_id": cid,
"two_year_transaction_period": cycle,
"per_page": PER_PAGE,
"sort": "-disbursement_date",
}
extra: dict[str, Any] = {}
seen: set[str] = set()
out: list[FecContributionRow] = []
n = 0
while True:
n += 1
path = (
None
if cache is None
else cache / "schedule_b" / cid / str(cycle) / f"{n}.json"
)
got = _page(
cl,
f"{FEC_API}schedules/schedule_b/",
{**base, **extra},
pacer=pacer,
path=path,
refresh=refresh,
budget=budget,
)
if got is None:
break
payload, src = got
recs = payload.get("results") or ()
for rec in recs:
out.append(to_contribution(rec, source_file=src, cycle=cycle))
last = (payload.get("pagination") or {}).get("last_indexes") or {}
idx = _text(last.get("last_index"))
if not recs or not idx or idx in seen:
break
seen.add(idx)
extra = {
"last_index": last.get("last_index"),
"last_disbursement_date": last.get("last_disbursement_date") or "",
}
return out
def contributions_by_cycle(con: Any, entity_key: str) -> dict[int, float]:
"""Schedule B amounts by cycle for *entity_key*. A memo row
(``memo_code`` ``X``) is not summed with the disbursement it restates."""
ensure_tables(con)
got = con.execute(
"SELECT c.cycle, coalesce(sum(c.amount), 0) "
"FROM market.fec_contribution c "
"JOIN market.fec_committee m ON m.committee_id = c.committee_id "
"WHERE m.entity_key = ? AND c.memo_code <> 'X' "
"GROUP BY c.cycle ORDER BY c.cycle",
[entity_key],
).fetchall()
return {int(cycle): float(total) for cycle, total in got}

View File

@@ -66,6 +66,25 @@ CREATE TABLE IF NOT EXISTS market.lda_filing (
);
CREATE TABLE IF NOT EXISTS market.lda_issue (
filing_uuid VARCHAR, general_issue_code VARCHAR, description VARCHAR
);
CREATE TABLE IF NOT EXISTS market.fec_committee (
committee_id VARCHAR, name VARCHAR, committee_type VARCHAR,
designation VARCHAR, entity_key VARCHAR, basis VARCHAR, cite_url VARCHAR,
first_file_date DATE, last_file_date DATE, source_file VARCHAR
);
CREATE TABLE IF NOT EXISTS market.fec_totals (
committee_id VARCHAR, cycle INTEGER, receipts DOUBLE,
disbursements DOUBLE, individual_contributions DOUBLE,
fed_candidate_committee_contributions DOUBLE,
independent_expenditures DOUBLE, coverage_start DATE,
coverage_end DATE, source_file VARCHAR
);
CREATE TABLE IF NOT EXISTS market.fec_contribution (
sub_id VARCHAR, committee_id VARCHAR, cycle INTEGER, day DATE,
amount DOUBLE, recipient_committee_id VARCHAR, recipient_name VARCHAR,
candidate_id VARCHAR, candidate_name VARCHAR, candidate_office VARCHAR,
candidate_state VARCHAR, candidate_district VARCHAR, memo_code VARCHAR,
source_file VARCHAR
)
"""
@@ -265,6 +284,60 @@ class LdaIssueRow:
description: str
@dataclass(frozen=True)
class FecCommitteeRow:
"""One FEC committee tied to an entity. ``basis`` is ``curated`` (a seed
id) or ``exact-name`` (the normalised name equals an alias)."""
committee_id: str
name: str
committee_type: str
designation: str
entity_key: str
basis: str
cite_url: str
first_file_date: date | None
last_file_date: date | None
source_file: str
@dataclass(frozen=True)
class FecTotalsRow:
"""One committee's FEC totals for one two-year cycle."""
committee_id: str
cycle: int
receipts: float
disbursements: float
individual_contributions: float
fed_candidate_committee_contributions: float
independent_expenditures: float
coverage_start: date | None
coverage_end: date | None
source_file: str
@dataclass(frozen=True)
class FecContributionRow:
"""One Schedule B disbursement, stored as filed. ``memo_code`` ``X``
restates another row and is left out of cycle sums."""
sub_id: str
committee_id: str
cycle: int
day: date | None
amount: float
recipient_committee_id: str
recipient_name: str
candidate_id: str
candidate_name: str
candidate_office: str
candidate_state: str
candidate_district: str
memo_code: str
source_file: str
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
a statement per row, far too slow for filing lists)."""
@@ -610,6 +683,96 @@ def read_lda_issues(con: Any) -> list[LdaIssueRow]:
)
# ── fec ──────────────────────────────────────────────────────────────
def write_fec_committees(con: Any, rows: Sequence[FecCommitteeRow]) -> int:
"""Replace each committee by ``committee_id``."""
ensure_tables(con)
if not rows:
return 0
for chunk in _chunks(sorted({r.committee_id for r in rows})):
con.execute(
"DELETE FROM market.fec_committee WHERE committee_id IN "
f"({','.join('?' * len(chunk))})",
list(chunk),
)
return _insert(con, "market.fec_committee", rows)
def read_fec_committees(
con: Any, entity_key: str | None = None
) -> list[FecCommitteeRow]:
rows = _select(
con,
FecCommitteeRow,
"market.fec_committee",
"entity_key = ?" if entity_key is not None else "",
[entity_key] if entity_key is not None else [],
)
return sorted((FecCommitteeRow(*r) for r in rows), key=lambda r: r.committee_id)
def write_fec_totals(con: Any, rows: Sequence[FecTotalsRow]) -> int:
"""Replace the totals of each ``(committee_id, cycle)`` in *rows*."""
ensure_tables(con)
if not rows:
return 0
keys = sorted({(r.committee_id, r.cycle) for r in rows})
for part in _chunks(keys, 400):
con.execute(
"DELETE FROM market.fec_totals WHERE (committee_id, cycle) IN "
"(SELECT * FROM (VALUES " + ", ".join(["(?, ?)"] * len(part)) + "))",
[v for k in part for v in k],
)
return _insert(con, "market.fec_totals", rows)
def read_fec_totals(con: Any, committee_id: str | None = None) -> list[FecTotalsRow]:
rows = _select(
con,
FecTotalsRow,
"market.fec_totals",
"committee_id = ?" if committee_id is not None else "",
[committee_id] if committee_id is not None else [],
)
return sorted(
(FecTotalsRow(*r) for r in rows), key=lambda r: (r.committee_id, r.cycle)
)
def write_fec_contributions(con: Any, rows: Sequence[FecContributionRow]) -> int:
"""Replace every Schedule B row of each ``(committee_id, cycle)`` in
*rows* (a rerun drops disbursements the source no longer returns)."""
ensure_tables(con)
if not rows:
return 0
keys = sorted({(r.committee_id, r.cycle) for r in rows})
for part in _chunks(keys, 400):
con.execute(
"DELETE FROM market.fec_contribution WHERE (committee_id, cycle) IN "
"(SELECT * FROM (VALUES " + ", ".join(["(?, ?)"] * len(part)) + "))",
[v for k in part for v in k],
)
return _insert(con, "market.fec_contribution", rows)
def read_fec_contributions(
con: Any, committee_id: str | None = None
) -> list[FecContributionRow]:
rows = _select(
con,
FecContributionRow,
"market.fec_contribution",
"committee_id = ?" if committee_id is not None else "",
[committee_id] if committee_id is not None else [],
)
return sorted(
(FecContributionRow(*r) for r in rows),
key=lambda r: (r.committee_id, r.cycle, r.sub_id),
)
# ── districts ────────────────────────────────────────────────────────

View File

@@ -1912,3 +1912,197 @@ def test_lda_cli_tracks_seed_companies_only(lda):
assert res.exit_code == 0, res.output
assert {r.url.params["client_name"] for r in seen} >= {seed.name}
assert "0 filings" in res.output
# ── fec ──────────────────────────────────────────────────────────────
FEC_SECRET = "fec-cli-key-do-not-leak"
def _fec_entities(c):
T.write_entities(
c,
[
T.EntityRow(
"AAFP",
"American Academy of Family Physicians",
"organization",
aliases="AMERICAN ACADEMY OF FAMILY PHYSICIANS",
),
T.EntityRow("AMA", "AMA", "organization", aliases="AMA"),
T.EntityRow(
"PRIMARY-CARE-SEAT",
"Primary care rotating-seat societies",
"organization",
aliases="PRIMARY CARE ROTATING SEAT",
pseudo=True,
),
],
)
@pytest.fixture
def fec(env, monkeypatch):
from market import fec as fec_api
c, store, published, hoard = env
_fec_entities(c)
seen: list[httpx.Request] = []
page1 = json.loads((FX / "fec_schedule_b_page1.json").read_text())
totals = json.loads((FX / "fec_totals_C00411553_2024.json").read_text())
committee = json.loads((FX / "fec_committees_aafp.json").read_text())
monkeypatch.setattr(fec_api, "new_pacer", lambda: _Pacer())
state = {"status": 200}
def handler(request):
seen.append(request)
if state["status"] != 200:
return httpx.Response(state["status"], text="no")
path = request.url.path
if path.endswith("/totals/"):
return httpx.Response(200, json=totals)
if path.endswith("/schedule_b/"):
if "last_index" in request.url.params:
return httpx.Response(200, json={"results": [], "pagination": {}})
return httpx.Response(200, json=page1)
if path.startswith("/v1/committees"):
return httpx.Response(
200,
json={
"results": [
{
"committee_id": "C00999999",
"name": "AMALGAMATED TRANSIT UNION PAC",
"committee_type": "Q",
"designation": "U",
}
]
},
)
return httpx.Response(200, json=committee)
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(handler),
headers={"X-Api-Key": key} if key else {},
),
)
monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET)
return c, store, published, hoard, seen, state
def test_fec_cli_write_from_cache(fec):
c, _, published, hoard, seen, state = fec
T.write_gap(c, "fec:fec", "old")
dry = runner.invoke(app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024"])
assert dry.exit_code == 0, dry.output
assert "dry run: pass --write" in dry.output
assert "1 committees (1 curated, 0 exact-name)" in dry.output
assert _count(c, "market.fec_committee") == 0
assert published == []
assert seen and all("/committees" not in r.url.path for r in seen)
assert all(FEC_SECRET not in str(r.url) for r in seen)
n = len(seen)
state["status"] = 500
res = runner.invoke(
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
)
assert res.exit_code == 0, res.output
assert len(seen) == n # the hoard answered; the 500 was never requested
assert "stored: 1 committees, 1 totals, 3 schedule B rows" in res.output
(committee,) = T.read_fec_committees(c)
assert committee.basis == "curated" and committee.committee_id == "C00411553"
assert committee.source_file.endswith("fec/committee/C00411553.json")
(totals,) = T.read_fec_totals(c)
assert totals.receipts == 513798.01
assert totals.fed_candidate_committee_contributions == 683300.0
names = {r.recipient_name for r in T.read_fec_contributions(c)}
assert "JANE Q PUBLIC" in names and "MEMO OF THE 100" in names
from market.fec import contributions_by_cycle
assert contributions_by_cycle(c, "AAFP") == {2024: 125.0}
assert T.read_gaps(c) == []
assert published == [True]
assert {
(t, n)
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
} == {
("market.fec_committee", 1),
("market.fec_totals", 1),
("market.fec_contribution", 3),
}
manifest = (hoard / "MANIFEST.jsonl").read_text()
assert FEC_SECRET not in manifest
assert (hoard / "fec" / "schedule_b" / "C00411553" / "2024" / "1.json").is_file()
def test_fec_cli_no_write_prints_candidates(fec):
c, _, published, _, seen, _ = fec
res = runner.invoke(app, ["market", "fec", "--entity", "AMA", "--cycles", "2024"])
assert res.exit_code == 0, res.output
assert "candidate: AMA C00999999 AMALGAMATED TRANSIT UNION PAC" in res.output
assert "0 committees" in res.output and "1 candidates" in res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.fec_committee") == 0
assert _count(c, "market.fec_contribution") == 0
assert published == []
assert seen[0].url.params["q"] == "AMA"
assert all(
"/committee/" not in r.url.path or r.url.path.startswith("/v1/committees")
for r in seen
)
default = runner.invoke(app, ["market", "fec", "--entity", "AMA"])
assert default.exit_code == 0, default.output
assert "1 candidates" in default.output
bad = runner.invoke(app, ["market", "fec", "--entity", "NOPE", "--cycles", "2024"])
assert bad.exit_code == 2 and "unknown entity: NOPE" in bad.output
cycles = runner.invoke(app, ["market", "fec", "--cycles", "nope"])
assert cycles.exit_code == 2 and "not a list of years" in cycles.output
def test_fec_cli_refusal_prints_gap(fec, monkeypatch):
c, _, published, _, seen, state = fec
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
missing = runner.invoke(
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
)
assert missing.exit_code == 0, missing.output
assert "gap: fec:fec:" in missing.output
assert "FEC_API_KEY" in missing.output
assert seen == []
assert [g.source for g in T.read_gaps(c)] == ["fec:fec"]
assert published == [True]
monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET)
state["status"] = 403
res = runner.invoke(
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: fec:fec: fec: HTTP 403" in res.output
assert FEC_SECRET not in res.output
assert _count(c, "market.fec_committee") == 0
assert len(seen) == 1
assert all(FEC_SECRET not in str(r.url) for r in seen)
assert seen[0].headers["x-api-key"] == FEC_SECRET
def test_fec_cli_limit_keeps_the_gap_and_empty_registry(fec, monkeypatch):
c, _, _, _, seen, _ = fec
T.write_gap(c, "fec:fec", "old")
res = runner.invoke(
app,
["market", "fec", "--cycles", "2024,2022", "--limit", "1", "--write"],
)
assert res.exit_code == 0, res.output
assert "stopped: request limit" in res.output
assert [g.note for g in T.read_gaps(c)] == ["old"]
assert _count(c, "market.fec_committee") == 1
assert _count(c, "market.fec_totals") == 0
assert len(seen) == 1
monkeypatch.setattr(mk, "_lda_tracked", lambda con: [])
empty = runner.invoke(app, ["market", "fec", "--cycles", "2024"])
assert empty.exit_code == 1 and "no tracked entities" in empty.output
blank = runner.invoke(app, ["market", "fec", "--cycles", ","])
assert blank.exit_code == 2 and "--cycles is empty" in blank.output

View File

@@ -0,0 +1,13 @@
{
"api_version": "1.0",
"results": [
{
"committee_id": "C00411553",
"name": "AMERICAN ACADEMY OF FAMILY PHYSICIANS POLITICAL ACTION COMMITTEE (FAMMEDPAC)",
"committee_type": "Q",
"designation": "U",
"first_file_date": "2003-07-15",
"last_file_date": "2024-10-15"
}
]
}

View File

@@ -0,0 +1,59 @@
{
"pagination": {
"count": 3,
"page": 1,
"pages": 2,
"per_page": 100,
"last_indexes": {
"last_index": 123,
"last_disbursement_date": "2024-03-01"
}
},
"results": [
{
"sub_id": "4010120240001",
"committee_id": "C00411553",
"two_year_transaction_period": 2024,
"disbursement_date": "2024-06-15",
"disbursement_amount": 100.0,
"recipient_committee_id": "C00111111",
"recipient_name": "FRIENDS OF SOMEONE",
"candidate_id": "H0KS00001",
"candidate_name": "SOMEONE, A",
"candidate_office": "H",
"candidate_office_state": "KS",
"candidate_office_district": "02",
"memo_code": null
},
{
"sub_id": "4010120240002",
"committee_id": "C00411553",
"two_year_transaction_period": 2024,
"disbursement_date": "2024-05-01",
"disbursement_amount": 25.0,
"recipient_committee_id": null,
"recipient_name": "JANE Q PUBLIC",
"candidate_id": null,
"candidate_name": null,
"candidate_office": null,
"candidate_office_state": null,
"candidate_office_district": null,
"memo_code": null
},
{
"sub_id": "4010120240003",
"committee_id": "C00411553",
"two_year_transaction_period": 2024,
"disbursement_date": "2024-04-01",
"disbursement_amount": 100.0,
"recipient_committee_id": null,
"recipient_name": "MEMO OF THE 100",
"candidate_id": null,
"candidate_name": null,
"candidate_office": null,
"candidate_office_state": null,
"candidate_office_district": null,
"memo_code": "X"
}
]
}

View File

@@ -0,0 +1,16 @@
{
"api_version": "1.0",
"results": [
{
"committee_id": "C00411553",
"cycle": 2024,
"receipts": 513798.01,
"disbursements": 696685.91,
"individual_contributions": 480000.0,
"fed_candidate_committee_contributions": 683300.0,
"independent_expenditures": 0.0,
"coverage_start_date": "2023-01-01",
"coverage_end_date": "2024-12-31"
}
]
}

399
tests/market/test_fec.py Normal file
View File

@@ -0,0 +1,399 @@
"""market.fec: curated committee ids, exact-name fallback, totals, Schedule B."""
from __future__ import annotations
import json
from datetime import date
from pathlib import Path
import httpx
import pytest
from market import fec as F
from market import http
from market import tables as T
FX = Path(__file__).parent / "fixtures"
SECRET = "fec-test-key-do-not-leak"
PAC = "AMERICAN ACADEMY OF FAMILY PHYSICIANS POLITICAL ACTION COMMITTEE (FAMMEDPAC)"
PAC_ALIAS = "AMERICAN ACADEMY OF FAMILY PHYSICIANS POLITICAL ACTION COMMITTEE FAMMEDPAC"
SHORT = "AMERICAN ACADEMY OF FAMILY PHYSICIANS"
def _load(name: str) -> dict:
return json.loads((FX / name).read_text())
@pytest.fixture
def con():
c = __import__("duckdb").connect(":memory:")
T.ensure_tables(c)
yield c
c.close()
class Pacer:
def __init__(self):
self.n = 0
def wait(self):
self.n += 1
def entity(key, name, aliases=""):
return T.EntityRow(key, name, "organization", aliases=aliases or name)
def _client(handler, *, key: str | None = None) -> httpx.Client:
headers = {"X-Api-Key": key} if key else {}
return httpx.Client(transport=httpx.MockTransport(handler), headers=headers)
def test_committee_curated_ids_win():
assert F.FEC_API == "https://api.open.fec.gov/v1/"
assert F.new_pacer().min_interval == 3.7
seen = []
body = _load("fec_committees_aafp.json")
def handler(request):
seen.append(request)
return httpx.Response(200, json=body)
cl = _client(handler)
rows, cands = F.resolve(cl, entity("AAFP", SHORT), ["C00411553"], pacer=Pacer())
assert cands == []
assert [(r.committee_id, r.basis, r.entity_key) for r in rows] == [
("C00411553", "curated", "AAFP")
]
assert rows[0].cite_url == "https://www.fec.gov/data/committee/C00411553/"
assert [r.url.host for r in seen] == ["api.open.fec.gov"]
assert seen[0].url.path == "/v1/committee/C00411553/"
assert all("/committees" not in r.url.path for r in seen)
assert "api_key" not in str(seen[0].url)
def test_committee_exact_name_fallback_cited():
body = {"results": _load("fec_committees_aafp.json")["results"]}
def handler(request):
return httpx.Response(200, json=body)
cl = _client(handler)
rows, cands = F.resolve(cl, entity("AAFP", SHORT), [], pacer=Pacer())
assert rows == []
assert cands[0]["name"] == PAC
assert cands[0]["committee_id"] == "C00411553"
rows, cands = F.resolve(cl, entity("AAFP", PAC_ALIAS), [], pacer=Pacer())
assert cands == []
assert rows[0].basis == "exact-name"
assert rows[0].cite_url == "https://www.fec.gov/data/committee/C00411553/"
assert rows[0].name == PAC
def test_short_alias_never_matches_inside_names():
name = "AMALGAMATED TRANSIT UNION PAC"
def handler(request):
assert request.url.params["q"] == "AMA"
return httpx.Response(
200,
json={"results": [{"committee_id": "C00999999", "name": name}]},
)
rows, cands = F.resolve(_client(handler), entity("AMA", "AMA"), [], pacer=Pacer())
assert rows == []
assert [(c["committee_id"], c["name"]) for c in cands] == [("C00999999", name)]
def test_totals_row():
body = _load("fec_totals_C00411553_2024.json")
def handler(request):
assert request.url.path == "/v1/committee/C00411553/totals/"
assert request.url.params["cycle"] == "2024"
return httpx.Response(200, json=body)
row = F.totals(_client(handler), "C00411553", 2024, pacer=Pacer())
assert row is not None
assert row.receipts == 513798.01
assert row.disbursements == 696685.91
assert row.fed_candidate_committee_contributions == 683300.0
assert (row.coverage_start, row.coverage_end) == (
date(2023, 1, 1),
date(2024, 12, 31),
)
def test_schedule_b_keyset_paging():
page1 = _load("fec_schedule_b_page1.json")
page2 = {
"results": [
{
"sub_id": "4010120240004",
"committee_id": "C00411553",
"two_year_transaction_period": 2024,
"disbursement_date": "2024-02-01",
"disbursement_amount": 1.0,
"recipient_name": "SECOND PAGE",
"memo_code": None,
}
],
"pagination": {},
}
seen = []
def handler(request):
seen.append(request)
if "last_index" in request.url.params:
return httpx.Response(200, json=page2)
return httpx.Response(200, json=page1)
rows = F.contributions(_client(handler), "C00411553", 2024, pacer=Pacer())
assert len(seen) == 2
assert seen[0].url.path == "/v1/schedules/schedule_b/"
assert seen[0].url.params["per_page"] == "100"
assert seen[0].url.params["sort"] == "-disbursement_date"
assert seen[0].url.params["two_year_transaction_period"] == "2024"
assert "last_index" not in seen[0].url.params
assert seen[1].url.params["last_index"] == "123"
assert seen[1].url.params["last_disbursement_date"] == "2024-03-01"
assert [r.sub_id for r in rows] == [
"4010120240001",
"4010120240002",
"4010120240003",
"4010120240004",
]
def test_memo_rows_excluded_from_totals(con):
T.write_fec_committees(
con,
[
T.FecCommitteeRow(
"C00411553",
PAC,
"Q",
"U",
"AAFP",
"curated",
"https://www.fec.gov/data/committee/C00411553/",
None,
None,
"",
)
],
)
rows = [
T.FecContributionRow(
"a",
"C00411553",
2024,
date(2024, 6, 1),
100.0,
"C1",
"PAC",
"",
"",
"",
"",
"",
"",
"",
),
T.FecContributionRow(
"b",
"C00411553",
2024,
date(2024, 5, 1),
25.0,
"",
"JANE Q PUBLIC",
"",
"",
"",
"",
"",
"",
"",
),
T.FecContributionRow(
"c",
"C00411553",
2024,
date(2024, 4, 1),
100.0,
"",
"MEMO",
"",
"",
"",
"",
"",
"X",
"",
),
]
T.write_fec_contributions(con, rows)
assert F.contributions_by_cycle(con, "AAFP") == {2024: 125.0}
def test_schedule_b_rows_stored_as_filed(con):
page = _load("fec_schedule_b_page1.json")
def handler(request):
body = dict(page)
body["pagination"] = {}
return httpx.Response(200, json=body)
rows = F.contributions(_client(handler), "C00411553", 2024, pacer=Pacer())
person = next(r for r in rows if r.recipient_name == "JANE Q PUBLIC")
assert person.recipient_committee_id == ""
assert person.amount == 25.0
memo = next(r for r in rows if r.memo_code == "X")
assert memo.recipient_name == "MEMO OF THE 100"
T.write_fec_contributions(con, rows)
stored = T.read_fec_contributions(con)
assert [r.recipient_name for r in stored if r.sub_id == "4010120240002"] == [
"JANE Q PUBLIC"
]
assert any(r.memo_code == "X" for r in stored)
def test_fec_error_message_has_no_key():
seen = []
def handler(request):
seen.append(request)
return httpx.Response(403, text="no")
cl = _client(handler, key=SECRET)
with pytest.raises(http.Refused) as exc:
F.totals(cl, "C00411553", 2024, pacer=Pacer())
assert exc.value.source == "fec"
assert SECRET not in str(exc.value)
assert SECRET not in str(seen[0].url)
assert "api_key" not in str(seen[0].url)
assert seen[0].headers["x-api-key"] == SECRET
def bot(request):
return httpx.Response(200, text="<html>Just a moment...</html>")
with pytest.raises(http.Refused) as exc2:
F.totals(_client(bot, key=SECRET), "C00411553", 2024, pacer=Pacer())
assert "not JSON" in str(exc2.value)
assert SECRET not in str(exc2.value)
def test_budget_edges_and_blank_fields(monkeypatch):
company = type("C", (), {"cik": "0000000001", "fec_committee_ids": ("C1",)})()
org = type(
"O", (), {"key": "AAFP", "fec_committee_ids": ("C2",), "pseudo": False}
)()
pseudo = type(
"O", (), {"key": "SEAT", "fec_committee_ids": ("C9",), "pseudo": True}
)()
monkeypatch.setattr(F, "load_seed", lambda: ([company], [org, pseudo]))
assert F.committee_ids() == {"AAFP": ("C2",), "0000000001": ("C1",)}
limited = F.Budget(1)
assert limited.take() is True
assert limited.take() is False and limited.why == "request limit"
stopped = F.Budget(0, stop=lambda: True)
assert stopped.take() is False and stopped.why == "time limit"
assert F._date(None) is None and F._date("") is None
assert F._money("") == 0.0
row = F.to_contribution(
{
"sub_id": "z",
"amount": "",
"candidate_state": "KS",
"candidate_district": "01",
},
source_file="s",
cycle=2024,
)
assert (row.cycle, row.amount, row.day, row.candidate_state) == (
2024,
0.0,
None,
"KS",
)
seen = []
def handler(request):
seen.append(request.url.path)
return httpx.Response(
200,
json={
"results": [
{"committee_id": "", "name": "NO ID"},
{"committee_id": "C1", "name": PAC},
{"committee_id": "C1", "name": PAC},
]
},
)
cl = _client(handler)
org_row = entity("AAFP", SHORT, aliases=f"{SHORT}|{PAC_ALIAS}")
rows, cands = F.resolve(cl, org_row, [], pacer=Pacer(), budget=F.Budget(1))
assert [r.committee_id for r in rows] == ["C1"]
assert rows[0].basis == "exact-name"
assert cands == []
assert seen == ["/v1/committees/"] # the second alias stops on the budget
def curated(request):
return httpx.Response(200, json=_load("fec_committees_aafp.json"))
one, _ = F.resolve(
_client(curated),
entity("AAFP", SHORT),
["C00411553", "C00000001"],
pacer=Pacer(),
budget=F.Budget(1),
)
assert [r.committee_id for r in one] == ["C00411553"]
spent = F.Budget(1)
spent.take()
def boom(request):
raise AssertionError("budget should have stopped the download")
assert (
F.totals(_client(boom), "C00411553", 2024, pacer=Pacer(), budget=spent) is None
)
assert (
F.contributions(_client(boom), "C00411553", 2024, pacer=Pacer(), budget=spent)
== []
)
def empty(request):
return httpx.Response(200, json={"results": []})
assert F.totals(_client(empty), "C00411553", 2022, pacer=Pacer()) is None
def test_fec_hoard_reuses_and_skips_a_missing_committee(tmp_path, monkeypatch):
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
calls = []
body = _load("fec_committees_aafp.json")
def handler(request):
calls.append(request.url.path)
if "C00000000" in request.url.path:
return httpx.Response(404, json={"results": []})
return httpx.Response(200, json=body)
cl = _client(handler)
cache = tmp_path / "fec"
kw = dict(pacer=Pacer(), cache=cache)
first, _ = F.resolve(cl, entity("AAFP", SHORT), ["C00411553"], **kw)
again, _ = F.resolve(cl, entity("AAFP", SHORT), ["C00411553"], **kw)
assert calls == ["/v1/committee/C00411553/"]
assert first[0].source_file.endswith("fec/committee/C00411553.json")
assert again[0].committee_id == "C00411553"
empty, cands = F.resolve(cl, entity("AAFP", SHORT), ["C00000000"], **kw)
assert empty == [] and cands == []

View File

@@ -13,6 +13,7 @@ MODULES = [
"market.edgar",
"market.tiingo",
"market.lda",
"market.fec",
]
_PROBE = """

View File

@@ -43,6 +43,9 @@ def test_ensure_tables_idempotent():
"series",
"lda_filing",
"lda_issue",
"fec_committee",
"fec_totals",
"fec_contribution",
}
@@ -141,6 +144,46 @@ LDA_FILING = T.LdaFilingRow(
"f.json",
)
LDA_ISSUE = T.LdaIssueRow(LDA_FILING.filing_uuid, "BUD", "No Bill No.")
FEC_COMMITTEE = T.FecCommitteeRow(
"C00411553",
"FAMMEDPAC",
"Q",
"U",
"AAFP",
"curated",
"https://www.fec.gov/data/committee/C00411553/",
date(2003, 7, 15),
date(2024, 10, 15),
"committee/C00411553.json",
)
FEC_TOTALS = T.FecTotalsRow(
"C00411553",
2024,
513798.01,
696685.91,
480000.0,
683300.0,
0.0,
date(2023, 1, 1),
date(2024, 12, 31),
"totals/C00411553/2024.json",
)
FEC_CONTRIBUTION = T.FecContributionRow(
"4010120240002",
"C00411553",
2024,
date(2024, 5, 1),
25.0,
"",
"JANE Q PUBLIC",
"",
"",
"",
"",
"",
"",
"schedule_b/C00411553/2024/1.json",
)
SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY")
ROUNDTRIP = {
@@ -154,6 +197,21 @@ ROUNDTRIP = {
lambda c, r: T.write_lda_issues(c, [r]),
lambda c: T.read_lda_issues(c),
),
"fec_committee": (
FEC_COMMITTEE,
lambda c, r: T.write_fec_committees(c, [r]),
lambda c: T.read_fec_committees(c),
),
"fec_totals": (
FEC_TOTALS,
lambda c, r: T.write_fec_totals(c, [r]),
lambda c: T.read_fec_totals(c),
),
"fec_contribution": (
FEC_CONTRIBUTION,
lambda c, r: T.write_fec_contributions(c, [r]),
lambda c: T.read_fec_contributions(c),
),
"entity": (
ENTITY,
lambda c, r: T.write_entities(c, [r]),
@@ -212,6 +270,9 @@ def test_write_empty_is_noop(con):
assert T.write_securities(con, "1", []) == 0
assert T.write_lda_filings(con, []) == 0
assert T.write_lda_issues(con, []) == 0
assert T.write_fec_committees(con, []) == 0
assert T.write_fec_totals(con, []) == 0
assert T.write_fec_contributions(con, []) == 0
def test_lda_readers_filter_by_entity(con):