feat(market): FEC PAC committees (curated ids, exact-name fallback), cycle totals and Schedule B
A curated committee id is fetched directly. A search hit is stored only when its normalised name equals an alias, and every other hit stays a candidate. Schedule B rows are kept as filed, with memo rows left out of the cycle sum.
This commit is contained in:
@@ -21,6 +21,7 @@ import httpx
|
||||
import typer
|
||||
|
||||
from market import edgar, http, tiingo
|
||||
from market import fec as fec_api
|
||||
from market import lda as lda_api
|
||||
from market.seed import (
|
||||
SEAT_LABEL_KEYS,
|
||||
@@ -32,6 +33,9 @@ from market.seed import (
|
||||
)
|
||||
from market.tables import (
|
||||
EntityRow,
|
||||
FecCommitteeRow,
|
||||
FecContributionRow,
|
||||
FecTotalsRow,
|
||||
FilingRow,
|
||||
HitRow,
|
||||
LdaFilingRow,
|
||||
@@ -50,6 +54,9 @@ from market.tables import (
|
||||
retire_links,
|
||||
retire_organizations,
|
||||
write_entities,
|
||||
write_fec_committees,
|
||||
write_fec_contributions,
|
||||
write_fec_totals,
|
||||
write_filings,
|
||||
write_gap,
|
||||
write_hits,
|
||||
@@ -67,6 +74,7 @@ ENTITIES_GAP = "sec-edgar:entities"
|
||||
PRICES_GAP = "tiingo:prices"
|
||||
FTS_GAP = "sec-edgar:fts"
|
||||
LDA_GAP = "lda:lda"
|
||||
FEC_GAP = "fec:fec"
|
||||
|
||||
|
||||
# ── indirections the tests monkeypatch ───────────────────────────────
|
||||
@@ -1258,6 +1266,152 @@ def lda(
|
||||
typer.echo(f"stored: {n} filings, {len(issues)} issue rows")
|
||||
|
||||
|
||||
def _parse_cycles(text: str) -> tuple[int, ...]:
|
||||
if not text.strip():
|
||||
return fec_api.CYCLES
|
||||
try:
|
||||
return tuple(int(part.strip()) for part in text.split(",") if part.strip())
|
||||
except ValueError:
|
||||
typer.echo(f"--cycles {text} is not a list of years")
|
||||
raise typer.Exit(2) from None
|
||||
|
||||
|
||||
@app.command()
|
||||
def fec(
|
||||
entity: list[str] = typer.Option(
|
||||
[], "--entity", help="A tracked entity key (repeatable; all when none given)."
|
||||
),
|
||||
cycles: str = typer.Option(
|
||||
"",
|
||||
"--cycles",
|
||||
help="Comma-separated cycles (default 2008 through 2026, even years).",
|
||||
),
|
||||
limit: int = typer.Option(
|
||||
0, "--limit", help="At most N network requests (hoarded pages are free)."
|
||||
),
|
||||
write: bool = typer.Option(
|
||||
False, "--write", help="Store committees, totals and Schedule B."
|
||||
),
|
||||
) -> None:
|
||||
"""FEC committees, cycle totals and Schedule B disbursements
|
||||
(market.fec_committee, market.fec_totals, market.fec_contribution). A
|
||||
curated committee id is fetched directly; otherwise a search hit is kept
|
||||
only when its name equals the entity's name or an alias. Other hits are
|
||||
printed as candidates and not stored. A rerun resumes from the hoard
|
||||
(data/market/fec)."""
|
||||
chosen_cycles = _parse_cycles(cycles)
|
||||
if not chosen_cycles:
|
||||
typer.echo("--cycles is empty")
|
||||
raise typer.Exit(2)
|
||||
key = _require_key("fec", FEC_GAP, write=write)
|
||||
if key is None:
|
||||
return
|
||||
con = _read()
|
||||
try:
|
||||
tracked = _lda_tracked(con)
|
||||
finally:
|
||||
con.close()
|
||||
if not tracked:
|
||||
typer.echo("no tracked entities — run `stack market entities --write`")
|
||||
raise typer.Exit(1)
|
||||
by_key = {e.key: e for e in tracked}
|
||||
for k in entity:
|
||||
if k not in by_key:
|
||||
typer.echo(f"unknown entity: {k}")
|
||||
raise typer.Exit(2)
|
||||
chosen = [by_key[k] for k in dict.fromkeys(entity)] or tracked
|
||||
ids = fec_api.committee_ids()
|
||||
pacer = fec_api.new_pacer()
|
||||
budget = fec_api.Budget(limit)
|
||||
root = http.hoard("fec")
|
||||
committees: list[FecCommitteeRow] = []
|
||||
candidates: list[dict[str, str]] = []
|
||||
totals: list[FecTotalsRow] = []
|
||||
contribs: list[FecContributionRow] = []
|
||||
cl = http.client("fec", key)
|
||||
try:
|
||||
for e in chosen:
|
||||
if budget.exhausted:
|
||||
break
|
||||
rows, cands = fec_api.resolve(
|
||||
cl,
|
||||
e,
|
||||
ids.get(e.key, ()),
|
||||
pacer=pacer,
|
||||
cache=root,
|
||||
budget=budget,
|
||||
)
|
||||
committees.extend(rows)
|
||||
candidates.extend(cands)
|
||||
for row in rows:
|
||||
for cycle in chosen_cycles:
|
||||
if budget.exhausted:
|
||||
break
|
||||
got = fec_api.totals(
|
||||
cl,
|
||||
row.committee_id,
|
||||
cycle,
|
||||
pacer=pacer,
|
||||
cache=root,
|
||||
budget=budget,
|
||||
)
|
||||
if got is not None:
|
||||
totals.append(got)
|
||||
contribs.extend(
|
||||
fec_api.contributions(
|
||||
cl,
|
||||
row.committee_id,
|
||||
cycle,
|
||||
pacer=pacer,
|
||||
cache=root,
|
||||
budget=budget,
|
||||
)
|
||||
)
|
||||
except http.Refused as exc:
|
||||
_gap(FEC_GAP, str(exc), write=write)
|
||||
return
|
||||
finally:
|
||||
cl.close()
|
||||
|
||||
curated = sum(c.basis == "curated" for c in committees)
|
||||
exact = sum(c.basis == "exact-name" for c in committees)
|
||||
typer.echo(
|
||||
f"{len(committees)} committees ({curated} curated, {exact} exact-name), "
|
||||
f"{len(candidates)} candidates, {len(totals)} totals, "
|
||||
f"{len(contribs)} schedule B rows; {budget.used} requests"
|
||||
)
|
||||
for cand in candidates:
|
||||
typer.echo(
|
||||
f"candidate: {cand['entity_key']} {cand['committee_id']} {cand['name']}"
|
||||
)
|
||||
if budget.exhausted:
|
||||
typer.echo(f"stopped: {budget.why} (rerun resumes from the hoard)")
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write to store")
|
||||
return
|
||||
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
n_c = write_fec_committees(con, committees)
|
||||
n_t = write_fec_totals(con, totals)
|
||||
n_b = write_fec_contributions(con, contribs)
|
||||
if not budget.exhausted:
|
||||
clear_gap(con, FEC_GAP)
|
||||
src = str(http.hoard(http.MANIFEST))
|
||||
for table, count in (
|
||||
("market.fec_committee", n_c),
|
||||
("market.fec_totals", n_t),
|
||||
("market.fec_contribution", n_b),
|
||||
):
|
||||
log_ingest(
|
||||
con, module="market.fec", table_name=table, rows=count, source_file=src
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: {n_c} committees, {n_t} totals, {n_b} schedule B rows")
|
||||
|
||||
|
||||
@app.command()
|
||||
def gaps() -> None:
|
||||
"""Sources that could not be read (market.source_gap)."""
|
||||
|
||||
391
src/market/fec.py
Normal file
391
src/market/fec.py
Normal file
@@ -0,0 +1,391 @@
|
||||
"""FEC committees, cycle totals and Schedule B disbursements.
|
||||
|
||||
The key (``FEC_API_KEY``, else ``API_DATA_GOV_KEY``) travels only in the
|
||||
``X-Api-Key`` header. A committee is stored from a curated
|
||||
``fec_committee_ids`` entry, or, failing that, when its normalised name
|
||||
equals an alias (``alias_match(prefix=False)``). Every other search hit is
|
||||
a candidate: printed, not stored. Schedule B rows are stored as filed,
|
||||
including an individual recipient's name; a memo row (``memo_code`` ``X``)
|
||||
is stored and left out of the cycle sum.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from market import http
|
||||
from market.seed import alias_match, load_seed, norm_org
|
||||
from market.tables import (
|
||||
EntityRow,
|
||||
FecCommitteeRow,
|
||||
FecContributionRow,
|
||||
FecTotalsRow,
|
||||
ensure_tables,
|
||||
)
|
||||
|
||||
SOURCE = "fec"
|
||||
FEC_API = "https://api.open.fec.gov/v1/"
|
||||
PACER_INTERVAL = 3.7 # the published limit is 1,000 requests an hour
|
||||
CYCLES = tuple(range(2008, 2028, 2))
|
||||
PER_PAGE = 100
|
||||
|
||||
|
||||
def new_pacer() -> http.Pacer:
|
||||
return http.Pacer(PACER_INTERVAL)
|
||||
|
||||
|
||||
def committee_ids() -> dict[str, tuple[str, ...]]:
|
||||
"""Curated FEC committee ids by entity key (an org key, or a company's
|
||||
10-digit CIK). Pseudo seats are skipped."""
|
||||
companies, orgs = load_seed()
|
||||
out: dict[str, tuple[str, ...]] = {}
|
||||
for org in orgs:
|
||||
if org.fec_committee_ids and not org.pseudo:
|
||||
out[org.key] = org.fec_committee_ids
|
||||
for company in companies:
|
||||
if company.fec_committee_ids:
|
||||
out[company.cik] = company.fec_committee_ids
|
||||
return out
|
||||
|
||||
|
||||
def search_names(name: str, aliases: str) -> list[str]:
|
||||
"""The distinct (by normalised form) names to search: the entity's own
|
||||
name first, then its ``|``-joined aliases."""
|
||||
seen: dict[str, str] = {}
|
||||
for n in (name, *aliases.split("|")):
|
||||
k = norm_org(n)
|
||||
if k and k not in seen:
|
||||
seen[k] = n.strip()
|
||||
return list(seen.values())
|
||||
|
||||
|
||||
class Budget:
|
||||
"""At most *limit* network requests (0 = unlimited). A hoarded page
|
||||
costs nothing."""
|
||||
|
||||
def __init__(self, limit: int = 0, stop: Callable[[], bool] | None = None) -> None:
|
||||
self.limit = limit
|
||||
self.stop = stop
|
||||
self.used = 0
|
||||
self.exhausted = False
|
||||
self.why = ""
|
||||
|
||||
def take(self) -> bool:
|
||||
if self.limit and self.used >= self.limit:
|
||||
self.exhausted, self.why = True, "request limit"
|
||||
return False
|
||||
if self.stop is not None and self.stop():
|
||||
self.exhausted, self.why = True, "time limit"
|
||||
return False
|
||||
self.used += 1
|
||||
return True
|
||||
|
||||
|
||||
def _date(value: Any) -> dt.date | None:
|
||||
if not value:
|
||||
return None
|
||||
return dt.date.fromisoformat(str(value)[:10])
|
||||
|
||||
|
||||
def _money(value: Any) -> float:
|
||||
return 0.0 if value in (None, "") else float(value)
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
return "" if value is None else str(value)
|
||||
|
||||
|
||||
def cite_url(committee_id: str) -> str:
|
||||
return f"https://www.fec.gov/data/committee/{committee_id}/"
|
||||
|
||||
|
||||
def to_committee(
|
||||
rec: Mapping[str, Any], *, entity_key: str, basis: str, source_file: str
|
||||
) -> FecCommitteeRow:
|
||||
cid = _text(rec.get("committee_id"))
|
||||
return FecCommitteeRow(
|
||||
committee_id=cid,
|
||||
name=_text(rec.get("name")),
|
||||
committee_type=_text(rec.get("committee_type")),
|
||||
designation=_text(rec.get("designation")),
|
||||
entity_key=entity_key,
|
||||
basis=basis,
|
||||
cite_url=cite_url(cid),
|
||||
first_file_date=_date(rec.get("first_file_date")),
|
||||
last_file_date=_date(rec.get("last_file_date")),
|
||||
source_file=source_file,
|
||||
)
|
||||
|
||||
|
||||
def to_totals(rec: Mapping[str, Any], *, source_file: str) -> FecTotalsRow:
|
||||
return FecTotalsRow(
|
||||
committee_id=_text(rec.get("committee_id")),
|
||||
cycle=int(rec.get("cycle") or 0),
|
||||
receipts=_money(rec.get("receipts")),
|
||||
disbursements=_money(rec.get("disbursements")),
|
||||
individual_contributions=_money(rec.get("individual_contributions")),
|
||||
fed_candidate_committee_contributions=_money(
|
||||
rec.get("fed_candidate_committee_contributions")
|
||||
),
|
||||
independent_expenditures=_money(rec.get("independent_expenditures")),
|
||||
coverage_start=_date(
|
||||
rec.get("coverage_start_date") or rec.get("coverage_start")
|
||||
),
|
||||
coverage_end=_date(rec.get("coverage_end_date") or rec.get("coverage_end")),
|
||||
source_file=source_file,
|
||||
)
|
||||
|
||||
|
||||
def to_contribution(
|
||||
rec: Mapping[str, Any], *, source_file: str, cycle: int | None = None
|
||||
) -> FecContributionRow:
|
||||
raw_cycle = rec.get("two_year_transaction_period")
|
||||
if raw_cycle in (None, ""):
|
||||
raw_cycle = rec.get("cycle", cycle or 0)
|
||||
amount = rec.get("disbursement_amount", rec.get("amount"))
|
||||
return FecContributionRow(
|
||||
sub_id=_text(rec.get("sub_id")),
|
||||
committee_id=_text(rec.get("committee_id")),
|
||||
cycle=int(raw_cycle or 0),
|
||||
day=_date(rec.get("disbursement_date") or rec.get("day")),
|
||||
amount=_money(amount),
|
||||
recipient_committee_id=_text(rec.get("recipient_committee_id")),
|
||||
recipient_name=_text(rec.get("recipient_name")),
|
||||
candidate_id=_text(rec.get("candidate_id")),
|
||||
candidate_name=_text(rec.get("candidate_name")),
|
||||
candidate_office=_text(rec.get("candidate_office")),
|
||||
candidate_state=_text(
|
||||
rec.get("candidate_office_state") or rec.get("candidate_state")
|
||||
),
|
||||
candidate_district=_text(
|
||||
rec.get("candidate_office_district") or rec.get("candidate_district")
|
||||
),
|
||||
memo_code=_text(rec.get("memo_code")),
|
||||
source_file=source_file,
|
||||
)
|
||||
|
||||
|
||||
def _page(
|
||||
cl: httpx.Client,
|
||||
url: str,
|
||||
params: Mapping[str, Any] | None,
|
||||
*,
|
||||
pacer: Any,
|
||||
path: Path | None,
|
||||
refresh: bool,
|
||||
budget: Budget | None,
|
||||
) -> tuple[dict[str, Any], str] | None:
|
||||
"""One document: the hoarded copy when present, else a download (never
|
||||
overwriting). ``None`` when the budget forbids the download."""
|
||||
if path is not None:
|
||||
have = http.newest(path)
|
||||
if have.is_file() and not refresh:
|
||||
return json.loads(have.read_bytes()), str(have)
|
||||
if budget is not None and not budget.take():
|
||||
return None
|
||||
r = http.get(cl, url, params=params, pacer=pacer, source=SOURCE, ok_404=True)
|
||||
if r.status_code == 404:
|
||||
return {"results": []}, ""
|
||||
try:
|
||||
payload = r.json()
|
||||
except ValueError:
|
||||
payload = None
|
||||
if not isinstance(payload, dict):
|
||||
raise http.Refused(
|
||||
SOURCE, r.status_code, "response is not JSON (a bot or challenge page?)"
|
||||
)
|
||||
where = path
|
||||
if path is not None:
|
||||
where = http.save(path, r.content)
|
||||
os.utime(where)
|
||||
http.manifest(where, str(r.request.url))
|
||||
return payload, str(where or "")
|
||||
|
||||
|
||||
def resolve(
|
||||
cl: httpx.Client,
|
||||
entity: EntityRow,
|
||||
seed_ids: Sequence[str],
|
||||
*,
|
||||
pacer: Any,
|
||||
cache: Path | None = None,
|
||||
refresh: bool = False,
|
||||
budget: Budget | None = None,
|
||||
) -> tuple[list[FecCommitteeRow], list[dict[str, str]]]:
|
||||
"""Curated ids win (no name search). Otherwise each alias is searched
|
||||
and a hit is stored only when its name equals an alias; every other hit
|
||||
is a candidate. Returns ``(rows, candidates)``."""
|
||||
ids = [i for i in seed_ids if i]
|
||||
if ids:
|
||||
rows: list[FecCommitteeRow] = []
|
||||
for cid in ids:
|
||||
path = None if cache is None else cache / "committee" / f"{cid}.json"
|
||||
got = _page(
|
||||
cl,
|
||||
f"{FEC_API}committee/{cid}/",
|
||||
None,
|
||||
pacer=pacer,
|
||||
path=path,
|
||||
refresh=refresh,
|
||||
budget=budget,
|
||||
)
|
||||
if got is None:
|
||||
break
|
||||
payload, src = got
|
||||
for rec in payload.get("results") or ():
|
||||
rows.append(
|
||||
to_committee(
|
||||
rec, entity_key=entity.key, basis="curated", source_file=src
|
||||
)
|
||||
)
|
||||
return rows, []
|
||||
|
||||
aliases = search_names(entity.name, entity.aliases)
|
||||
found: dict[str, FecCommitteeRow] = {}
|
||||
pending: list[dict[str, str]] = []
|
||||
for alias in aliases:
|
||||
path = (
|
||||
None if cache is None else cache / "committees" / f"{http.slug(alias)}.json"
|
||||
)
|
||||
got = _page(
|
||||
cl,
|
||||
f"{FEC_API}committees/",
|
||||
{"q": alias, "per_page": 20},
|
||||
pacer=pacer,
|
||||
path=path,
|
||||
refresh=refresh,
|
||||
budget=budget,
|
||||
)
|
||||
if got is None:
|
||||
break
|
||||
payload, src = got
|
||||
for rec in payload.get("results") or ():
|
||||
cid = _text(rec.get("committee_id"))
|
||||
name = _text(rec.get("name"))
|
||||
if not cid or cid in found:
|
||||
continue
|
||||
if alias_match(name, aliases, prefix=False):
|
||||
found[cid] = to_committee(
|
||||
rec, entity_key=entity.key, basis="exact-name", source_file=src
|
||||
)
|
||||
else:
|
||||
pending.append(
|
||||
{
|
||||
"committee_id": cid,
|
||||
"name": name,
|
||||
"entity_key": entity.key,
|
||||
"query": alias,
|
||||
}
|
||||
)
|
||||
matched = set(found)
|
||||
candidates = [c for c in pending if c["committee_id"] not in matched]
|
||||
return list(found.values()), candidates
|
||||
|
||||
|
||||
def totals(
|
||||
cl: httpx.Client,
|
||||
cid: str,
|
||||
cycle: int,
|
||||
*,
|
||||
pacer: Any,
|
||||
cache: Path | None = None,
|
||||
refresh: bool = False,
|
||||
budget: Budget | None = None,
|
||||
) -> FecTotalsRow | None:
|
||||
"""``/committee/{cid}/totals/?cycle=``, or ``None`` when that cycle has
|
||||
no totals (or the budget stopped the download)."""
|
||||
path = None if cache is None else cache / "totals" / cid / f"{cycle}.json"
|
||||
got = _page(
|
||||
cl,
|
||||
f"{FEC_API}committee/{cid}/totals/",
|
||||
{"cycle": cycle},
|
||||
pacer=pacer,
|
||||
path=path,
|
||||
refresh=refresh,
|
||||
budget=budget,
|
||||
)
|
||||
if got is None:
|
||||
return None
|
||||
payload, src = got
|
||||
results = payload.get("results") or ()
|
||||
if not results:
|
||||
return None
|
||||
return to_totals(results[0], source_file=src)
|
||||
|
||||
|
||||
def contributions(
|
||||
cl: httpx.Client,
|
||||
cid: str,
|
||||
cycle: int,
|
||||
*,
|
||||
pacer: Any,
|
||||
cache: Path | None = None,
|
||||
refresh: bool = False,
|
||||
budget: Budget | None = None,
|
||||
) -> list[FecContributionRow]:
|
||||
"""Every Schedule B row for *cid* in *cycle*, keyset-paged on
|
||||
``pagination.last_indexes``. Memo rows are kept."""
|
||||
base = {
|
||||
"committee_id": cid,
|
||||
"two_year_transaction_period": cycle,
|
||||
"per_page": PER_PAGE,
|
||||
"sort": "-disbursement_date",
|
||||
}
|
||||
extra: dict[str, Any] = {}
|
||||
seen: set[str] = set()
|
||||
out: list[FecContributionRow] = []
|
||||
n = 0
|
||||
while True:
|
||||
n += 1
|
||||
path = (
|
||||
None
|
||||
if cache is None
|
||||
else cache / "schedule_b" / cid / str(cycle) / f"{n}.json"
|
||||
)
|
||||
got = _page(
|
||||
cl,
|
||||
f"{FEC_API}schedules/schedule_b/",
|
||||
{**base, **extra},
|
||||
pacer=pacer,
|
||||
path=path,
|
||||
refresh=refresh,
|
||||
budget=budget,
|
||||
)
|
||||
if got is None:
|
||||
break
|
||||
payload, src = got
|
||||
recs = payload.get("results") or ()
|
||||
for rec in recs:
|
||||
out.append(to_contribution(rec, source_file=src, cycle=cycle))
|
||||
last = (payload.get("pagination") or {}).get("last_indexes") or {}
|
||||
idx = _text(last.get("last_index"))
|
||||
if not recs or not idx or idx in seen:
|
||||
break
|
||||
seen.add(idx)
|
||||
extra = {
|
||||
"last_index": last.get("last_index"),
|
||||
"last_disbursement_date": last.get("last_disbursement_date") or "",
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def contributions_by_cycle(con: Any, entity_key: str) -> dict[int, float]:
|
||||
"""Schedule B amounts by cycle for *entity_key*. A memo row
|
||||
(``memo_code`` ``X``) is not summed with the disbursement it restates."""
|
||||
ensure_tables(con)
|
||||
got = con.execute(
|
||||
"SELECT c.cycle, coalesce(sum(c.amount), 0) "
|
||||
"FROM market.fec_contribution c "
|
||||
"JOIN market.fec_committee m ON m.committee_id = c.committee_id "
|
||||
"WHERE m.entity_key = ? AND c.memo_code <> 'X' "
|
||||
"GROUP BY c.cycle ORDER BY c.cycle",
|
||||
[entity_key],
|
||||
).fetchall()
|
||||
return {int(cycle): float(total) for cycle, total in got}
|
||||
@@ -66,6 +66,25 @@ CREATE TABLE IF NOT EXISTS market.lda_filing (
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.lda_issue (
|
||||
filing_uuid VARCHAR, general_issue_code VARCHAR, description VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.fec_committee (
|
||||
committee_id VARCHAR, name VARCHAR, committee_type VARCHAR,
|
||||
designation VARCHAR, entity_key VARCHAR, basis VARCHAR, cite_url VARCHAR,
|
||||
first_file_date DATE, last_file_date DATE, source_file VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.fec_totals (
|
||||
committee_id VARCHAR, cycle INTEGER, receipts DOUBLE,
|
||||
disbursements DOUBLE, individual_contributions DOUBLE,
|
||||
fed_candidate_committee_contributions DOUBLE,
|
||||
independent_expenditures DOUBLE, coverage_start DATE,
|
||||
coverage_end DATE, source_file VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.fec_contribution (
|
||||
sub_id VARCHAR, committee_id VARCHAR, cycle INTEGER, day DATE,
|
||||
amount DOUBLE, recipient_committee_id VARCHAR, recipient_name VARCHAR,
|
||||
candidate_id VARCHAR, candidate_name VARCHAR, candidate_office VARCHAR,
|
||||
candidate_state VARCHAR, candidate_district VARCHAR, memo_code VARCHAR,
|
||||
source_file VARCHAR
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -265,6 +284,60 @@ class LdaIssueRow:
|
||||
description: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FecCommitteeRow:
|
||||
"""One FEC committee tied to an entity. ``basis`` is ``curated`` (a seed
|
||||
id) or ``exact-name`` (the normalised name equals an alias)."""
|
||||
|
||||
committee_id: str
|
||||
name: str
|
||||
committee_type: str
|
||||
designation: str
|
||||
entity_key: str
|
||||
basis: str
|
||||
cite_url: str
|
||||
first_file_date: date | None
|
||||
last_file_date: date | None
|
||||
source_file: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FecTotalsRow:
|
||||
"""One committee's FEC totals for one two-year cycle."""
|
||||
|
||||
committee_id: str
|
||||
cycle: int
|
||||
receipts: float
|
||||
disbursements: float
|
||||
individual_contributions: float
|
||||
fed_candidate_committee_contributions: float
|
||||
independent_expenditures: float
|
||||
coverage_start: date | None
|
||||
coverage_end: date | None
|
||||
source_file: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FecContributionRow:
|
||||
"""One Schedule B disbursement, stored as filed. ``memo_code`` ``X``
|
||||
restates another row and is left out of cycle sums."""
|
||||
|
||||
sub_id: str
|
||||
committee_id: str
|
||||
cycle: int
|
||||
day: date | None
|
||||
amount: float
|
||||
recipient_committee_id: str
|
||||
recipient_name: str
|
||||
candidate_id: str
|
||||
candidate_name: str
|
||||
candidate_office: str
|
||||
candidate_state: str
|
||||
candidate_district: str
|
||||
memo_code: str
|
||||
source_file: str
|
||||
|
||||
|
||||
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
|
||||
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
|
||||
a statement per row, far too slow for filing lists)."""
|
||||
@@ -610,6 +683,96 @@ def read_lda_issues(con: Any) -> list[LdaIssueRow]:
|
||||
)
|
||||
|
||||
|
||||
# ── fec ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def write_fec_committees(con: Any, rows: Sequence[FecCommitteeRow]) -> int:
|
||||
"""Replace each committee by ``committee_id``."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
for chunk in _chunks(sorted({r.committee_id for r in rows})):
|
||||
con.execute(
|
||||
"DELETE FROM market.fec_committee WHERE committee_id IN "
|
||||
f"({','.join('?' * len(chunk))})",
|
||||
list(chunk),
|
||||
)
|
||||
return _insert(con, "market.fec_committee", rows)
|
||||
|
||||
|
||||
def read_fec_committees(
|
||||
con: Any, entity_key: str | None = None
|
||||
) -> list[FecCommitteeRow]:
|
||||
rows = _select(
|
||||
con,
|
||||
FecCommitteeRow,
|
||||
"market.fec_committee",
|
||||
"entity_key = ?" if entity_key is not None else "",
|
||||
[entity_key] if entity_key is not None else [],
|
||||
)
|
||||
return sorted((FecCommitteeRow(*r) for r in rows), key=lambda r: r.committee_id)
|
||||
|
||||
|
||||
def write_fec_totals(con: Any, rows: Sequence[FecTotalsRow]) -> int:
|
||||
"""Replace the totals of each ``(committee_id, cycle)`` in *rows*."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
keys = sorted({(r.committee_id, r.cycle) for r in rows})
|
||||
for part in _chunks(keys, 400):
|
||||
con.execute(
|
||||
"DELETE FROM market.fec_totals WHERE (committee_id, cycle) IN "
|
||||
"(SELECT * FROM (VALUES " + ", ".join(["(?, ?)"] * len(part)) + "))",
|
||||
[v for k in part for v in k],
|
||||
)
|
||||
return _insert(con, "market.fec_totals", rows)
|
||||
|
||||
|
||||
def read_fec_totals(con: Any, committee_id: str | None = None) -> list[FecTotalsRow]:
|
||||
rows = _select(
|
||||
con,
|
||||
FecTotalsRow,
|
||||
"market.fec_totals",
|
||||
"committee_id = ?" if committee_id is not None else "",
|
||||
[committee_id] if committee_id is not None else [],
|
||||
)
|
||||
return sorted(
|
||||
(FecTotalsRow(*r) for r in rows), key=lambda r: (r.committee_id, r.cycle)
|
||||
)
|
||||
|
||||
|
||||
def write_fec_contributions(con: Any, rows: Sequence[FecContributionRow]) -> int:
|
||||
"""Replace every Schedule B row of each ``(committee_id, cycle)`` in
|
||||
*rows* (a rerun drops disbursements the source no longer returns)."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
keys = sorted({(r.committee_id, r.cycle) for r in rows})
|
||||
for part in _chunks(keys, 400):
|
||||
con.execute(
|
||||
"DELETE FROM market.fec_contribution WHERE (committee_id, cycle) IN "
|
||||
"(SELECT * FROM (VALUES " + ", ".join(["(?, ?)"] * len(part)) + "))",
|
||||
[v for k in part for v in k],
|
||||
)
|
||||
return _insert(con, "market.fec_contribution", rows)
|
||||
|
||||
|
||||
def read_fec_contributions(
|
||||
con: Any, committee_id: str | None = None
|
||||
) -> list[FecContributionRow]:
|
||||
rows = _select(
|
||||
con,
|
||||
FecContributionRow,
|
||||
"market.fec_contribution",
|
||||
"committee_id = ?" if committee_id is not None else "",
|
||||
[committee_id] if committee_id is not None else [],
|
||||
)
|
||||
return sorted(
|
||||
(FecContributionRow(*r) for r in rows),
|
||||
key=lambda r: (r.committee_id, r.cycle, r.sub_id),
|
||||
)
|
||||
|
||||
|
||||
# ── districts ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -1912,3 +1912,197 @@ def test_lda_cli_tracks_seed_companies_only(lda):
|
||||
assert res.exit_code == 0, res.output
|
||||
assert {r.url.params["client_name"] for r in seen} >= {seed.name}
|
||||
assert "0 filings" in res.output
|
||||
|
||||
|
||||
# ── fec ──────────────────────────────────────────────────────────────
|
||||
|
||||
FEC_SECRET = "fec-cli-key-do-not-leak"
|
||||
|
||||
|
||||
def _fec_entities(c):
|
||||
T.write_entities(
|
||||
c,
|
||||
[
|
||||
T.EntityRow(
|
||||
"AAFP",
|
||||
"American Academy of Family Physicians",
|
||||
"organization",
|
||||
aliases="AMERICAN ACADEMY OF FAMILY PHYSICIANS",
|
||||
),
|
||||
T.EntityRow("AMA", "AMA", "organization", aliases="AMA"),
|
||||
T.EntityRow(
|
||||
"PRIMARY-CARE-SEAT",
|
||||
"Primary care rotating-seat societies",
|
||||
"organization",
|
||||
aliases="PRIMARY CARE ROTATING SEAT",
|
||||
pseudo=True,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fec(env, monkeypatch):
|
||||
from market import fec as fec_api
|
||||
|
||||
c, store, published, hoard = env
|
||||
_fec_entities(c)
|
||||
seen: list[httpx.Request] = []
|
||||
page1 = json.loads((FX / "fec_schedule_b_page1.json").read_text())
|
||||
totals = json.loads((FX / "fec_totals_C00411553_2024.json").read_text())
|
||||
committee = json.loads((FX / "fec_committees_aafp.json").read_text())
|
||||
monkeypatch.setattr(fec_api, "new_pacer", lambda: _Pacer())
|
||||
state = {"status": 200}
|
||||
|
||||
def handler(request):
|
||||
seen.append(request)
|
||||
if state["status"] != 200:
|
||||
return httpx.Response(state["status"], text="no")
|
||||
path = request.url.path
|
||||
if path.endswith("/totals/"):
|
||||
return httpx.Response(200, json=totals)
|
||||
if path.endswith("/schedule_b/"):
|
||||
if "last_index" in request.url.params:
|
||||
return httpx.Response(200, json={"results": [], "pagination": {}})
|
||||
return httpx.Response(200, json=page1)
|
||||
if path.startswith("/v1/committees"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"results": [
|
||||
{
|
||||
"committee_id": "C00999999",
|
||||
"name": "AMALGAMATED TRANSIT UNION PAC",
|
||||
"committee_type": "Q",
|
||||
"designation": "U",
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
return httpx.Response(200, json=committee)
|
||||
|
||||
monkeypatch.setattr(
|
||||
http,
|
||||
"client",
|
||||
lambda source, key=None: httpx.Client(
|
||||
transport=httpx.MockTransport(handler),
|
||||
headers={"X-Api-Key": key} if key else {},
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET)
|
||||
return c, store, published, hoard, seen, state
|
||||
|
||||
|
||||
def test_fec_cli_write_from_cache(fec):
|
||||
c, _, published, hoard, seen, state = fec
|
||||
T.write_gap(c, "fec:fec", "old")
|
||||
dry = runner.invoke(app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024"])
|
||||
assert dry.exit_code == 0, dry.output
|
||||
assert "dry run: pass --write" in dry.output
|
||||
assert "1 committees (1 curated, 0 exact-name)" in dry.output
|
||||
assert _count(c, "market.fec_committee") == 0
|
||||
assert published == []
|
||||
assert seen and all("/committees" not in r.url.path for r in seen)
|
||||
assert all(FEC_SECRET not in str(r.url) for r in seen)
|
||||
n = len(seen)
|
||||
state["status"] = 500
|
||||
res = runner.invoke(
|
||||
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert len(seen) == n # the hoard answered; the 500 was never requested
|
||||
assert "stored: 1 committees, 1 totals, 3 schedule B rows" in res.output
|
||||
(committee,) = T.read_fec_committees(c)
|
||||
assert committee.basis == "curated" and committee.committee_id == "C00411553"
|
||||
assert committee.source_file.endswith("fec/committee/C00411553.json")
|
||||
(totals,) = T.read_fec_totals(c)
|
||||
assert totals.receipts == 513798.01
|
||||
assert totals.fed_candidate_committee_contributions == 683300.0
|
||||
names = {r.recipient_name for r in T.read_fec_contributions(c)}
|
||||
assert "JANE Q PUBLIC" in names and "MEMO OF THE 100" in names
|
||||
from market.fec import contributions_by_cycle
|
||||
|
||||
assert contributions_by_cycle(c, "AAFP") == {2024: 125.0}
|
||||
assert T.read_gaps(c) == []
|
||||
assert published == [True]
|
||||
assert {
|
||||
(t, n)
|
||||
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||||
} == {
|
||||
("market.fec_committee", 1),
|
||||
("market.fec_totals", 1),
|
||||
("market.fec_contribution", 3),
|
||||
}
|
||||
manifest = (hoard / "MANIFEST.jsonl").read_text()
|
||||
assert FEC_SECRET not in manifest
|
||||
assert (hoard / "fec" / "schedule_b" / "C00411553" / "2024" / "1.json").is_file()
|
||||
|
||||
|
||||
def test_fec_cli_no_write_prints_candidates(fec):
|
||||
c, _, published, _, seen, _ = fec
|
||||
res = runner.invoke(app, ["market", "fec", "--entity", "AMA", "--cycles", "2024"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "candidate: AMA C00999999 AMALGAMATED TRANSIT UNION PAC" in res.output
|
||||
assert "0 committees" in res.output and "1 candidates" in res.output
|
||||
assert "dry run: pass --write" in res.output
|
||||
assert _count(c, "market.fec_committee") == 0
|
||||
assert _count(c, "market.fec_contribution") == 0
|
||||
assert published == []
|
||||
assert seen[0].url.params["q"] == "AMA"
|
||||
assert all(
|
||||
"/committee/" not in r.url.path or r.url.path.startswith("/v1/committees")
|
||||
for r in seen
|
||||
)
|
||||
default = runner.invoke(app, ["market", "fec", "--entity", "AMA"])
|
||||
assert default.exit_code == 0, default.output
|
||||
assert "1 candidates" in default.output
|
||||
bad = runner.invoke(app, ["market", "fec", "--entity", "NOPE", "--cycles", "2024"])
|
||||
assert bad.exit_code == 2 and "unknown entity: NOPE" in bad.output
|
||||
cycles = runner.invoke(app, ["market", "fec", "--cycles", "nope"])
|
||||
assert cycles.exit_code == 2 and "not a list of years" in cycles.output
|
||||
|
||||
|
||||
def test_fec_cli_refusal_prints_gap(fec, monkeypatch):
|
||||
c, _, published, _, seen, state = fec
|
||||
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
|
||||
missing = runner.invoke(
|
||||
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
|
||||
)
|
||||
assert missing.exit_code == 0, missing.output
|
||||
assert "gap: fec:fec:" in missing.output
|
||||
assert "FEC_API_KEY" in missing.output
|
||||
assert seen == []
|
||||
assert [g.source for g in T.read_gaps(c)] == ["fec:fec"]
|
||||
assert published == [True]
|
||||
monkeypatch.setattr(http, "api_key", lambda source, env=None: FEC_SECRET)
|
||||
state["status"] = 403
|
||||
res = runner.invoke(
|
||||
app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: fec:fec: fec: HTTP 403" in res.output
|
||||
assert FEC_SECRET not in res.output
|
||||
assert _count(c, "market.fec_committee") == 0
|
||||
assert len(seen) == 1
|
||||
assert all(FEC_SECRET not in str(r.url) for r in seen)
|
||||
assert seen[0].headers["x-api-key"] == FEC_SECRET
|
||||
|
||||
|
||||
def test_fec_cli_limit_keeps_the_gap_and_empty_registry(fec, monkeypatch):
|
||||
c, _, _, _, seen, _ = fec
|
||||
T.write_gap(c, "fec:fec", "old")
|
||||
res = runner.invoke(
|
||||
app,
|
||||
["market", "fec", "--cycles", "2024,2022", "--limit", "1", "--write"],
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stopped: request limit" in res.output
|
||||
assert [g.note for g in T.read_gaps(c)] == ["old"]
|
||||
assert _count(c, "market.fec_committee") == 1
|
||||
assert _count(c, "market.fec_totals") == 0
|
||||
assert len(seen) == 1
|
||||
monkeypatch.setattr(mk, "_lda_tracked", lambda con: [])
|
||||
empty = runner.invoke(app, ["market", "fec", "--cycles", "2024"])
|
||||
assert empty.exit_code == 1 and "no tracked entities" in empty.output
|
||||
blank = runner.invoke(app, ["market", "fec", "--cycles", ","])
|
||||
assert blank.exit_code == 2 and "--cycles is empty" in blank.output
|
||||
|
||||
13
tests/market/fixtures/fec_committees_aafp.json
Normal file
13
tests/market/fixtures/fec_committees_aafp.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"api_version": "1.0",
|
||||
"results": [
|
||||
{
|
||||
"committee_id": "C00411553",
|
||||
"name": "AMERICAN ACADEMY OF FAMILY PHYSICIANS POLITICAL ACTION COMMITTEE (FAMMEDPAC)",
|
||||
"committee_type": "Q",
|
||||
"designation": "U",
|
||||
"first_file_date": "2003-07-15",
|
||||
"last_file_date": "2024-10-15"
|
||||
}
|
||||
]
|
||||
}
|
||||
59
tests/market/fixtures/fec_schedule_b_page1.json
Normal file
59
tests/market/fixtures/fec_schedule_b_page1.json
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"pagination": {
|
||||
"count": 3,
|
||||
"page": 1,
|
||||
"pages": 2,
|
||||
"per_page": 100,
|
||||
"last_indexes": {
|
||||
"last_index": 123,
|
||||
"last_disbursement_date": "2024-03-01"
|
||||
}
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"sub_id": "4010120240001",
|
||||
"committee_id": "C00411553",
|
||||
"two_year_transaction_period": 2024,
|
||||
"disbursement_date": "2024-06-15",
|
||||
"disbursement_amount": 100.0,
|
||||
"recipient_committee_id": "C00111111",
|
||||
"recipient_name": "FRIENDS OF SOMEONE",
|
||||
"candidate_id": "H0KS00001",
|
||||
"candidate_name": "SOMEONE, A",
|
||||
"candidate_office": "H",
|
||||
"candidate_office_state": "KS",
|
||||
"candidate_office_district": "02",
|
||||
"memo_code": null
|
||||
},
|
||||
{
|
||||
"sub_id": "4010120240002",
|
||||
"committee_id": "C00411553",
|
||||
"two_year_transaction_period": 2024,
|
||||
"disbursement_date": "2024-05-01",
|
||||
"disbursement_amount": 25.0,
|
||||
"recipient_committee_id": null,
|
||||
"recipient_name": "JANE Q PUBLIC",
|
||||
"candidate_id": null,
|
||||
"candidate_name": null,
|
||||
"candidate_office": null,
|
||||
"candidate_office_state": null,
|
||||
"candidate_office_district": null,
|
||||
"memo_code": null
|
||||
},
|
||||
{
|
||||
"sub_id": "4010120240003",
|
||||
"committee_id": "C00411553",
|
||||
"two_year_transaction_period": 2024,
|
||||
"disbursement_date": "2024-04-01",
|
||||
"disbursement_amount": 100.0,
|
||||
"recipient_committee_id": null,
|
||||
"recipient_name": "MEMO OF THE 100",
|
||||
"candidate_id": null,
|
||||
"candidate_name": null,
|
||||
"candidate_office": null,
|
||||
"candidate_office_state": null,
|
||||
"candidate_office_district": null,
|
||||
"memo_code": "X"
|
||||
}
|
||||
]
|
||||
}
|
||||
16
tests/market/fixtures/fec_totals_C00411553_2024.json
Normal file
16
tests/market/fixtures/fec_totals_C00411553_2024.json
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"api_version": "1.0",
|
||||
"results": [
|
||||
{
|
||||
"committee_id": "C00411553",
|
||||
"cycle": 2024,
|
||||
"receipts": 513798.01,
|
||||
"disbursements": 696685.91,
|
||||
"individual_contributions": 480000.0,
|
||||
"fed_candidate_committee_contributions": 683300.0,
|
||||
"independent_expenditures": 0.0,
|
||||
"coverage_start_date": "2023-01-01",
|
||||
"coverage_end_date": "2024-12-31"
|
||||
}
|
||||
]
|
||||
}
|
||||
399
tests/market/test_fec.py
Normal file
399
tests/market/test_fec.py
Normal file
@@ -0,0 +1,399 @@
|
||||
"""market.fec: curated committee ids, exact-name fallback, totals, Schedule B."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from market import fec as F
|
||||
from market import http
|
||||
from market import tables as T
|
||||
|
||||
FX = Path(__file__).parent / "fixtures"
|
||||
SECRET = "fec-test-key-do-not-leak"
|
||||
PAC = "AMERICAN ACADEMY OF FAMILY PHYSICIANS POLITICAL ACTION COMMITTEE (FAMMEDPAC)"
|
||||
PAC_ALIAS = "AMERICAN ACADEMY OF FAMILY PHYSICIANS POLITICAL ACTION COMMITTEE FAMMEDPAC"
|
||||
SHORT = "AMERICAN ACADEMY OF FAMILY PHYSICIANS"
|
||||
|
||||
|
||||
def _load(name: str) -> dict:
|
||||
return json.loads((FX / name).read_text())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def con():
|
||||
c = __import__("duckdb").connect(":memory:")
|
||||
T.ensure_tables(c)
|
||||
yield c
|
||||
c.close()
|
||||
|
||||
|
||||
class Pacer:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
|
||||
def wait(self):
|
||||
self.n += 1
|
||||
|
||||
|
||||
def entity(key, name, aliases=""):
|
||||
return T.EntityRow(key, name, "organization", aliases=aliases or name)
|
||||
|
||||
|
||||
def _client(handler, *, key: str | None = None) -> httpx.Client:
|
||||
headers = {"X-Api-Key": key} if key else {}
|
||||
return httpx.Client(transport=httpx.MockTransport(handler), headers=headers)
|
||||
|
||||
|
||||
def test_committee_curated_ids_win():
|
||||
assert F.FEC_API == "https://api.open.fec.gov/v1/"
|
||||
assert F.new_pacer().min_interval == 3.7
|
||||
seen = []
|
||||
body = _load("fec_committees_aafp.json")
|
||||
|
||||
def handler(request):
|
||||
seen.append(request)
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
cl = _client(handler)
|
||||
rows, cands = F.resolve(cl, entity("AAFP", SHORT), ["C00411553"], pacer=Pacer())
|
||||
assert cands == []
|
||||
assert [(r.committee_id, r.basis, r.entity_key) for r in rows] == [
|
||||
("C00411553", "curated", "AAFP")
|
||||
]
|
||||
assert rows[0].cite_url == "https://www.fec.gov/data/committee/C00411553/"
|
||||
assert [r.url.host for r in seen] == ["api.open.fec.gov"]
|
||||
assert seen[0].url.path == "/v1/committee/C00411553/"
|
||||
assert all("/committees" not in r.url.path for r in seen)
|
||||
assert "api_key" not in str(seen[0].url)
|
||||
|
||||
|
||||
def test_committee_exact_name_fallback_cited():
|
||||
body = {"results": _load("fec_committees_aafp.json")["results"]}
|
||||
|
||||
def handler(request):
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
cl = _client(handler)
|
||||
rows, cands = F.resolve(cl, entity("AAFP", SHORT), [], pacer=Pacer())
|
||||
assert rows == []
|
||||
assert cands[0]["name"] == PAC
|
||||
assert cands[0]["committee_id"] == "C00411553"
|
||||
|
||||
rows, cands = F.resolve(cl, entity("AAFP", PAC_ALIAS), [], pacer=Pacer())
|
||||
assert cands == []
|
||||
assert rows[0].basis == "exact-name"
|
||||
assert rows[0].cite_url == "https://www.fec.gov/data/committee/C00411553/"
|
||||
assert rows[0].name == PAC
|
||||
|
||||
|
||||
def test_short_alias_never_matches_inside_names():
|
||||
name = "AMALGAMATED TRANSIT UNION PAC"
|
||||
|
||||
def handler(request):
|
||||
assert request.url.params["q"] == "AMA"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"results": [{"committee_id": "C00999999", "name": name}]},
|
||||
)
|
||||
|
||||
rows, cands = F.resolve(_client(handler), entity("AMA", "AMA"), [], pacer=Pacer())
|
||||
assert rows == []
|
||||
assert [(c["committee_id"], c["name"]) for c in cands] == [("C00999999", name)]
|
||||
|
||||
|
||||
def test_totals_row():
|
||||
body = _load("fec_totals_C00411553_2024.json")
|
||||
|
||||
def handler(request):
|
||||
assert request.url.path == "/v1/committee/C00411553/totals/"
|
||||
assert request.url.params["cycle"] == "2024"
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
row = F.totals(_client(handler), "C00411553", 2024, pacer=Pacer())
|
||||
assert row is not None
|
||||
assert row.receipts == 513798.01
|
||||
assert row.disbursements == 696685.91
|
||||
assert row.fed_candidate_committee_contributions == 683300.0
|
||||
assert (row.coverage_start, row.coverage_end) == (
|
||||
date(2023, 1, 1),
|
||||
date(2024, 12, 31),
|
||||
)
|
||||
|
||||
|
||||
def test_schedule_b_keyset_paging():
|
||||
page1 = _load("fec_schedule_b_page1.json")
|
||||
page2 = {
|
||||
"results": [
|
||||
{
|
||||
"sub_id": "4010120240004",
|
||||
"committee_id": "C00411553",
|
||||
"two_year_transaction_period": 2024,
|
||||
"disbursement_date": "2024-02-01",
|
||||
"disbursement_amount": 1.0,
|
||||
"recipient_name": "SECOND PAGE",
|
||||
"memo_code": None,
|
||||
}
|
||||
],
|
||||
"pagination": {},
|
||||
}
|
||||
seen = []
|
||||
|
||||
def handler(request):
|
||||
seen.append(request)
|
||||
if "last_index" in request.url.params:
|
||||
return httpx.Response(200, json=page2)
|
||||
return httpx.Response(200, json=page1)
|
||||
|
||||
rows = F.contributions(_client(handler), "C00411553", 2024, pacer=Pacer())
|
||||
assert len(seen) == 2
|
||||
assert seen[0].url.path == "/v1/schedules/schedule_b/"
|
||||
assert seen[0].url.params["per_page"] == "100"
|
||||
assert seen[0].url.params["sort"] == "-disbursement_date"
|
||||
assert seen[0].url.params["two_year_transaction_period"] == "2024"
|
||||
assert "last_index" not in seen[0].url.params
|
||||
assert seen[1].url.params["last_index"] == "123"
|
||||
assert seen[1].url.params["last_disbursement_date"] == "2024-03-01"
|
||||
assert [r.sub_id for r in rows] == [
|
||||
"4010120240001",
|
||||
"4010120240002",
|
||||
"4010120240003",
|
||||
"4010120240004",
|
||||
]
|
||||
|
||||
|
||||
def test_memo_rows_excluded_from_totals(con):
|
||||
T.write_fec_committees(
|
||||
con,
|
||||
[
|
||||
T.FecCommitteeRow(
|
||||
"C00411553",
|
||||
PAC,
|
||||
"Q",
|
||||
"U",
|
||||
"AAFP",
|
||||
"curated",
|
||||
"https://www.fec.gov/data/committee/C00411553/",
|
||||
None,
|
||||
None,
|
||||
"",
|
||||
)
|
||||
],
|
||||
)
|
||||
rows = [
|
||||
T.FecContributionRow(
|
||||
"a",
|
||||
"C00411553",
|
||||
2024,
|
||||
date(2024, 6, 1),
|
||||
100.0,
|
||||
"C1",
|
||||
"PAC",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
),
|
||||
T.FecContributionRow(
|
||||
"b",
|
||||
"C00411553",
|
||||
2024,
|
||||
date(2024, 5, 1),
|
||||
25.0,
|
||||
"",
|
||||
"JANE Q PUBLIC",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
),
|
||||
T.FecContributionRow(
|
||||
"c",
|
||||
"C00411553",
|
||||
2024,
|
||||
date(2024, 4, 1),
|
||||
100.0,
|
||||
"",
|
||||
"MEMO",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"X",
|
||||
"",
|
||||
),
|
||||
]
|
||||
T.write_fec_contributions(con, rows)
|
||||
assert F.contributions_by_cycle(con, "AAFP") == {2024: 125.0}
|
||||
|
||||
|
||||
def test_schedule_b_rows_stored_as_filed(con):
|
||||
page = _load("fec_schedule_b_page1.json")
|
||||
|
||||
def handler(request):
|
||||
body = dict(page)
|
||||
body["pagination"] = {}
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
rows = F.contributions(_client(handler), "C00411553", 2024, pacer=Pacer())
|
||||
person = next(r for r in rows if r.recipient_name == "JANE Q PUBLIC")
|
||||
assert person.recipient_committee_id == ""
|
||||
assert person.amount == 25.0
|
||||
memo = next(r for r in rows if r.memo_code == "X")
|
||||
assert memo.recipient_name == "MEMO OF THE 100"
|
||||
T.write_fec_contributions(con, rows)
|
||||
stored = T.read_fec_contributions(con)
|
||||
assert [r.recipient_name for r in stored if r.sub_id == "4010120240002"] == [
|
||||
"JANE Q PUBLIC"
|
||||
]
|
||||
assert any(r.memo_code == "X" for r in stored)
|
||||
|
||||
|
||||
def test_fec_error_message_has_no_key():
|
||||
seen = []
|
||||
|
||||
def handler(request):
|
||||
seen.append(request)
|
||||
return httpx.Response(403, text="no")
|
||||
|
||||
cl = _client(handler, key=SECRET)
|
||||
with pytest.raises(http.Refused) as exc:
|
||||
F.totals(cl, "C00411553", 2024, pacer=Pacer())
|
||||
assert exc.value.source == "fec"
|
||||
assert SECRET not in str(exc.value)
|
||||
assert SECRET not in str(seen[0].url)
|
||||
assert "api_key" not in str(seen[0].url)
|
||||
assert seen[0].headers["x-api-key"] == SECRET
|
||||
|
||||
def bot(request):
|
||||
return httpx.Response(200, text="<html>Just a moment...</html>")
|
||||
|
||||
with pytest.raises(http.Refused) as exc2:
|
||||
F.totals(_client(bot, key=SECRET), "C00411553", 2024, pacer=Pacer())
|
||||
assert "not JSON" in str(exc2.value)
|
||||
assert SECRET not in str(exc2.value)
|
||||
|
||||
|
||||
def test_budget_edges_and_blank_fields(monkeypatch):
|
||||
company = type("C", (), {"cik": "0000000001", "fec_committee_ids": ("C1",)})()
|
||||
org = type(
|
||||
"O", (), {"key": "AAFP", "fec_committee_ids": ("C2",), "pseudo": False}
|
||||
)()
|
||||
pseudo = type(
|
||||
"O", (), {"key": "SEAT", "fec_committee_ids": ("C9",), "pseudo": True}
|
||||
)()
|
||||
monkeypatch.setattr(F, "load_seed", lambda: ([company], [org, pseudo]))
|
||||
assert F.committee_ids() == {"AAFP": ("C2",), "0000000001": ("C1",)}
|
||||
|
||||
limited = F.Budget(1)
|
||||
assert limited.take() is True
|
||||
assert limited.take() is False and limited.why == "request limit"
|
||||
stopped = F.Budget(0, stop=lambda: True)
|
||||
assert stopped.take() is False and stopped.why == "time limit"
|
||||
assert F._date(None) is None and F._date("") is None
|
||||
assert F._money("") == 0.0
|
||||
|
||||
row = F.to_contribution(
|
||||
{
|
||||
"sub_id": "z",
|
||||
"amount": "",
|
||||
"candidate_state": "KS",
|
||||
"candidate_district": "01",
|
||||
},
|
||||
source_file="s",
|
||||
cycle=2024,
|
||||
)
|
||||
assert (row.cycle, row.amount, row.day, row.candidate_state) == (
|
||||
2024,
|
||||
0.0,
|
||||
None,
|
||||
"KS",
|
||||
)
|
||||
|
||||
seen = []
|
||||
|
||||
def handler(request):
|
||||
seen.append(request.url.path)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"results": [
|
||||
{"committee_id": "", "name": "NO ID"},
|
||||
{"committee_id": "C1", "name": PAC},
|
||||
{"committee_id": "C1", "name": PAC},
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
cl = _client(handler)
|
||||
org_row = entity("AAFP", SHORT, aliases=f"{SHORT}|{PAC_ALIAS}")
|
||||
rows, cands = F.resolve(cl, org_row, [], pacer=Pacer(), budget=F.Budget(1))
|
||||
assert [r.committee_id for r in rows] == ["C1"]
|
||||
assert rows[0].basis == "exact-name"
|
||||
assert cands == []
|
||||
assert seen == ["/v1/committees/"] # the second alias stops on the budget
|
||||
|
||||
def curated(request):
|
||||
return httpx.Response(200, json=_load("fec_committees_aafp.json"))
|
||||
|
||||
one, _ = F.resolve(
|
||||
_client(curated),
|
||||
entity("AAFP", SHORT),
|
||||
["C00411553", "C00000001"],
|
||||
pacer=Pacer(),
|
||||
budget=F.Budget(1),
|
||||
)
|
||||
assert [r.committee_id for r in one] == ["C00411553"]
|
||||
|
||||
spent = F.Budget(1)
|
||||
spent.take()
|
||||
|
||||
def boom(request):
|
||||
raise AssertionError("budget should have stopped the download")
|
||||
|
||||
assert (
|
||||
F.totals(_client(boom), "C00411553", 2024, pacer=Pacer(), budget=spent) is None
|
||||
)
|
||||
assert (
|
||||
F.contributions(_client(boom), "C00411553", 2024, pacer=Pacer(), budget=spent)
|
||||
== []
|
||||
)
|
||||
|
||||
def empty(request):
|
||||
return httpx.Response(200, json={"results": []})
|
||||
|
||||
assert F.totals(_client(empty), "C00411553", 2022, pacer=Pacer()) is None
|
||||
|
||||
|
||||
def test_fec_hoard_reuses_and_skips_a_missing_committee(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
|
||||
calls = []
|
||||
body = _load("fec_committees_aafp.json")
|
||||
|
||||
def handler(request):
|
||||
calls.append(request.url.path)
|
||||
if "C00000000" in request.url.path:
|
||||
return httpx.Response(404, json={"results": []})
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
cl = _client(handler)
|
||||
cache = tmp_path / "fec"
|
||||
kw = dict(pacer=Pacer(), cache=cache)
|
||||
first, _ = F.resolve(cl, entity("AAFP", SHORT), ["C00411553"], **kw)
|
||||
again, _ = F.resolve(cl, entity("AAFP", SHORT), ["C00411553"], **kw)
|
||||
assert calls == ["/v1/committee/C00411553/"]
|
||||
assert first[0].source_file.endswith("fec/committee/C00411553.json")
|
||||
assert again[0].committee_id == "C00411553"
|
||||
empty, cands = F.resolve(cl, entity("AAFP", SHORT), ["C00000000"], **kw)
|
||||
assert empty == [] and cands == []
|
||||
@@ -13,6 +13,7 @@ MODULES = [
|
||||
"market.edgar",
|
||||
"market.tiingo",
|
||||
"market.lda",
|
||||
"market.fec",
|
||||
]
|
||||
|
||||
_PROBE = """
|
||||
|
||||
@@ -43,6 +43,9 @@ def test_ensure_tables_idempotent():
|
||||
"series",
|
||||
"lda_filing",
|
||||
"lda_issue",
|
||||
"fec_committee",
|
||||
"fec_totals",
|
||||
"fec_contribution",
|
||||
}
|
||||
|
||||
|
||||
@@ -141,6 +144,46 @@ LDA_FILING = T.LdaFilingRow(
|
||||
"f.json",
|
||||
)
|
||||
LDA_ISSUE = T.LdaIssueRow(LDA_FILING.filing_uuid, "BUD", "No Bill No.")
|
||||
FEC_COMMITTEE = T.FecCommitteeRow(
|
||||
"C00411553",
|
||||
"FAMMEDPAC",
|
||||
"Q",
|
||||
"U",
|
||||
"AAFP",
|
||||
"curated",
|
||||
"https://www.fec.gov/data/committee/C00411553/",
|
||||
date(2003, 7, 15),
|
||||
date(2024, 10, 15),
|
||||
"committee/C00411553.json",
|
||||
)
|
||||
FEC_TOTALS = T.FecTotalsRow(
|
||||
"C00411553",
|
||||
2024,
|
||||
513798.01,
|
||||
696685.91,
|
||||
480000.0,
|
||||
683300.0,
|
||||
0.0,
|
||||
date(2023, 1, 1),
|
||||
date(2024, 12, 31),
|
||||
"totals/C00411553/2024.json",
|
||||
)
|
||||
FEC_CONTRIBUTION = T.FecContributionRow(
|
||||
"4010120240002",
|
||||
"C00411553",
|
||||
2024,
|
||||
date(2024, 5, 1),
|
||||
25.0,
|
||||
"",
|
||||
"JANE Q PUBLIC",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"schedule_b/C00411553/2024/1.json",
|
||||
)
|
||||
SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY")
|
||||
|
||||
ROUNDTRIP = {
|
||||
@@ -154,6 +197,21 @@ ROUNDTRIP = {
|
||||
lambda c, r: T.write_lda_issues(c, [r]),
|
||||
lambda c: T.read_lda_issues(c),
|
||||
),
|
||||
"fec_committee": (
|
||||
FEC_COMMITTEE,
|
||||
lambda c, r: T.write_fec_committees(c, [r]),
|
||||
lambda c: T.read_fec_committees(c),
|
||||
),
|
||||
"fec_totals": (
|
||||
FEC_TOTALS,
|
||||
lambda c, r: T.write_fec_totals(c, [r]),
|
||||
lambda c: T.read_fec_totals(c),
|
||||
),
|
||||
"fec_contribution": (
|
||||
FEC_CONTRIBUTION,
|
||||
lambda c, r: T.write_fec_contributions(c, [r]),
|
||||
lambda c: T.read_fec_contributions(c),
|
||||
),
|
||||
"entity": (
|
||||
ENTITY,
|
||||
lambda c, r: T.write_entities(c, [r]),
|
||||
@@ -212,6 +270,9 @@ def test_write_empty_is_noop(con):
|
||||
assert T.write_securities(con, "1", []) == 0
|
||||
assert T.write_lda_filings(con, []) == 0
|
||||
assert T.write_lda_issues(con, []) == 0
|
||||
assert T.write_fec_committees(con, []) == 0
|
||||
assert T.write_fec_totals(con, []) == 0
|
||||
assert T.write_fec_contributions(con, []) == 0
|
||||
|
||||
|
||||
def test_lda_readers_filter_by_entity(con):
|
||||
|
||||
Reference in New Issue
Block a user