feat(market): hoard namesake FEC committees and the ACP and AOA PACs

A name-search hit that is not the tracked entity's PAC is kept on a full pull: committee page, cycle totals, and Schedule B, stored with an empty entity key. ACP Services PAC and the AOA affiliate's Osteopathic PAC are curated ids. The AAO communication-cost committee is kept on AAO.
This commit is contained in:
kert
2026-11-14 15:00:39 -05:00
parent 0424166bb8
commit cba6cca08f
8 changed files with 249 additions and 35 deletions

View File

@@ -1297,8 +1297,9 @@ def fec(
(market.fec_committee, market.fec_totals, market.fec_contribution). A
curated committee id is fetched directly; otherwise a search hit is kept
only when its name equals the entity's name or an alias. Other hits are
printed as candidates and not stored. A rerun resumes from the hoard
(data/market/fec)."""
printed as candidates and not stored as that entity's committee. A pull
with no ``--entity`` also keeps every ``fec_hoard`` id (empty entity key).
A rerun resumes from the hoard (data/market/fec)."""
chosen_cycles = _parse_cycles(cycles)
if not chosen_cycles:
typer.echo("--cycles is empty")
@@ -1367,6 +1368,36 @@ def fec(
budget=budget,
)
)
if not entity and not budget.exhausted:
have = {row.committee_id for row in committees}
extra = [cid for cid in fec_api.hoard_ids() if cid not in have]
for row in fec_api.hoard_committees(
cl, extra, pacer=pacer, cache=root, budget=budget
):
committees.append(row)
for cycle in chosen_cycles:
if budget.exhausted:
break
got = fec_api.totals(
cl,
row.committee_id,
cycle,
pacer=pacer,
cache=root,
budget=budget,
)
if got is not None:
totals.append(got)
contribs.extend(
fec_api.contributions(
cl,
row.committee_id,
cycle,
pacer=pacer,
cache=root,
budget=budget,
)
)
except http.Refused as exc:
_gap(FEC_GAP, str(exc), write=write)
return
@@ -1375,8 +1406,10 @@ def fec(
curated = sum(c.basis == "curated" for c in committees)
exact = sum(c.basis == "exact-name" for c in committees)
hoarded = sum(c.basis == "hoarded" for c in committees)
typer.echo(
f"{len(committees)} committees ({curated} curated, {exact} exact-name), "
f"{len(committees)} committees "
f"({curated} curated, {exact} exact-name, {hoarded} hoarded), "
f"{len(candidates)} candidates, {len(totals)} totals, "
f"{len(contribs)} schedule B rows; {budget.used} requests"
)
@@ -1384,6 +1417,9 @@ def fec(
typer.echo(
f"candidate: {cand['entity_key']} {cand['committee_id']} {cand['name']}"
)
for row in committees:
if row.basis == "hoarded":
typer.echo(f"hoarded: {row.committee_id} {row.name}")
if budget.exhausted:
typer.echo(f"stopped: {budget.why} (rerun resumes from the hoard)")
if not write:

View File

@@ -4,9 +4,12 @@ The key (``FEC_API_KEY``, else ``API_DATA_GOV_KEY``) travels only in the
``X-Api-Key`` header. A committee is stored from a curated
``fec_committee_ids`` entry, or, failing that, when its normalised name
equals an alias (``alias_match(prefix=False)``). Every other search hit is
a candidate: printed, not stored. Schedule B rows are stored as filed,
including an individual recipient's name; a memo row (``memo_code`` ``X``)
is stored and left out of the cycle sum.
a candidate: printed, not stored as that entity's committee. Committees
listed in the seed's ``fec_hoard`` are fetched in full on a pull of every
entity and stored with an empty entity key (basis ``hoarded``), so the
filing is kept and the money is not attributed to the alias that found it.
Schedule B rows are stored as filed, including an individual recipient's
name; a memo row (``memo_code`` ``X``) is stored and left out of the cycle sum.
"""
from __future__ import annotations
@@ -21,7 +24,7 @@ from typing import Any
import httpx
from market import http
from market.seed import alias_match, load_seed, norm_org
from market.seed import alias_match, load_fec_hoard, load_seed, norm_org
from market.tables import (
EntityRow,
FecCommitteeRow,
@@ -55,6 +58,11 @@ def committee_ids() -> dict[str, tuple[str, ...]]:
return out
def hoard_ids() -> tuple[str, ...]:
"""Committee ids to keep that are not a tracked entity's own PAC."""
return load_fec_hoard()
def search_names(name: str, aliases: str) -> list[str]:
"""The distinct (by normalised form) names to search: the entity's own
name first, then its ``|``-joined aliases."""
@@ -289,6 +297,43 @@ def resolve(
return list(found.values()), candidates
def hoard_committees(
cl: httpx.Client,
ids: Sequence[str],
*,
pacer: Any,
cache: Path | None = None,
refresh: bool = False,
budget: Budget | None = None,
) -> list[FecCommitteeRow]:
"""Committee records for ids that are not a tracked entity's PAC.
``basis`` is ``hoarded`` and ``entity_key`` is empty, so a later sum by
entity does not pick the money up. A 404 id is skipped."""
rows: list[FecCommitteeRow] = []
for cid in ids:
if not cid:
continue
path = None if cache is None else cache / "committee" / f"{cid}.json"
got = _page(
cl,
f"{FEC_API}committee/{cid}/",
None,
pacer=pacer,
path=path,
refresh=refresh,
budget=budget,
)
if got is None:
break
payload, src = got
for rec in payload.get("results") or ():
rows.append(
to_committee(rec, entity_key="", basis="hoarded", source_file=src)
)
return rows
def totals(
cl: httpx.Client,
cid: str,

View File

@@ -211,6 +211,22 @@ def _org(raw: Mapping[str, Any]) -> SeedOrg:
)
def load_fec_hoard(path: Path | str | None = None) -> tuple[str, ...]:
"""Committee ids the name search found that are not a tracked entity's
PAC. The FEC pull keeps their pages. A curated id belongs on the entity,
not in this list."""
raw = yaml.safe_load(Path(path or SEED_PATH).read_text()) or {}
out: list[str] = []
for item in raw.get("fec_hoard") or ():
cid = str(item.get("id") if isinstance(item, Mapping) else item).strip()
if len(cid) != 9 or not cid.startswith("C") or not cid[1:].isdigit():
raise ValueError(f"fec_hoard: {cid!r} is not a committee id")
out.append(cid)
if len(out) != len(set(out)):
raise ValueError("fec_hoard: duplicate committee id")
return tuple(out)
def load_gaps(path: Path | str | None = None) -> tuple[SeedGap, ...]:
"""Sources the seed records as unreadable (``gaps:`` in the YAML)."""
raw = yaml.safe_load(Path(path or SEED_PATH).read_text()) or {}

View File

@@ -452,6 +452,8 @@ orgs:
org_type: specialty_society
specialty: "INTERNAL MEDICINE"
aliases: ["AMERICAN COLLEGE OF PHYSICIANS"]
# Registered as AMERICAN COLLEGE OF PHYSICIAN SERVICES INC PAC (ACP SERVICES PAC).
fec_committee_ids: ["C00403881"]
cite: {item_key: "N8RVIA96", title: "AMA RUC roster 2023 (saved copy): society row for INTERNAL MEDICINE", accessed: "2026-11-12"}
- key: RPA
name: "Renal Physicians Association"
@@ -485,7 +487,8 @@ orgs:
org_type: specialty_society
specialty: "OPHTHALMOLOGY"
aliases: ["AMERICAN ACADEMY OF OPHTHALMOLOGY"]
fec_committee_ids: ["C00196246"]
# C70003785 is the same-name communication-cost committee filed beside OPHTHPAC.
fec_committee_ids: ["C00196246", "C70003785"]
cite: {item_key: "N8RVIA96", title: "AMA RUC roster 2023 (saved copy): society row for OPHTHALMOLOGY", accessed: "2026-11-12"}
- key: AAOS
name: "American Academy of Orthopaedic Surgeons"
@@ -615,6 +618,8 @@ orgs:
name: "American Osteopathic Association"
org_type: professional_assoc
aliases: ["AMERICAN OSTEOPATHIC ASSOCIATION"]
# FEC connects C00113803 to the American Osteopathic Information Association, the AOA PAC affiliate at 142 E. Ontario.
fec_committee_ids: ["C00113803"]
cite: {url: "https://osteopathic.org/", title: "American Osteopathic Association official website", accessed: "2026-11-12"}
- key: AHA
name: "American Hospital Association"
@@ -675,6 +680,23 @@ orgs:
aliases: ["ONCOLOGY SEAT"]
cite: {item_key: "KRWZGMGI", title: "AMA RUC roster 2026 (saved copy): oncology seat", accessed: "2026-11-12"}
# Name-search hits that are a different committee from the tracked entity's
# PAC. A full pull keeps each committee page, cycle totals, and Schedule B,
# stored with an empty entity key. NEMPAC C00140061 is already ACEP's curated
# id, so it is not repeated here.
fec_hoard:
- {id: "C00765230", name: "AMERICAN ANTHEM PAC", found_via: ELV}
- {id: "C00700799", name: "AMERICAN COLLEGE OF OSTEOPATHIC SURGEONS PAC", found_via: ACS}
- {id: "C00362152", name: "AMERICAN COLLEGE OF NUCLEAR PHYSICIANS NUCLEAR MEDICINE PAC", found_via: ACP}
- {id: "C00101493", name: "POLITICAL ACTION COMMITTEE OF THE AMERICAN OSTEOPATHIC HEALTHCARE ASSOCIATION", found_via: AOA}
- {id: "C00570291", name: "ASSOCIATION PROMOTING GOVERNMENTAL OBJECTIVE VIEWS APGOV", found_via: APG}
- {id: "C00941138", name: "AMERICAN INDEPENDENT MEDICAL PRACTICE ASSOCIATION PAC (AIMPA PAC)", found_via: AMA}
- {id: "C00190272", name: "AMERICAN MEDICAL CARE AND REVIEW ASSOCIATION POLITICAL ACTION COMMITTEE (AMCRA PAC)", found_via: AMA}
- {id: "C00898726", name: "AMERICAN MEDICAL MANUFACTURERS ASSOCIATION PAC (AMMA PAC)", found_via: AMA}
- {id: "C00678722", name: "AMERICAN MEDICAL REHABILITATION PROVIDERS ASSOCIATION INC. PAC (AMRPA PAC)", found_via: AMA}
- {id: "C00008839", name: "AMERICAN PODIATRIC MEDICAL ASSOCIATION POLITICAL ACTION COMMITTEE", found_via: AMA}
- {id: "C00114132", name: "AMERICAN VETERINARY MEDICAL ASSOCIATION POLITICAL ACTION COMMITTEE", found_via: AMA}
gaps:
- {source: "web:renalmd.org", note: "HTTP 403 to stack-pfs-dossier/1.0 on 2026-11-12; Renal Physicians Association site not read, so it is not cited as accessed"}
- {source: "web:asco.org", note: "HTTP 403 to stack-pfs-dossier/1.0 on 2026-11-12; American Society of Clinical Oncology site not read, so it is not cited as accessed"}

View File

@@ -286,8 +286,9 @@ class LdaIssueRow:
@dataclass(frozen=True)
class FecCommitteeRow:
"""One FEC committee tied to an entity. ``basis`` is ``curated`` (a seed
id) or ``exact-name`` (the normalised name equals an alias)."""
"""One FEC committee. ``basis`` is ``curated`` (a seed id), ``exact-name``
(the normalised name equals an alias), or ``hoarded`` (a namesake kept
with an empty ``entity_key``)."""
committee_id: str
name: str

View File

@@ -1930,6 +1930,7 @@ def _fec_entities(c):
aliases="AMERICAN ACADEMY OF FAMILY PHYSICIANS",
),
T.EntityRow("AMA", "AMA", "organization", aliases="AMA"),
T.EntityRow("LOOKALIKE", "LOOKALIKE", "organization", aliases="LOOKALIKE"),
T.EntityRow(
"PRIMARY-CARE-SEAT",
"Primary care rotating-seat societies",
@@ -1999,7 +2000,7 @@ def test_fec_cli_write_from_cache(fec):
dry = runner.invoke(app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024"])
assert dry.exit_code == 0, dry.output
assert "dry run: pass --write" in dry.output
assert "1 committees (1 curated, 0 exact-name)" in dry.output
assert "1 committees (1 curated, 0 exact-name, 0 hoarded)" in dry.output
assert _count(c, "market.fec_committee") == 0
assert published == []
assert seen and all("/committees" not in r.url.path for r in seen)
@@ -2040,20 +2041,22 @@ def test_fec_cli_write_from_cache(fec):
def test_fec_cli_no_write_prints_candidates(fec):
c, _, published, _, seen, _ = fec
res = runner.invoke(app, ["market", "fec", "--entity", "AMA", "--cycles", "2024"])
res = runner.invoke(
app, ["market", "fec", "--entity", "LOOKALIKE", "--cycles", "2024"]
)
assert res.exit_code == 0, res.output
assert "candidate: AMA C00999999 AMALGAMATED TRANSIT UNION PAC" in res.output
assert "candidate: LOOKALIKE C00999999 AMALGAMATED TRANSIT UNION PAC" in res.output
assert "0 committees" in res.output and "1 candidates" in res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.fec_committee") == 0
assert _count(c, "market.fec_contribution") == 0
assert published == []
assert seen[0].url.params["q"] == "AMA"
assert seen[0].url.params["q"] == "LOOKALIKE"
assert all(
"/committee/" not in r.url.path or r.url.path.startswith("/v1/committees")
for r in seen
)
default = runner.invoke(app, ["market", "fec", "--entity", "AMA"])
default = runner.invoke(app, ["market", "fec", "--entity", "LOOKALIKE"])
assert default.exit_code == 0, default.output
assert "1 candidates" in default.output
bad = runner.invoke(app, ["market", "fec", "--entity", "NOPE", "--cycles", "2024"])
@@ -2062,6 +2065,69 @@ def test_fec_cli_no_write_prints_candidates(fec):
assert cycles.exit_code == 2 and "not a list of years" in cycles.output
def test_fec_cli_full_pull_keeps_hoard_ids(fec, monkeypatch):
from market import fec as fec_api
c, _, published, hoard, seen, _ = fec
monkeypatch.setattr(fec_api, "hoard_ids", lambda: ("C00765230",))
committee = json.loads((FX / "fec_committees_aafp.json").read_text())
totals = json.loads((FX / "fec_totals_C00411553_2024.json").read_text())
page1 = json.loads((FX / "fec_schedule_b_page1.json").read_text())
def handler(request):
seen.append(request)
path = request.url.path
if path == "/v1/committee/C00765230/":
return httpx.Response(
200,
json={
"results": [
{
"committee_id": "C00765230",
"name": "AMERICAN ANTHEM PAC",
"committee_type": "V",
"designation": "U",
}
]
},
)
cid = request.url.params.get("committee_id", "")
if path.endswith("/totals/"):
if cid == "C00765230":
return httpx.Response(200, json={"results": []})
return httpx.Response(200, json=totals)
if path.endswith("/schedule_b/"):
if cid == "C00765230" or "last_index" in request.url.params:
return httpx.Response(200, json={"results": [], "pagination": {}})
return httpx.Response(200, json=page1)
if path.startswith("/v1/committees"):
return httpx.Response(200, json={"results": []})
if path.endswith("/C00411553/"):
return httpx.Response(200, json=committee)
return httpx.Response(200, json={"results": []})
monkeypatch.setattr(
http,
"client",
lambda source, key=None: httpx.Client(
transport=httpx.MockTransport(handler),
headers={"X-Api-Key": key} if key else {},
),
)
res = runner.invoke(app, ["market", "fec", "--cycles", "2024", "--write"])
assert res.exit_code == 0, res.output
assert "hoarded: C00765230 AMERICAN ANTHEM PAC" in res.output
rows = {r.committee_id: r for r in T.read_fec_committees(c)}
assert rows["C00765230"].basis == "hoarded" and rows["C00765230"].entity_key == ""
assert rows["C00411553"].basis == "curated"
assert (hoard / "fec" / "committee" / "C00765230.json").is_file()
from market.fec import contributions_by_cycle
assert contributions_by_cycle(c, "") == {}
assert contributions_by_cycle(c, "AMA") == {}
assert published == [True]
def test_fec_cli_refusal_prints_gap(fec, monkeypatch):
c, _, published, _, seen, state = fec
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)

View File

@@ -91,6 +91,30 @@ def test_committee_exact_name_fallback_cited():
assert rows[0].name == PAC
def test_hoard_committee_is_kept_unattached():
def handler(request):
assert request.url.path == "/v1/committee/C00765230/"
return httpx.Response(
200,
json={
"results": [
{
"committee_id": "C00765230",
"name": "AMERICAN ANTHEM PAC",
"committee_type": "V",
"designation": "U",
}
]
},
)
rows = F.hoard_committees(_client(handler), ["C00765230"], pacer=Pacer())
assert [(r.committee_id, r.basis, r.entity_key, r.name) for r in rows] == [
("C00765230", "hoarded", "", "AMERICAN ANTHEM PAC")
]
assert rows[0].cite_url == "https://www.fec.gov/data/committee/C00765230/"
def test_short_alias_never_matches_inside_names():
name = "AMALGAMATED TRANSIT UNION PAC"

View File

@@ -394,7 +394,7 @@ def test_only_symbol_records_are_unpriced():
def test_shipped_fec_committee_ids_are_the_connected_pacs():
"""Curated ids are the sponsor's own PAC. A short-alias search hit is not."""
from market.fec import committee_ids
from market.fec import committee_ids, hoard_ids
companies, orgs = S.load_seed()
by_handle = {c.handle: c.fec_committee_ids for c in companies}
@@ -405,7 +405,9 @@ def test_shipped_fec_committee_ids_are_the_connected_pacs():
by_org = {o.key: o.fec_committee_ids for o in orgs}
assert by_org["AMA"] == ("C00000422",)
assert by_org["AAFP"] == ("C00411553",)
assert by_org["ACP"] == () and by_org["AOA"] == ()
assert by_org["ACP"] == ("C00403881",)
assert by_org["AOA"] == ("C00113803",)
assert by_org["AAO"] == ("C00196246", "C70003785")
assert by_org["ACEP"] == ("C00140061",)
expect = {
"0000019584": ("C00279786",),
@@ -430,7 +432,9 @@ def test_shipped_fec_committee_ids_are_the_connected_pacs():
"AAN": ("C00435933",),
"AANS": ("C00413955",),
"AAFP": ("C00411553",),
"AAO": ("C00196246",),
"AAO": ("C00196246", "C70003785"),
"ACP": ("C00403881",),
"AOA": ("C00113803",),
"AAO-HNS": ("C00306449",),
"ACC": ("C00375360",),
"ACEP": ("C00140061",),
@@ -456,20 +460,20 @@ def test_shipped_fec_committee_ids_are_the_connected_pacs():
got = committee_ids()
assert got == expect
flat = {cid for row in got.values() for cid in row}
refused = {
"C00765230", # American Anthem PAC, not Elevance
"C00700799", # osteopathic surgeons, not ACS
"C00362152", # nuclear physicians, not ACP
"C00113803", # osteopathic information association, not AOA
"C00101493", # osteopathic healthcare association, not AOA
"C00570291", # APGOV, not America's Physician Groups
"C00941138", # AIMPA
"C00190272", # AMCRA
"C00898726", # AMMA
"C00678722", # AMRPA
"C00008839", # APMA
"C00114132", # AVMA
"C70003785", # AAO communication-cost committee, not OPHTHPAC
}
assert flat.isdisjoint(refused)
assert "ACP" not in got and "AOA" not in got
hoard = S.load_fec_hoard()
assert hoard == (
"C00765230",
"C00700799",
"C00362152",
"C00101493",
"C00570291",
"C00941138",
"C00190272",
"C00898726",
"C00678722",
"C00008839",
"C00114132",
)
assert hoard_ids() == hoard
assert set(hoard).isdisjoint(flat)
assert "C00140061" not in hoard