From cba6cca08f034b7c9580d510183c820d8e01f265 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 14 Nov 2026 15:00:39 -0500 Subject: [PATCH] feat(market): hoard namesake FEC committees and the ACP and AOA PACs A name-search hit that is not the tracked entity's PAC is kept on a full pull: committee page, cycle totals, and Schedule B, stored with an empty entity key. ACP Services PAC and the AOA affiliate's Osteopathic PAC are curated ids. The AAO communication-cost committee is kept on AAO. --- src/cli/market.py | 42 ++++++++++++++++++-- src/market/fec.py | 53 +++++++++++++++++++++++-- src/market/seed.py | 16 ++++++++ src/market/seed.yaml | 24 +++++++++++- src/market/tables.py | 5 ++- tests/cli/test_market_cli.py | 76 +++++++++++++++++++++++++++++++++--- tests/market/test_fec.py | 24 ++++++++++++ tests/market/test_seed.py | 44 +++++++++++---------- 8 files changed, 249 insertions(+), 35 deletions(-) diff --git a/src/cli/market.py b/src/cli/market.py index 3a073d2..83f4a1d 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -1297,8 +1297,9 @@ def fec( (market.fec_committee, market.fec_totals, market.fec_contribution). A curated committee id is fetched directly; otherwise a search hit is kept only when its name equals the entity's name or an alias. Other hits are - printed as candidates and not stored. A rerun resumes from the hoard - (data/market/fec).""" + printed as candidates and not stored as that entity's committee. A pull + with no ``--entity`` also keeps every ``fec_hoard`` id (empty entity key). + A rerun resumes from the hoard (data/market/fec).""" chosen_cycles = _parse_cycles(cycles) if not chosen_cycles: typer.echo("--cycles is empty") @@ -1367,6 +1368,36 @@ def fec( budget=budget, ) ) + if not entity and not budget.exhausted: + have = {row.committee_id for row in committees} + extra = [cid for cid in fec_api.hoard_ids() if cid not in have] + for row in fec_api.hoard_committees( + cl, extra, pacer=pacer, cache=root, budget=budget + ): + committees.append(row) + for cycle in chosen_cycles: + if budget.exhausted: + break + got = fec_api.totals( + cl, + row.committee_id, + cycle, + pacer=pacer, + cache=root, + budget=budget, + ) + if got is not None: + totals.append(got) + contribs.extend( + fec_api.contributions( + cl, + row.committee_id, + cycle, + pacer=pacer, + cache=root, + budget=budget, + ) + ) except http.Refused as exc: _gap(FEC_GAP, str(exc), write=write) return @@ -1375,8 +1406,10 @@ def fec( curated = sum(c.basis == "curated" for c in committees) exact = sum(c.basis == "exact-name" for c in committees) + hoarded = sum(c.basis == "hoarded" for c in committees) typer.echo( - f"{len(committees)} committees ({curated} curated, {exact} exact-name), " + f"{len(committees)} committees " + f"({curated} curated, {exact} exact-name, {hoarded} hoarded), " f"{len(candidates)} candidates, {len(totals)} totals, " f"{len(contribs)} schedule B rows; {budget.used} requests" ) @@ -1384,6 +1417,9 @@ def fec( typer.echo( f"candidate: {cand['entity_key']} {cand['committee_id']} {cand['name']}" ) + for row in committees: + if row.basis == "hoarded": + typer.echo(f"hoarded: {row.committee_id} {row.name}") if budget.exhausted: typer.echo(f"stopped: {budget.why} (rerun resumes from the hoard)") if not write: diff --git a/src/market/fec.py b/src/market/fec.py index dfb2ada..97293b9 100644 --- a/src/market/fec.py +++ b/src/market/fec.py @@ -4,9 +4,12 @@ The key (``FEC_API_KEY``, else ``API_DATA_GOV_KEY``) travels only in the ``X-Api-Key`` header. A committee is stored from a curated ``fec_committee_ids`` entry, or, failing that, when its normalised name equals an alias (``alias_match(prefix=False)``). Every other search hit is -a candidate: printed, not stored. Schedule B rows are stored as filed, -including an individual recipient's name; a memo row (``memo_code`` ``X``) -is stored and left out of the cycle sum. +a candidate: printed, not stored as that entity's committee. Committees +listed in the seed's ``fec_hoard`` are fetched in full on a pull of every +entity and stored with an empty entity key (basis ``hoarded``), so the +filing is kept and the money is not attributed to the alias that found it. +Schedule B rows are stored as filed, including an individual recipient's +name; a memo row (``memo_code`` ``X``) is stored and left out of the cycle sum. """ from __future__ import annotations @@ -21,7 +24,7 @@ from typing import Any import httpx from market import http -from market.seed import alias_match, load_seed, norm_org +from market.seed import alias_match, load_fec_hoard, load_seed, norm_org from market.tables import ( EntityRow, FecCommitteeRow, @@ -55,6 +58,11 @@ def committee_ids() -> dict[str, tuple[str, ...]]: return out +def hoard_ids() -> tuple[str, ...]: + """Committee ids to keep that are not a tracked entity's own PAC.""" + return load_fec_hoard() + + def search_names(name: str, aliases: str) -> list[str]: """The distinct (by normalised form) names to search: the entity's own name first, then its ``|``-joined aliases.""" @@ -289,6 +297,43 @@ def resolve( return list(found.values()), candidates +def hoard_committees( + cl: httpx.Client, + ids: Sequence[str], + *, + pacer: Any, + cache: Path | None = None, + refresh: bool = False, + budget: Budget | None = None, +) -> list[FecCommitteeRow]: + """Committee records for ids that are not a tracked entity's PAC. + + ``basis`` is ``hoarded`` and ``entity_key`` is empty, so a later sum by + entity does not pick the money up. A 404 id is skipped.""" + rows: list[FecCommitteeRow] = [] + for cid in ids: + if not cid: + continue + path = None if cache is None else cache / "committee" / f"{cid}.json" + got = _page( + cl, + f"{FEC_API}committee/{cid}/", + None, + pacer=pacer, + path=path, + refresh=refresh, + budget=budget, + ) + if got is None: + break + payload, src = got + for rec in payload.get("results") or (): + rows.append( + to_committee(rec, entity_key="", basis="hoarded", source_file=src) + ) + return rows + + def totals( cl: httpx.Client, cid: str, diff --git a/src/market/seed.py b/src/market/seed.py index fcacb80..df9fd98 100644 --- a/src/market/seed.py +++ b/src/market/seed.py @@ -211,6 +211,22 @@ def _org(raw: Mapping[str, Any]) -> SeedOrg: ) +def load_fec_hoard(path: Path | str | None = None) -> tuple[str, ...]: + """Committee ids the name search found that are not a tracked entity's + PAC. The FEC pull keeps their pages. A curated id belongs on the entity, + not in this list.""" + raw = yaml.safe_load(Path(path or SEED_PATH).read_text()) or {} + out: list[str] = [] + for item in raw.get("fec_hoard") or (): + cid = str(item.get("id") if isinstance(item, Mapping) else item).strip() + if len(cid) != 9 or not cid.startswith("C") or not cid[1:].isdigit(): + raise ValueError(f"fec_hoard: {cid!r} is not a committee id") + out.append(cid) + if len(out) != len(set(out)): + raise ValueError("fec_hoard: duplicate committee id") + return tuple(out) + + def load_gaps(path: Path | str | None = None) -> tuple[SeedGap, ...]: """Sources the seed records as unreadable (``gaps:`` in the YAML).""" raw = yaml.safe_load(Path(path or SEED_PATH).read_text()) or {} diff --git a/src/market/seed.yaml b/src/market/seed.yaml index 0330497..b294d7d 100644 --- a/src/market/seed.yaml +++ b/src/market/seed.yaml @@ -452,6 +452,8 @@ orgs: org_type: specialty_society specialty: "INTERNAL MEDICINE" aliases: ["AMERICAN COLLEGE OF PHYSICIANS"] + # Registered as AMERICAN COLLEGE OF PHYSICIAN SERVICES INC PAC (ACP SERVICES PAC). + fec_committee_ids: ["C00403881"] cite: {item_key: "N8RVIA96", title: "AMA RUC roster 2023 (saved copy): society row for INTERNAL MEDICINE", accessed: "2026-11-12"} - key: RPA name: "Renal Physicians Association" @@ -485,7 +487,8 @@ orgs: org_type: specialty_society specialty: "OPHTHALMOLOGY" aliases: ["AMERICAN ACADEMY OF OPHTHALMOLOGY"] - fec_committee_ids: ["C00196246"] + # C70003785 is the same-name communication-cost committee filed beside OPHTHPAC. + fec_committee_ids: ["C00196246", "C70003785"] cite: {item_key: "N8RVIA96", title: "AMA RUC roster 2023 (saved copy): society row for OPHTHALMOLOGY", accessed: "2026-11-12"} - key: AAOS name: "American Academy of Orthopaedic Surgeons" @@ -615,6 +618,8 @@ orgs: name: "American Osteopathic Association" org_type: professional_assoc aliases: ["AMERICAN OSTEOPATHIC ASSOCIATION"] + # FEC connects C00113803 to the American Osteopathic Information Association, the AOA PAC affiliate at 142 E. Ontario. + fec_committee_ids: ["C00113803"] cite: {url: "https://osteopathic.org/", title: "American Osteopathic Association official website", accessed: "2026-11-12"} - key: AHA name: "American Hospital Association" @@ -675,6 +680,23 @@ orgs: aliases: ["ONCOLOGY SEAT"] cite: {item_key: "KRWZGMGI", title: "AMA RUC roster 2026 (saved copy): oncology seat", accessed: "2026-11-12"} +# Name-search hits that are a different committee from the tracked entity's +# PAC. A full pull keeps each committee page, cycle totals, and Schedule B, +# stored with an empty entity key. NEMPAC C00140061 is already ACEP's curated +# id, so it is not repeated here. +fec_hoard: + - {id: "C00765230", name: "AMERICAN ANTHEM PAC", found_via: ELV} + - {id: "C00700799", name: "AMERICAN COLLEGE OF OSTEOPATHIC SURGEONS PAC", found_via: ACS} + - {id: "C00362152", name: "AMERICAN COLLEGE OF NUCLEAR PHYSICIANS NUCLEAR MEDICINE PAC", found_via: ACP} + - {id: "C00101493", name: "POLITICAL ACTION COMMITTEE OF THE AMERICAN OSTEOPATHIC HEALTHCARE ASSOCIATION", found_via: AOA} + - {id: "C00570291", name: "ASSOCIATION PROMOTING GOVERNMENTAL OBJECTIVE VIEWS APGOV", found_via: APG} + - {id: "C00941138", name: "AMERICAN INDEPENDENT MEDICAL PRACTICE ASSOCIATION PAC (AIMPA PAC)", found_via: AMA} + - {id: "C00190272", name: "AMERICAN MEDICAL CARE AND REVIEW ASSOCIATION POLITICAL ACTION COMMITTEE (AMCRA PAC)", found_via: AMA} + - {id: "C00898726", name: "AMERICAN MEDICAL MANUFACTURERS ASSOCIATION PAC (AMMA PAC)", found_via: AMA} + - {id: "C00678722", name: "AMERICAN MEDICAL REHABILITATION PROVIDERS ASSOCIATION INC. PAC (AMRPA PAC)", found_via: AMA} + - {id: "C00008839", name: "AMERICAN PODIATRIC MEDICAL ASSOCIATION POLITICAL ACTION COMMITTEE", found_via: AMA} + - {id: "C00114132", name: "AMERICAN VETERINARY MEDICAL ASSOCIATION POLITICAL ACTION COMMITTEE", found_via: AMA} + gaps: - {source: "web:renalmd.org", note: "HTTP 403 to stack-pfs-dossier/1.0 on 2026-11-12; Renal Physicians Association site not read, so it is not cited as accessed"} - {source: "web:asco.org", note: "HTTP 403 to stack-pfs-dossier/1.0 on 2026-11-12; American Society of Clinical Oncology site not read, so it is not cited as accessed"} diff --git a/src/market/tables.py b/src/market/tables.py index 84d9cb4..e81e8c3 100644 --- a/src/market/tables.py +++ b/src/market/tables.py @@ -286,8 +286,9 @@ class LdaIssueRow: @dataclass(frozen=True) class FecCommitteeRow: - """One FEC committee tied to an entity. ``basis`` is ``curated`` (a seed - id) or ``exact-name`` (the normalised name equals an alias).""" + """One FEC committee. ``basis`` is ``curated`` (a seed id), ``exact-name`` + (the normalised name equals an alias), or ``hoarded`` (a namesake kept + with an empty ``entity_key``).""" committee_id: str name: str diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index 3782c81..3f1a5ea 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -1930,6 +1930,7 @@ def _fec_entities(c): aliases="AMERICAN ACADEMY OF FAMILY PHYSICIANS", ), T.EntityRow("AMA", "AMA", "organization", aliases="AMA"), + T.EntityRow("LOOKALIKE", "LOOKALIKE", "organization", aliases="LOOKALIKE"), T.EntityRow( "PRIMARY-CARE-SEAT", "Primary care rotating-seat societies", @@ -1999,7 +2000,7 @@ def test_fec_cli_write_from_cache(fec): dry = runner.invoke(app, ["market", "fec", "--entity", "AAFP", "--cycles", "2024"]) assert dry.exit_code == 0, dry.output assert "dry run: pass --write" in dry.output - assert "1 committees (1 curated, 0 exact-name)" in dry.output + assert "1 committees (1 curated, 0 exact-name, 0 hoarded)" in dry.output assert _count(c, "market.fec_committee") == 0 assert published == [] assert seen and all("/committees" not in r.url.path for r in seen) @@ -2040,20 +2041,22 @@ def test_fec_cli_write_from_cache(fec): def test_fec_cli_no_write_prints_candidates(fec): c, _, published, _, seen, _ = fec - res = runner.invoke(app, ["market", "fec", "--entity", "AMA", "--cycles", "2024"]) + res = runner.invoke( + app, ["market", "fec", "--entity", "LOOKALIKE", "--cycles", "2024"] + ) assert res.exit_code == 0, res.output - assert "candidate: AMA C00999999 AMALGAMATED TRANSIT UNION PAC" in res.output + assert "candidate: LOOKALIKE C00999999 AMALGAMATED TRANSIT UNION PAC" in res.output assert "0 committees" in res.output and "1 candidates" in res.output assert "dry run: pass --write" in res.output assert _count(c, "market.fec_committee") == 0 assert _count(c, "market.fec_contribution") == 0 assert published == [] - assert seen[0].url.params["q"] == "AMA" + assert seen[0].url.params["q"] == "LOOKALIKE" assert all( "/committee/" not in r.url.path or r.url.path.startswith("/v1/committees") for r in seen ) - default = runner.invoke(app, ["market", "fec", "--entity", "AMA"]) + default = runner.invoke(app, ["market", "fec", "--entity", "LOOKALIKE"]) assert default.exit_code == 0, default.output assert "1 candidates" in default.output bad = runner.invoke(app, ["market", "fec", "--entity", "NOPE", "--cycles", "2024"]) @@ -2062,6 +2065,69 @@ def test_fec_cli_no_write_prints_candidates(fec): assert cycles.exit_code == 2 and "not a list of years" in cycles.output +def test_fec_cli_full_pull_keeps_hoard_ids(fec, monkeypatch): + from market import fec as fec_api + + c, _, published, hoard, seen, _ = fec + monkeypatch.setattr(fec_api, "hoard_ids", lambda: ("C00765230",)) + committee = json.loads((FX / "fec_committees_aafp.json").read_text()) + totals = json.loads((FX / "fec_totals_C00411553_2024.json").read_text()) + page1 = json.loads((FX / "fec_schedule_b_page1.json").read_text()) + + def handler(request): + seen.append(request) + path = request.url.path + if path == "/v1/committee/C00765230/": + return httpx.Response( + 200, + json={ + "results": [ + { + "committee_id": "C00765230", + "name": "AMERICAN ANTHEM PAC", + "committee_type": "V", + "designation": "U", + } + ] + }, + ) + cid = request.url.params.get("committee_id", "") + if path.endswith("/totals/"): + if cid == "C00765230": + return httpx.Response(200, json={"results": []}) + return httpx.Response(200, json=totals) + if path.endswith("/schedule_b/"): + if cid == "C00765230" or "last_index" in request.url.params: + return httpx.Response(200, json={"results": [], "pagination": {}}) + return httpx.Response(200, json=page1) + if path.startswith("/v1/committees"): + return httpx.Response(200, json={"results": []}) + if path.endswith("/C00411553/"): + return httpx.Response(200, json=committee) + return httpx.Response(200, json={"results": []}) + + monkeypatch.setattr( + http, + "client", + lambda source, key=None: httpx.Client( + transport=httpx.MockTransport(handler), + headers={"X-Api-Key": key} if key else {}, + ), + ) + res = runner.invoke(app, ["market", "fec", "--cycles", "2024", "--write"]) + assert res.exit_code == 0, res.output + assert "hoarded: C00765230 AMERICAN ANTHEM PAC" in res.output + rows = {r.committee_id: r for r in T.read_fec_committees(c)} + assert rows["C00765230"].basis == "hoarded" and rows["C00765230"].entity_key == "" + assert rows["C00411553"].basis == "curated" + assert (hoard / "fec" / "committee" / "C00765230.json").is_file() + from market.fec import contributions_by_cycle + + assert contributions_by_cycle(c, "") == {} + assert contributions_by_cycle(c, "AMA") == {} + assert published == [True] + + def test_fec_cli_refusal_prints_gap(fec, monkeypatch): c, _, published, _, seen, state = fec monkeypatch.setattr(http, "api_key", lambda source, env=None: None) diff --git a/tests/market/test_fec.py b/tests/market/test_fec.py index 080f601..0ac7f7d 100644 --- a/tests/market/test_fec.py +++ b/tests/market/test_fec.py @@ -91,6 +91,30 @@ def test_committee_exact_name_fallback_cited(): assert rows[0].name == PAC +def test_hoard_committee_is_kept_unattached(): + def handler(request): + assert request.url.path == "/v1/committee/C00765230/" + return httpx.Response( + 200, + json={ + "results": [ + { + "committee_id": "C00765230", + "name": "AMERICAN ANTHEM PAC", + "committee_type": "V", + "designation": "U", + } + ] + }, + ) + + rows = F.hoard_committees(_client(handler), ["C00765230"], pacer=Pacer()) + assert [(r.committee_id, r.basis, r.entity_key, r.name) for r in rows] == [ + ("C00765230", "hoarded", "", "AMERICAN ANTHEM PAC") + ] + assert rows[0].cite_url == "https://www.fec.gov/data/committee/C00765230/" + + def test_short_alias_never_matches_inside_names(): name = "AMALGAMATED TRANSIT UNION PAC" diff --git a/tests/market/test_seed.py b/tests/market/test_seed.py index a9a4d22..e5e92ed 100644 --- a/tests/market/test_seed.py +++ b/tests/market/test_seed.py @@ -394,7 +394,7 @@ def test_only_symbol_records_are_unpriced(): def test_shipped_fec_committee_ids_are_the_connected_pacs(): """Curated ids are the sponsor's own PAC. A short-alias search hit is not.""" - from market.fec import committee_ids + from market.fec import committee_ids, hoard_ids companies, orgs = S.load_seed() by_handle = {c.handle: c.fec_committee_ids for c in companies} @@ -405,7 +405,9 @@ def test_shipped_fec_committee_ids_are_the_connected_pacs(): by_org = {o.key: o.fec_committee_ids for o in orgs} assert by_org["AMA"] == ("C00000422",) assert by_org["AAFP"] == ("C00411553",) - assert by_org["ACP"] == () and by_org["AOA"] == () + assert by_org["ACP"] == ("C00403881",) + assert by_org["AOA"] == ("C00113803",) + assert by_org["AAO"] == ("C00196246", "C70003785") assert by_org["ACEP"] == ("C00140061",) expect = { "0000019584": ("C00279786",), @@ -430,7 +432,9 @@ def test_shipped_fec_committee_ids_are_the_connected_pacs(): "AAN": ("C00435933",), "AANS": ("C00413955",), "AAFP": ("C00411553",), - "AAO": ("C00196246",), + "AAO": ("C00196246", "C70003785"), + "ACP": ("C00403881",), + "AOA": ("C00113803",), "AAO-HNS": ("C00306449",), "ACC": ("C00375360",), "ACEP": ("C00140061",), @@ -456,20 +460,20 @@ def test_shipped_fec_committee_ids_are_the_connected_pacs(): got = committee_ids() assert got == expect flat = {cid for row in got.values() for cid in row} - refused = { - "C00765230", # American Anthem PAC, not Elevance - "C00700799", # osteopathic surgeons, not ACS - "C00362152", # nuclear physicians, not ACP - "C00113803", # osteopathic information association, not AOA - "C00101493", # osteopathic healthcare association, not AOA - "C00570291", # APGOV, not America's Physician Groups - "C00941138", # AIMPA - "C00190272", # AMCRA - "C00898726", # AMMA - "C00678722", # AMRPA - "C00008839", # APMA - "C00114132", # AVMA - "C70003785", # AAO communication-cost committee, not OPHTHPAC - } - assert flat.isdisjoint(refused) - assert "ACP" not in got and "AOA" not in got + hoard = S.load_fec_hoard() + assert hoard == ( + "C00765230", + "C00700799", + "C00362152", + "C00101493", + "C00570291", + "C00941138", + "C00190272", + "C00898726", + "C00678722", + "C00008839", + "C00114132", + ) + assert hoard_ids() == hoard + assert set(hoard).isdisjoint(flat) + assert "C00140061" not in hoard