diff --git a/src/cli/market.py b/src/cli/market.py index 83f4a1d..52680cc 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -20,6 +20,7 @@ from typing import Any import httpx import typer +from market import congress as congress_api from market import edgar, http, tiingo from market import fec as fec_api from market import lda as lda_api @@ -53,6 +54,8 @@ from market.tables import ( read_prices, retire_links, retire_organizations, + state_codes, + write_assignments, write_entities, write_fec_committees, write_fec_contributions, @@ -61,6 +64,7 @@ from market.tables import ( write_gap, write_hits, write_links, + write_members, write_passages, write_prices, write_securities, @@ -75,6 +79,8 @@ PRICES_GAP = "tiingo:prices" FTS_GAP = "sec-edgar:fts" LDA_GAP = "lda:lda" FEC_GAP = "fec:fec" +MEMBERS_GAP = "congress:members" +COMMITTEES_GAP = "legislators:committees" # ── indirections the tests monkeypatch ─────────────────────────────── @@ -1448,6 +1454,135 @@ def fec( typer.echo(f"stored: {n_c} committees, {n_t} totals, {n_b} schedule B rows") +def _congress_range(from_congress: int, to_congress: int) -> range: + if from_congress > to_congress: + typer.echo(f"--from {from_congress} is after --to {to_congress}") + raise typer.Exit(2) + return range(from_congress, to_congress + 1) + + +def _member_line(rows: list[Any]) -> str: + house = sum(r.chamber == "house" for r in rows) + senate = sum(r.chamber == "senate" for r in rows) + at_large = sum(r.district_id.endswith("-00") for r in rows) + return f"{len(rows)} members ({house} house, {senate} senate, {at_large} at-large)" + + +@app.command() +def members( + from_congress: int = typer.Option(congress_api.CONGRESSES.start, "--from"), + to_congress: int = typer.Option(congress_api.CONGRESSES.stop - 1, "--to"), + write: bool = typer.Option(False, "--write", help="Store market.congress_member."), +) -> None: + """Members of Congress (market.congress_member) for the 110th through the + 119th. Congress.gov supplies the roster; the legislators YAML supplies + FEC candidate ids and members the API does not return. A missing key + records the gap and writes nothing else.""" + congresses = _congress_range(from_congress, to_congress) + key = _require_key("congress", MEMBERS_GAP, write=write) + if key is None: + return + con = _read() + try: + states = state_codes(con) + finally: + con.close() + api = http.client("congress", key) + plain = http.client("legislators") + try: + try: + rows = congress_api.fetch_members( + api, + plain, + congresses, + states, + member_pacer=congress_api.member_pacer(), + raw_pacer=congress_api.raw_pacer(), + cache=http.hoard("congress"), + ) + except http.Refused as exc: + _gap(MEMBERS_GAP, str(exc), write=write) + return + finally: + api.close() + plain.close() + counts = Counter(r.congress for r in rows) + typer.echo(_member_line(rows)) + for n in congresses: + typer.echo(f"congress {n}: {counts[n]}") + if not write: + typer.echo("dry run: pass --write to store") + return + from cms.ingest_log import log_ingest + + with _batch() as con: + ensure_tables(con) + n = write_members(con, rows) + clear_gap(con, MEMBERS_GAP) + log_ingest( + con, + module="market.congress", + table_name="market.congress_member", + rows=n, + source_file=str(http.hoard(http.MANIFEST)), + ) + _publish() + typer.echo(f"stored: {n} members") + + +@app.command() +def committees( + from_congress: int = typer.Option( + congress_api.COMMITTEE_CONGRESSES.start, "--from" + ), + to_congress: int = typer.Option(congress_api.COMMITTEE_CONGRESSES.stop - 1, "--to"), + write: bool = typer.Option( + False, "--write", help="Store market.committee_assignment." + ), +) -> None: + """Health-committee assignments (market.committee_assignment), one + legislators snapshot per congress from the 113th on. Keyless. The GitHub + commit lookup is paced at 61 seconds.""" + congresses = _congress_range(from_congress, to_congress) + plain = http.client("legislators") + try: + try: + rows = congress_api.fetch_assignments( + plain, + congresses, + commit_pacer=congress_api.commit_pacer(), + raw_pacer=congress_api.raw_pacer(), + cache=http.hoard("congress"), + ) + except http.Refused as exc: + _gap(COMMITTEES_GAP, str(exc), write=write) + return + finally: + plain.close() + counts = Counter(r.congress for r in rows) + typer.echo(f"{len(rows)} assignments") + for n in congresses: + typer.echo(f"congress {n}: {counts[n]}") + if not write: + typer.echo("dry run: pass --write to store") + return + from cms.ingest_log import log_ingest + + with _batch() as con: + ensure_tables(con) + n = write_assignments(con, rows) + clear_gap(con, COMMITTEES_GAP) + log_ingest( + con, + module="market.congress", + table_name="market.committee_assignment", + rows=n, + source_file=str(http.hoard(http.MANIFEST)), + ) + _publish() + typer.echo(f"stored: {n} assignments") + + @app.command() def gaps() -> None: """Sources that could not be read (market.source_gap).""" diff --git a/src/market/congress.py b/src/market/congress.py new file mode 100644 index 0000000..c9aa531 --- /dev/null +++ b/src/market/congress.py @@ -0,0 +1,550 @@ +"""Members of Congress and health-committee assignments. + +Congress.gov (``X-Api-Key``, never the query string) lists who served. +``currentMember=false`` keeps members who have since left. The legislators +YAML (public domain, keyless) supplies FEC candidate ids and fills a +congress the API does not return. Committee seats are one +congress-legislators snapshot per congress: the last commit of +``committee-membership-current.yaml`` on or before April 1 of that +congress's first year (M7). ``pagination.next`` is never followed; it can +carry the key. +""" + +from __future__ import annotations + +import datetime as dt +import json +import os +from collections.abc import Mapping, Sequence +from dataclasses import replace +from pathlib import Path +from typing import Any + +import httpx +import yaml + +from market import http +from market.tables import AssignmentRow, MemberRow, district_id + +CONGRESS_API = "https://api.congress.gov/v3/" +LEGISLATORS = ( + "https://raw.githubusercontent.com/unitedstates/congress-legislators/{ref}/{name}" +) +GH_COMMITS = "https://api.github.com/repos/unitedstates/congress-legislators/commits" +REF_MAIN = "main" +CURRENT = "legislators-current.yaml" +HISTORICAL = "legislators-historical.yaml" +MEMBERSHIP = "committee-membership-current.yaml" + +CONGRESSES = range(110, 120) +COMMITTEE_CONGRESSES = range(113, 120) +HEALTH_COMMITTEES = ( + "HSWM", + "HSWM02", + "HSIF", + "HSIF14", + "SSFI", + "SSFI10", + "SSHR", + "SSHR12", +) +# Congress.gov sends district 0 for both an at-large state and a delegate. +# The canonical id for a non-voting delegate is POSTAL-98 (M7). +DELEGATES = frozenset({"AS", "DC", "GU", "MP", "PR", "VI"}) +LIMIT = 250 +MEMBER_PACE = 0.8 +COMMIT_PACE = 61.0 +RAW_PACE = 0.2 + + +def member_pacer() -> http.Pacer: + return http.Pacer(MEMBER_PACE) + + +def commit_pacer() -> http.Pacer: + return http.Pacer(COMMIT_PACE) + + +def raw_pacer() -> http.Pacer: + return http.Pacer(RAW_PACE) + + +def congress_of(year: int) -> int: + """The Congress in session during *year* (the 119th covers 2025–2026).""" + return ((year - 1789) // 2) + 1 + + +def first_year(congress: int) -> int: + """``1789 + 2·(congress − 1)``.""" + return 1789 + 2 * (congress - 1) + + +def snapshot_cutoff(congress: int) -> dt.date: + """April 1 of :func:`first_year` (the legislators snapshot for M7).""" + return dt.date(first_year(congress), 4, 1) + + +def congress_start(congress: int) -> dt.date: + """The day *congress* convened. The 74th Congress (1935) is the first + to open on January 3; earlier congresses open on March 4.""" + year = first_year(congress) + if congress >= 74: + return dt.date(year, 1, 3) + return dt.date(year, 3, 4) + + +def commit_params(congress: int) -> dict[str, Any]: + """GitHub commits query for the snapshot on or before the cutoff.""" + cutoff = snapshot_cutoff(congress) + return { + "path": MEMBERSHIP, + "until": f"{cutoff.isoformat()}T00:00:00Z", + "per_page": 1, + } + + +def _items(terms: Any) -> list[dict[str, Any]]: + if not isinstance(terms, dict): + return [] + item = terms.get("item") + if isinstance(item, dict): + return [item] + if isinstance(item, list): + return [x for x in item if isinstance(x, dict)] + return [] + + +def _chamber(items: Sequence[Mapping[str, Any]], congress: int) -> str: + """The chamber of the term that overlaps *congress*. A later overlapping + term wins. No term means the House.""" + y0 = first_year(congress) + y1 = y0 + 1 + chosen: Mapping[str, Any] | None = None + for item in items: + start = int(item.get("startYear") or 0) + end = item.get("endYear") + if start <= y1 and (end in (None, "") or int(end) >= y0): + chosen = item + text = str((chosen or {}).get("chamber") or "") + return "senate" if "senate" in text.lower() else "house" + + +def house_district(postal: str, number: Any) -> str: + """``AK-00`` for an at-large state; ``DC-98`` for a non-voting delegate.""" + if postal in DELEGATES: + return f"{postal}-98" + return district_id(postal, number) + + +def _postal(raw: str, states: Mapping[str, tuple[str, str, str]]) -> str | None: + hit = states.get(raw) + if hit is None: + return None + return hit[1] + + +def members_from_api( + payload: Mapping[str, Any], + congress: int, + states: Mapping[str, tuple[str, str, str]], +) -> list[MemberRow]: + """One Congress.gov member page. A state the reference table does not + know is skipped. The Senate has an empty ``district_id``.""" + rows: list[MemberRow] = [] + for member in payload.get("members") or []: + if not isinstance(member, dict): + continue + bioguide = str(member.get("bioguideId") or "") + postal = _postal(str(member.get("state") or ""), states) + if not bioguide or postal is None: + continue + chamber = _chamber(_items(member.get("terms")), congress) + district = ( + "" + if chamber == "senate" + else house_district(postal, member.get("district")) + ) + rows.append( + MemberRow( + congress=congress, + bioguide_id=bioguide, + name=str(member.get("name") or ""), + party=str(member.get("partyName") or ""), + state=postal, + district_id=district, + chamber=chamber, + fec_ids="", + source="congress.gov", + ) + ) + return rows + + +def _fec_tuple(person: Mapping[str, Any]) -> tuple[str, ...]: + raw = (person.get("id") or {}).get("fec") or [] + out: list[str] = [] + for item in raw: + text = str(item).strip() + if text and text not in out: + out.append(text) + return tuple(out) + + +def _covers(term: Mapping[str, Any], congress: int) -> bool: + """A legislators term is ``[start, end)``. *end* equal to the next + congress's first day does not include that congress.""" + start = dt.date.fromisoformat(str(term.get("start"))[:10]) + end_raw = term.get("end") + end = dt.date.fromisoformat(str(end_raw)[:10]) if end_raw else None + opened = congress_start(congress) + closed = congress_start(congress + 1) + return start < closed and (end is None or end > opened) + + +def _yaml_term(terms: Sequence[Mapping[str, Any]], congress: int) -> Mapping | None: + chosen: Mapping[str, Any] | None = None + for term in terms: + if not isinstance(term, dict) or not _covers(term, congress): + continue + if chosen is None or str(term.get("start")) > str(chosen.get("start")): + chosen = term + return chosen + + +def members_from_yaml( + docs: Sequence[Mapping[str, Any]], + congresses: Sequence[int], + states: Mapping[str, tuple[str, str, str]], +) -> list[MemberRow]: + """One row per requested congress a term covers. ``type: sen`` has an + empty ``district_id``; a representative gets ``district_id``.""" + rows: list[MemberRow] = [] + for person in docs: + if not isinstance(person, dict): + continue + bioguide = str((person.get("id") or {}).get("bioguide") or "") + if not bioguide: + continue + fec = "|".join(_fec_tuple(person)) + name = str((person.get("name") or {}).get("official_full") or "") + terms = person.get("terms") or [] + for congress in congresses: + term = _yaml_term(terms, congress) + if term is None: + continue + postal = _postal(str(term.get("state") or ""), states) + if postal is None: + continue + chamber = "senate" if str(term.get("type") or "") == "sen" else "house" + district = ( + "" + if chamber == "senate" + else house_district(postal, term.get("district")) + ) + rows.append( + MemberRow( + congress=congress, + bioguide_id=bioguide, + name=name, + party=str(term.get("party") or ""), + state=postal, + district_id=district, + chamber=chamber, + fec_ids=fec, + source="legislators", + ) + ) + return rows + + +def fec_ids_by_bioguide( + docs: Sequence[Mapping[str, Any]], +) -> dict[str, tuple[str, ...]]: + """FEC candidate ids from legislators YAML, keyed by bioguide.""" + out: dict[str, tuple[str, ...]] = {} + for person in docs: + if not isinstance(person, dict): + continue + bioguide = str((person.get("id") or {}).get("bioguide") or "") + ids = _fec_tuple(person) + if bioguide and ids: + out[bioguide] = ids + return out + + +def merge_members( + api_rows: Sequence[MemberRow], + yaml_rows: Sequence[MemberRow], + fec: Mapping[str, tuple[str, ...]], +) -> list[MemberRow]: + """API rows win. FEC ids overlay by bioguide. A YAML row is added only + when that congress does not already have the bioguide.""" + out: list[MemberRow] = [] + seen: set[tuple[int, str]] = set() + for row in api_rows: + ids = fec.get(row.bioguide_id, ()) + if ids: + row = replace(row, fec_ids="|".join(ids)) + out.append(row) + seen.add((row.congress, row.bioguide_id)) + for row in yaml_rows: + key = (row.congress, row.bioguide_id) + if key not in seen: + out.append(row) + seen.add(key) + return out + + +def membership_rows( + doc: Mapping[str, Any], + *, + congress: int, + snapshot: dt.date, + sha: str, +) -> list[AssignmentRow]: + """Health subcommittees only. A missing rank is 0 and a missing title + is empty.""" + rows: list[AssignmentRow] = [] + for committee_id, members in doc.items(): + if committee_id not in HEALTH_COMMITTEES or not isinstance(members, list): + continue + for member in members: + if not isinstance(member, dict): + continue + rank = member.get("rank") + rows.append( + AssignmentRow( + congress=congress, + snapshot=snapshot, + committee_id=str(committee_id), + bioguide_id=str(member.get("bioguide") or ""), + rank=0 if rank in (None, "") else int(rank), + title=str(member.get("title") or ""), + party=str(member.get("party") or ""), + source_sha=sha, + ) + ) + return rows + + +def _cached(path: Path | None) -> bytes | None: + if path is None: + return None + have = http.newest(path) + if have.is_file(): + return have.read_bytes() + return None + + +def _store(path: Path | None, body: bytes, url: str) -> None: + if path is None: + return + where = http.save(path, body) + os.utime(where) + http.manifest(where, url) + + +def _json_value(body: bytes, source: str, status: int) -> Any: + try: + return json.loads(body) + except ValueError: + raise http.Refused(source, status, "response is not JSON") from None + + +def _object(body: bytes, source: str, status: int) -> dict[str, Any]: + payload = _json_value(body, source, status) + if not isinstance(payload, dict): + raise http.Refused(source, status, "response is not JSON") + return payload + + +def _commit_list(body: bytes, status: int) -> list[Any]: + payload = _json_value(body, "legislators", status) + if not isinstance(payload, list) or not payload: + raise http.Refused("legislators", status, "no commit on or before the cutoff") + return payload + + +def _yaml_doc(body: bytes, expect: type, status: int) -> Any: + try: + doc = yaml.safe_load(body) + except yaml.YAMLError: + doc = None + kind = "list" if expect is list else "map" + if not isinstance(doc, expect): + raise http.Refused("legislators", status, f"response is not a YAML {kind}") + return doc + + +def _download( + cl: httpx.Client, + url: str, + params: Mapping[str, Any] | None, + *, + pacer: Any, + path: Path | None, + source: str, +) -> bytes: + cached = _cached(path) + if cached is not None: + return cached + response = http.get(cl, url, params=params, pacer=pacer, source=source) + _store(path, response.content, str(response.request.url)) + return response.content + + +def _member_path(cache: Path | None, congress: int, offset: int) -> Path | None: + if cache is None: + return None + return cache / "members" / str(congress) / f"{offset}.json" + + +def _leg_path(cache: Path | None, name: str) -> Path | None: + if cache is None: + return None + return cache / "legislators" / name + + +def _commit_path(cache: Path | None, congress: int) -> Path | None: + if cache is None: + return None + day = snapshot_cutoff(congress).isoformat() + return cache / "commits" / MEMBERSHIP / f"{day}.json" + + +def _membership_path(cache: Path | None, sha: str) -> Path | None: + if cache is None: + return None + return cache / "membership" / f"{sha}.yaml" + + +def _pages( + api: httpx.Client, + congress: int, + states: Mapping[str, tuple[str, str, str]], + *, + pacer: Any, + cache: Path | None, +) -> list[MemberRow]: + rows: list[MemberRow] = [] + offset = 0 + seen_first: str | None = None + got = 0 + while True: + url = f"{CONGRESS_API}member/congress/{congress}" + body = _download( + api, + url, + { + "limit": LIMIT, + "offset": offset, + "format": "json", + "currentMember": "false", + }, + pacer=pacer, + path=_member_path(cache, congress, offset), + source="congress", + ) + payload = _object(body, "congress", 200) + members = payload.get("members") or [] + if not isinstance(members, list): + members = [] + first = "" + if members and isinstance(members[0], dict): + first = str(members[0].get("bioguideId") or "") + if first and first == seen_first: + break + if first: + seen_first = first + rows.extend(members_from_api(payload, congress, states)) + got += len(members) if isinstance(members, list) else 0 + count = (payload.get("pagination") or {}).get("count") + if ( + not members + or len(members) < LIMIT + or (count is not None and got >= int(count)) + ): + break + offset += LIMIT + return rows + + +def _legislators( + plain: httpx.Client, *, pacer: Any, cache: Path | None +) -> list[Mapping[str, Any]]: + docs: list[Mapping[str, Any]] = [] + for name in (CURRENT, HISTORICAL): + url = LEGISLATORS.format(ref=REF_MAIN, name=name) + body = _download( + plain, + url, + None, + pacer=pacer, + path=_leg_path(cache, name), + source="legislators", + ) + doc = _yaml_doc(body, list, 200) + docs.extend(person for person in doc if isinstance(person, dict)) + return docs + + +def fetch_members( + api: httpx.Client, + plain: httpx.Client, + congresses: Sequence[int], + states: Mapping[str, tuple[str, str, str]], + *, + member_pacer: Any, + raw_pacer: Any, + cache: Path | None = None, +) -> list[MemberRow]: + """Congress.gov pages for *congresses*, then the legislators YAML for + FEC ids and members the API did not return.""" + api_rows: list[MemberRow] = [] + for congress in congresses: + api_rows.extend(_pages(api, congress, states, pacer=member_pacer, cache=cache)) + docs = _legislators(plain, pacer=raw_pacer, cache=cache) + yaml_rows = members_from_yaml(docs, list(congresses), states) + return merge_members(api_rows, yaml_rows, fec_ids_by_bioguide(docs)) + + +def fetch_assignments( + plain: httpx.Client, + congresses: Sequence[int], + *, + commit_pacer: Any, + raw_pacer: Any, + cache: Path | None = None, +) -> list[AssignmentRow]: + """One health-committee snapshot per congress. An empty commit list is + a refusal, so a missing history is not stored as an empty snapshot.""" + rows: list[AssignmentRow] = [] + for congress in congresses: + body = _download( + plain, + GH_COMMITS, + commit_params(congress), + pacer=commit_pacer, + path=_commit_path(cache, congress), + source="legislators", + ) + commits = _commit_list(body, 200) + sha = str(commits[0].get("sha") or "") + if not sha: + raise http.Refused("legislators", 200, "no commit on or before the cutoff") + url = LEGISLATORS.format(ref=sha, name=MEMBERSHIP) + raw = _download( + plain, + url, + None, + pacer=raw_pacer, + path=_membership_path(cache, sha), + source="legislators", + ) + doc = _yaml_doc(raw, dict, 200) + rows.extend( + membership_rows( + doc, congress=congress, snapshot=snapshot_cutoff(congress), sha=sha + ) + ) + return rows diff --git a/src/market/tables.py b/src/market/tables.py index e81e8c3..e735a23 100644 --- a/src/market/tables.py +++ b/src/market/tables.py @@ -85,6 +85,15 @@ CREATE TABLE IF NOT EXISTS market.fec_contribution ( candidate_id VARCHAR, candidate_name VARCHAR, candidate_office VARCHAR, candidate_state VARCHAR, candidate_district VARCHAR, memo_code VARCHAR, source_file VARCHAR +); +CREATE TABLE IF NOT EXISTS market.congress_member ( + congress INTEGER, bioguide_id VARCHAR, name VARCHAR, party VARCHAR, + state VARCHAR, district_id VARCHAR, chamber VARCHAR, fec_ids VARCHAR, + source VARCHAR +); +CREATE TABLE IF NOT EXISTS market.committee_assignment ( + congress INTEGER, snapshot DATE, committee_id VARCHAR, bioguide_id VARCHAR, + rank INTEGER, title VARCHAR, party VARCHAR, source_sha VARCHAR ) """ @@ -339,6 +348,38 @@ class FecContributionRow: source_file: str +@dataclass(frozen=True) +class MemberRow: + """One member of one Congress. ``fec_ids`` is a ``|``-joined list. + ``district_id`` is ``MI-12`` or at-large ``AK-00`` for the House and + ``""`` for the Senate. ``state`` is the postal code.""" + + congress: int + bioguide_id: str + name: str + party: str + state: str + district_id: str + chamber: str + fec_ids: str + source: str + + +@dataclass(frozen=True) +class AssignmentRow: + """One health-committee seat in one congress's legislators snapshot. + ``snapshot`` is April 1 of that congress's first year (M7).""" + + congress: int + snapshot: date + committee_id: str + bioguide_id: str + rank: int + title: str + party: str + source_sha: str + + def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int: """Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is a statement per row, far too slow for filing lists).""" @@ -774,6 +815,53 @@ def read_fec_contributions( ) +# ── congress ───────────────────────────────────────────────────────── + + +def write_members(con: Any, rows: Sequence[MemberRow]) -> int: + """Replace the members of each congress present in *rows*.""" + ensure_tables(con) + if not rows: + return 0 + for chunk in _chunks(sorted({r.congress for r in rows})): + con.execute( + "DELETE FROM market.congress_member WHERE congress IN " + f"({','.join('?' * len(chunk))})", + list(chunk), + ) + return _insert(con, "market.congress_member", rows) + + +def read_members(con: Any) -> list[MemberRow]: + rows = _select(con, MemberRow, "market.congress_member") + return sorted( + (MemberRow(*r) for r in rows), + key=lambda r: (r.congress, r.bioguide_id, r.district_id), + ) + + +def write_assignments(con: Any, rows: Sequence[AssignmentRow]) -> int: + """Replace the assignments of each congress present in *rows*.""" + ensure_tables(con) + if not rows: + return 0 + for chunk in _chunks(sorted({r.congress for r in rows})): + con.execute( + "DELETE FROM market.committee_assignment WHERE congress IN " + f"({','.join('?' * len(chunk))})", + list(chunk), + ) + return _insert(con, "market.committee_assignment", rows) + + +def read_assignments(con: Any) -> list[AssignmentRow]: + rows = _select(con, AssignmentRow, "market.committee_assignment") + return sorted( + (AssignmentRow(*r) for r in rows), + key=lambda r: (r.congress, r.committee_id, r.rank, r.bioguide_id), + ) + + # ── districts ──────────────────────────────────────────────────────── diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index a13e81f..a4201f1 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -2181,3 +2181,205 @@ def test_fec_cli_limit_keeps_the_gap_and_empty_registry(fec, monkeypatch): assert empty.exit_code == 1 and "no tracked entities" in empty.output blank = runner.invoke(app, ["market", "fec", "--cycles", ","]) assert blank.exit_code == 2 and "--cycles is empty" in _plain(blank.output) + + +# ── congress ───────────────────────────────────────────────────────── + +CONGRESS_SECRET = "congress-cli-key-do-not-leak" + + +def _congress_states(c) -> None: + c.execute("CREATE SCHEMA IF NOT EXISTS reference_data") + c.execute( + "CREATE TABLE IF NOT EXISTS reference_data.ansi_fips_state (" + "ansi_fips_state_code VARCHAR, ansi_fips_state_abbreviation VARCHAR, " + "ansi_fips_state_name VARCHAR)" + ) + c.execute( + "INSERT INTO reference_data.ansi_fips_state VALUES " + "('26', 'MI', 'Michigan'), ('02', 'AK', 'Alaska'), ('51', 'VA', 'Virginia')" + ) + + +@pytest.fixture +def congress(env, monkeypatch): + from market import congress as congress_api + + c, store, published, hoard = env + _congress_states(c) + T.ensure_tables(c) + seen: list[httpx.Request] = [] + members = json.loads((FX / "congress_members_118.json").read_text()) + people = (FX / "legislators_snippet.yaml").read_bytes() + membership = (FX / "membership_snippet.yaml").read_bytes() + state = {"status": 200} + for name in ("member_pacer", "raw_pacer", "commit_pacer"): + monkeypatch.setattr(congress_api, name, lambda: _Pacer()) + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + if state["status"] != 200: + return httpx.Response(state["status"], text="no") + host = request.url.host + if host == "api.congress.gov": + return httpx.Response(200, json=members) + if host == "api.github.com": + return httpx.Response(200, json=[{"sha": "deadbeef"}]) + if request.url.path.endswith("legislators-historical.yaml"): + return httpx.Response(200, content=b"[]\n") + if request.url.path.endswith("committee-membership-current.yaml"): + return httpx.Response(200, content=membership) + return httpx.Response(200, content=people) + + def client(source, key=None): + headers = {"X-Api-Key": key} if key else {} + return httpx.Client(transport=httpx.MockTransport(handler), headers=headers) + + monkeypatch.setattr(http, "client", client) + monkeypatch.setattr(http, "api_key", lambda source, env=None: CONGRESS_SECRET) + return c, store, published, hoard, seen, state + + +def test_members_cli_no_write_prints_counts(congress): + c, _, published, _, seen, _ = congress + res = runner.invoke(app, ["market", "members", "--from", "118", "--to", "118"]) + assert res.exit_code == 0, res.output + assert "2 members (2 house, 0 senate, 1 at-large)" in res.output + assert "congress 118: 2" in res.output + assert "dry run: pass --write" in res.output + assert _count(c, "market.congress_member") == 0 + assert published == [] + assert any(r.url.host == "api.congress.gov" for r in seen) + assert all(CONGRESS_SECRET not in str(r.url) for r in seen) + assert all("api_key" not in str(r.url) for r in seen) + congress_hits = [r for r in seen if r.url.host == "api.congress.gov"] + assert congress_hits[0].headers["x-api-key"] == CONGRESS_SECRET + github_hits = [r for r in seen if r.url.host != "api.congress.gov"] + assert github_hits + assert all(r.headers.get("x-api-key") in (None, "") for r in github_hits) + + +def test_members_cli_write(congress): + c, _, published, hoard, seen, _ = congress + T.write_gap(c, "congress:members", "old") + res = runner.invoke( + app, ["market", "members", "--from", "118", "--to", "118", "--write"] + ) + assert res.exit_code == 0, res.output + assert "stored: 2 members" in res.output + rows = {r.bioguide_id: r for r in T.read_members(c)} + assert rows["D000624"].district_id == "MI-12" + assert rows["D000624"].fec_ids == "H0XX00001" + assert rows["D000624"].source == "congress.gov" + assert rows["Y000033"].district_id == "AK-00" + assert rows["Y000033"].state == "AK" + assert T.read_gaps(c) == [] + assert published == [True] + assert { + (t, n) + for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall() + } == {("market.congress_member", 2)} + manifest = (hoard / "MANIFEST.jsonl").read_text() + assert CONGRESS_SECRET not in manifest + assert all(CONGRESS_SECRET not in str(r.url) for r in seen) + kept = T.MemberRow( + 117, + "Z000001", + "Other", + "Republican", + "MI", + "MI-01", + "house", + "", + "congress.gov", + ) + T.write_members(c, [kept]) + assert {r.congress for r in T.read_members(c)} == {117, 118} + + +def test_members_cli_missing_key_gap_only(congress, monkeypatch): + c, _, published, _, seen, _ = congress + monkeypatch.setattr(http, "api_key", lambda source, env=None: None) + res = runner.invoke( + app, ["market", "members", "--from", "118", "--to", "118", "--write"] + ) + assert res.exit_code == 0, res.output + assert "gap: congress:members:" in res.output + assert "CONGRESS_API_KEY" in res.output + assert "API_DATA_GOV_KEY" in res.output + assert seen == [] + assert [g.source for g in T.read_gaps(c)] == ["congress:members"] + assert _count(c, "market.congress_member") == 0 + assert published == [True] + + +def test_members_cli_refusal_prints_gap(congress): + c, _, published, _, seen, state = congress + state["status"] = 403 + res = runner.invoke( + app, ["market", "members", "--from", "118", "--to", "118", "--write"] + ) + assert res.exit_code == 0, res.output + assert "gap: congress:members: congress: HTTP 403" in res.output + assert CONGRESS_SECRET not in res.output + assert _count(c, "market.congress_member") == 0 + assert len(seen) == 1 + assert seen[0].url.host == "api.congress.gov" + assert seen[0].headers["x-api-key"] == CONGRESS_SECRET + assert CONGRESS_SECRET not in str(seen[0].url) + + +def test_members_cli_from_after_to(congress): + res = runner.invoke(app, ["market", "members", "--from", "119", "--to", "110"]) + assert res.exit_code == 2 + assert "--from 119 is after --to 110" in res.output + + +def test_committees_cli_write(congress): + c, _, published, hoard, seen, _ = congress + T.write_gap(c, "legislators:committees", "old") + dry = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "119"]) + assert dry.exit_code == 0, dry.output + assert "1 assignments" in dry.output + assert "dry run: pass --write" in dry.output + assert _count(c, "market.committee_assignment") == 0 + assert published == [] + res = runner.invoke( + app, ["market", "committees", "--from", "119", "--to", "119", "--write"] + ) + assert res.exit_code == 0, res.output + assert "stored: 1 assignments" in res.output + (row,) = T.read_assignments(c) + assert row.committee_id == "HSWM02" + assert row.bioguide_id == "E000001" + assert row.snapshot == date(2025, 4, 1) + assert row.source_sha == "deadbeef" + assert row.congress == 119 + assert T.read_gaps(c) == [] + assert published == [True] + assert all(CONGRESS_SECRET not in str(r.url) for r in seen) + assert all(r.headers.get("x-api-key") in (None, "") for r in seen) + manifest = (hoard / "MANIFEST.jsonl").read_text() + assert CONGRESS_SECRET not in manifest + logged = c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall() + assert ("market.committee_assignment", 1) in logged + + +def test_committees_cli_refusal_prints_gap(congress): + c, _, published, _, seen, state = congress + state["status"] = 403 + res = runner.invoke( + app, ["market", "committees", "--from", "119", "--to", "119", "--write"] + ) + assert res.exit_code == 0, res.output + assert "gap: legislators:committees: legislators: HTTP 403" in res.output + assert CONGRESS_SECRET not in res.output + assert _count(c, "market.committee_assignment") == 0 + assert seen + assert published == [True] + + +def test_committees_cli_from_after_to(congress): + res = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "113"]) + assert res.exit_code == 2 + assert "--from 119 is after --to 113" in res.output diff --git a/tests/market/fixtures/congress_members_118.json b/tests/market/fixtures/congress_members_118.json new file mode 100644 index 0000000..fca1f40 --- /dev/null +++ b/tests/market/fixtures/congress_members_118.json @@ -0,0 +1,29 @@ +{ + "members": [ + { + "bioguideId": "D000624", + "district": 12, + "name": "Dingell, Debbie", + "partyName": "Democratic", + "state": "Michigan", + "terms": { + "item": [ + {"chamber": "House of Representatives", "startYear": 2023} + ] + } + }, + { + "bioguideId": "Y000033", + "district": null, + "name": "Young, Don", + "partyName": "Republican", + "state": "Alaska", + "terms": { + "item": [ + {"chamber": "House of Representatives", "startYear": 2021} + ] + } + } + ], + "pagination": {"count": 2} +} diff --git a/tests/market/fixtures/legislators_snippet.yaml b/tests/market/fixtures/legislators_snippet.yaml new file mode 100644 index 0000000..57c074f --- /dev/null +++ b/tests/market/fixtures/legislators_snippet.yaml @@ -0,0 +1,35 @@ +- id: + bioguide: D000624 + fec: + - H0XX00001 + name: + official_full: Debbie Dingell + terms: + - type: rep + start: '2023-01-03' + end: '2025-01-03' + state: MI + district: 12 + party: Democrat +- id: + bioguide: V000001 + fec: [] + name: + official_full: Virginia Rep + terms: + - type: rep + start: '1789-03-04' + end: '1791-03-03' + state: VA + district: 9 + party: Federalist +- id: + bioguide: S000001 + name: + official_full: Virginia Senator + terms: + - type: sen + start: '1789-03-04' + end: '1791-03-03' + state: VA + party: Federalist diff --git a/tests/market/fixtures/membership_snippet.yaml b/tests/market/fixtures/membership_snippet.yaml new file mode 100644 index 0000000..8f99609 --- /dev/null +++ b/tests/market/fixtures/membership_snippet.yaml @@ -0,0 +1,11 @@ +HSWM02: +- name: Example, Member + party: majority + rank: 1 + title: Chair + bioguide: E000001 +HSAG: +- name: Other, Member + party: minority + rank: 2 + bioguide: X000001 diff --git a/tests/market/test_congress.py b/tests/market/test_congress.py new file mode 100644 index 0000000..7d39958 --- /dev/null +++ b/tests/market/test_congress.py @@ -0,0 +1,677 @@ +"""market.congress: members by district and health-committee snapshots.""" + +from __future__ import annotations + +import json +from datetime import date +from pathlib import Path + +import httpx +import pytest +import yaml + +from market import congress as C +from market import http +from market import tables as T + +FX = Path(__file__).parent / "fixtures" +STATES = { + "Michigan": ("26", "MI", "Michigan"), + "MI": ("26", "MI", "Michigan"), + "Alaska": ("02", "AK", "Alaska"), + "AK": ("02", "AK", "Alaska"), + "Virginia": ("51", "VA", "Virginia"), + "VA": ("51", "VA", "Virginia"), +} + + +def _people() -> list: + return yaml.safe_load((FX / "legislators_snippet.yaml").read_text()) + + +def _page() -> dict: + return json.loads((FX / "congress_members_118.json").read_text()) + + +class Pacer: + def wait(self): + pass + + +def _client(handler) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def test_congress_of_year(): + assert C.congress_of(2013) == 113 + assert C.congress_of(2026) == 119 + assert C.congress_of(2027) == 120 + assert C.first_year(119) == 2025 + assert C.first_year(1) == 1789 + assert C.member_pacer().min_interval == 0.8 + assert C.commit_pacer().min_interval == 61 + assert C.raw_pacer().min_interval == 0.2 + assert C.LIMIT == 250 + assert C.CONGRESSES == range(110, 120) + assert C.COMMITTEE_CONGRESSES == range(113, 120) + + +def test_snapshot_cutoff(): + assert C.snapshot_cutoff(119) == date(2025, 4, 1) + assert C.snapshot_cutoff(113) == date(2013, 4, 1) + + +def test_members_from_api_page(): + rows = C.members_from_api(_page(), 118, STATES) + dingell = next(r for r in rows if r.bioguide_id == "D000624") + assert dingell.state == "MI" + assert dingell.district_id == "MI-12" + assert dingell.chamber == "house" + assert dingell.party == "Democratic" + assert dingell.name == "Dingell, Debbie" + assert dingell.congress == 118 + assert dingell.source == "congress.gov" + assert dingell.fec_ids == "" + + +def test_delegate_district_is_98_and_at_large_state_stays_00(): + states = { + **STATES, + "District of Columbia": ("11", "DC", "District of Columbia"), + "DC": ("11", "DC", "District of Columbia"), + } + payload = { + "members": [ + { + "bioguideId": "N000147", + "district": 0, + "name": "Norton, Eleanor Holmes", + "partyName": "Democratic", + "state": "District of Columbia", + "terms": { + "item": [{"chamber": "House of Representatives", "startYear": 1991}] + }, + }, + { + "bioguideId": "B001323", + "district": 0, + "name": "Begich, Nicholas J.", + "partyName": "Republican", + "state": "Alaska", + "terms": { + "item": [{"chamber": "House of Representatives", "startYear": 2025}] + }, + }, + ] + } + rows = {r.bioguide_id: r for r in C.members_from_api(payload, 119, states)} + assert rows["N000147"].district_id == "DC-98" + assert rows["N000147"].chamber == "house" + assert rows["B001323"].district_id == "AK-00" + docs = [ + { + "id": {"bioguide": "N000147"}, + "name": {"official_full": "Eleanor Holmes Norton"}, + "terms": [ + { + "type": "rep", + "start": "2025-01-03", + "end": "2027-01-03", + "state": "DC", + "district": 0, + "party": "Democrat", + } + ], + } + ] + (yaml_row,) = C.members_from_yaml(docs, [119], states) + assert yaml_row.district_id == "DC-98" + + +def test_at_large_member_district(): + rows = C.members_from_api(_page(), 118, STATES) + young = next(r for r in rows if r.bioguide_id == "Y000033") + assert young.state == "AK" + assert young.district_id == "AK-00" + assert young.chamber == "house" + + +def test_senate_member_has_empty_district(): + payload = { + "members": [ + { + "bioguideId": "S000033", + "district": None, + "name": "Stabenow, Debbie", + "partyName": "Democratic", + "state": "Michigan", + "terms": {"item": [{"chamber": "Senate", "startYear": 2001}]}, + } + ] + } + (row,) = C.members_from_api(payload, 118, STATES) + assert row.chamber == "senate" + assert row.district_id == "" + assert row.state == "MI" + + +def test_api_skips_unknown_state_and_accepts_a_single_term_dict(): + payload = { + "members": [ + { + "bioguideId": "Z000001", + "district": 1, + "name": "Nobody", + "partyName": "Independent", + "state": "Narnia", + "terms": { + "item": [{"chamber": "House of Representatives", "startYear": 2023}] + }, + }, + { + "bioguideId": "D000001", + "district": 1, + "name": "Delegate, One", + "partyName": "Democratic", + "state": "Michigan", + "terms": { + "item": {"chamber": "House of Representatives", "startYear": 2023} + }, + }, + { + "bioguideId": "H000001", + "district": 2, + "name": "No Terms", + "partyName": "Republican", + "state": "Michigan", + }, + ] + } + rows = C.members_from_api(payload, 118, STATES) + assert [r.bioguide_id for r in rows] == ["D000001", "H000001"] + assert rows[0].district_id == "MI-01" + assert rows[1].chamber == "house" + assert rows[1].district_id == "MI-02" + + +def test_api_chamber_follows_the_term_overlapping_the_congress(): + payload = { + "members": [ + { + "bioguideId": "B000001", + "district": 4, + "name": "Both, Chambers", + "partyName": "Republican", + "state": "Michigan", + "terms": { + "item": [ + { + "chamber": "House of Representatives", + "startYear": 2013, + "endYear": 2019, + }, + {"chamber": "Senate", "startYear": 2019}, + ] + }, + } + ] + } + house = C.members_from_api(payload, 115, STATES)[0] + senate = C.members_from_api(payload, 116, STATES)[0] + assert house.chamber == "house" and house.district_id == "MI-04" + assert senate.chamber == "senate" and senate.district_id == "" + + +def test_members_from_legislators_yaml(): + docs = [ + *_people(), + {"id": {}, "name": {"official_full": "No Id"}, "terms": []}, + { + "id": {"bioguide": "N000009"}, + "name": {"official_full": "Narnia"}, + "terms": [ + { + "type": "rep", + "start": "1789-03-04", + "end": "1791-03-03", + "state": "Narnia", + "district": 1, + } + ], + }, + ] + rows = C.members_from_yaml(docs, [1], STATES) + by_id = {r.bioguide_id: r for r in rows} + assert set(by_id) == {"V000001", "S000001"} + assert by_id["V000001"].district_id == "VA-09" + assert by_id["V000001"].state == "VA" + assert by_id["V000001"].chamber == "house" + assert by_id["V000001"].source == "legislators" + assert by_id["V000001"].congress == 1 + assert by_id["S000001"].district_id == "" + assert by_id["S000001"].chamber == "senate" + later = C.members_from_yaml(docs, [1, 2], STATES) + assert {r.congress for r in later} == {1} + modern = C.members_from_yaml(docs, [118], STATES) + assert [r.bioguide_id for r in modern] == ["D000624"] + assert modern[0].district_id == "MI-12" + assert modern[0].fec_ids == "H0XX00001" + + +def test_yaml_term_ending_on_the_next_congress_start_is_excluded(): + docs = [ + { + "id": {"bioguide": "N000001", "fec": ["H0AA00001", "H0AA00001"]}, + "name": {"official_full": "Next Boundary"}, + "terms": [ + { + "type": "rep", + "start": "2025-01-03", + "end": "2027-01-03", + "state": "VA", + "district": 1, + "party": "Democrat", + } + ], + } + ] + rows = C.members_from_yaml(docs, [119, 120], STATES) + assert [(r.congress, r.district_id, r.fec_ids) for r in rows] == [ + (119, "VA-01", "H0AA00001") + ] + + +def test_fec_ids_by_bioguide(): + ids = C.fec_ids_by_bioguide(_people()) + assert ids["D000624"] == ("H0XX00001",) + assert set(ids) == {"D000624"} + + +def test_merge_members_overlays_fec_ids_and_adds_the_api_gap(): + api = C.members_from_api(_page(), 118, STATES) + yaml_rows = C.members_from_yaml(_people(), [1, 118], STATES) + fec = C.fec_ids_by_bioguide(_people()) + merged = C.merge_members(api, yaml_rows, fec) + by = {(r.congress, r.bioguide_id): r for r in merged} + assert by[(118, "D000624")].fec_ids == "H0XX00001" + assert by[(118, "D000624")].source == "congress.gov" + assert (118, "D000624") in by and (118, "Y000033") in by + assert (1, "V000001") in by and (1, "S000001") in by + assert (118, "V000001") not in by + assert len(merged) == len(by) + + +def test_membership_snapshot_rows(): + doc = yaml.safe_load((FX / "membership_snippet.yaml").read_text()) + rows = C.membership_rows(doc, congress=119, snapshot=date(2025, 4, 1), sha="abc123") + assert [r.committee_id for r in rows] == ["HSWM02"] + assert rows[0].bioguide_id == "E000001" + assert rows[0].rank == 1 + assert rows[0].title == "Chair" + assert rows[0].party == "majority" + assert rows[0].source_sha == "abc123" + assert rows[0].snapshot == date(2025, 4, 1) + assert C.HEALTH_COMMITTEES == ( + "HSWM", + "HSWM02", + "HSIF", + "HSIF14", + "SSFI", + "SSFI10", + "SSHR", + "SSHR12", + ) + + +def test_membership_row_defaults_missing_rank_and_title(): + rows = C.membership_rows( + {"HSIF14": [{"bioguide": "A000001", "party": "minority"}]}, + congress=119, + snapshot=date(2025, 4, 1), + sha="s", + ) + assert rows[0].rank == 0 and rows[0].title == "" and rows[0].party == "minority" + + +def test_github_commit_lookup_params(): + params = C.commit_params(119) + assert params["until"] == "2025-04-01T00:00:00Z" + assert params["path"] == "committee-membership-current.yaml" + assert params["per_page"] == 1 + assert C.commit_params(113)["until"] == "2013-04-01T00:00:00Z" + + +def test_fetch_assignments_uses_the_cutoff_and_keeps_the_sha(monkeypatch, tmp_path): + monkeypatch.setattr(http, "hoard_root", lambda: tmp_path) + seen: list[httpx.Request] = [] + membership = (FX / "membership_snippet.yaml").read_bytes() + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + if request.url.host == "api.github.com": + return httpx.Response(200, json=[{"sha": "deadbeef"}]) + return httpx.Response(200, content=membership) + + rows = C.fetch_assignments( + _client(handler), + [119], + commit_pacer=Pacer(), + raw_pacer=Pacer(), + cache=tmp_path / "congress", + ) + assert seen[0].url.params["until"] == "2025-04-01T00:00:00Z" + assert seen[0].url.params["path"] == "committee-membership-current.yaml" + assert seen[0].url.params["per_page"] == "1" + assert seen[1].url.path.endswith("/deadbeef/committee-membership-current.yaml") + assert [(r.committee_id, r.source_sha, r.snapshot) for r in rows] == [ + ("HSWM02", "deadbeef", date(2025, 4, 1)) + ] + n = len(seen) + again = C.fetch_assignments( + _client(handler), + [119], + commit_pacer=Pacer(), + raw_pacer=Pacer(), + cache=tmp_path / "congress", + ) + assert again == rows + assert len(seen) == n + + +def test_empty_commit_list_is_refused(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=[]) + + with pytest.raises(http.Refused, match="no commit"): + C.fetch_assignments( + _client(handler), + [119], + commit_pacer=Pacer(), + raw_pacer=Pacer(), + ) + + +def test_legislators_yaml_that_is_not_a_list_is_refused(): + def api(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"members": [], "pagination": {"count": 0}}) + + def plain(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b"123\n") + + with pytest.raises(http.Refused, match="YAML list"): + C.fetch_members( + _client(api), + _client(plain), + [118], + STATES, + member_pacer=Pacer(), + raw_pacer=Pacer(), + ) + + +def test_membership_yaml_that_is_not_a_map_is_refused(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.host == "api.github.com": + return httpx.Response(200, json=[{"sha": "abc"}]) + return httpx.Response(200, content=b"- just\n") + + with pytest.raises(http.Refused, match="YAML map"): + C.fetch_assignments( + _client(handler), + [119], + commit_pacer=Pacer(), + raw_pacer=Pacer(), + ) + + +def test_member_pages_stop_at_count_and_never_follow_next(monkeypatch, tmp_path): + monkeypatch.setattr(http, "hoard_root", lambda: tmp_path) + monkeypatch.setattr(C, "LIMIT", 1) + page = _page() + people = (FX / "legislators_snippet.yaml").read_bytes() + seen: list[httpx.Request] = [] + + def api(request: httpx.Request) -> httpx.Response: + seen.append(request) + offset = int(request.url.params["offset"]) + member = page["members"][offset] + return httpx.Response( + 200, + json={ + "members": [member], + "pagination": { + "count": 2, + "next": "https://api.congress.gov/v3/member/congress/118" + "?offset=1&api_key=SECRET", + }, + }, + ) + + def plain(request: httpx.Request) -> httpx.Response: + seen.append(request) + if request.url.path.endswith("legislators-historical.yaml"): + return httpx.Response(200, content=b"[]\n") + return httpx.Response(200, content=people) + + rows = C.fetch_members( + _client(api), + _client(plain), + [118], + STATES, + member_pacer=Pacer(), + raw_pacer=Pacer(), + cache=tmp_path / "congress", + ) + offsets = [r.url.params["offset"] for r in seen if r.url.host == "api.congress.gov"] + assert offsets == ["0", "1"] + assert all("api_key" not in str(r.url) for r in seen) + congress_hits = [r for r in seen if r.url.host == "api.congress.gov"] + assert [r.url.params["currentMember"] for r in congress_hits] == ["false", "false"] + assert [r.url.params["format"] for r in congress_hits] == ["json", "json"] + by = {r.bioguide_id: r for r in rows} + assert by["D000624"].fec_ids == "H0XX00001" + assert by["Y000033"].district_id == "AK-00" + + +def test_repeated_first_bioguide_stops_paging(monkeypatch): + monkeypatch.setattr(C, "LIMIT", 1) + member = _page()["members"][0] + calls = {"n": 0} + + def api(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response( + 200, + json={"members": [member], "pagination": {"count": 50}}, + ) + + def plain(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b"[]\n") + + rows = C.fetch_members( + _client(api), + _client(plain), + [118], + STATES, + member_pacer=Pacer(), + raw_pacer=Pacer(), + ) + assert [r.bioguide_id for r in rows] == ["D000624"] + assert calls["n"] == 2 + + +def test_member_json_that_is_not_an_object_is_refused(): + def api(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=["nope"]) + + def plain(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b"[]\n") + + with pytest.raises(http.Refused, match="not JSON"): + C.fetch_members( + _client(api), + _client(plain), + [118], + STATES, + member_pacer=Pacer(), + raw_pacer=Pacer(), + ) + + +def test_members_replace_one_congress_and_keep_the_other(): + import duckdb + + c = duckdb.connect(":memory:") + first = T.MemberRow( + 118, + "D000624", + "Dingell, Debbie", + "Democratic", + "MI", + "MI-12", + "house", + "", + "congress.gov", + ) + second = T.MemberRow( + 119, + "D000624", + "Dingell, Debbie", + "Democratic", + "MI", + "MI-06", + "house", + "", + "congress.gov", + ) + T.write_members(c, [first, second]) + T.write_members( + c, + [ + T.MemberRow( + 118, + "D000624", + "Renamed", + "Democratic", + "MI", + "MI-12", + "house", + "H0XX00001", + "congress.gov", + ) + ], + ) + got = {(r.congress, r.name, r.fec_ids) for r in T.read_members(c)} + assert got == {(118, "Renamed", "H0XX00001"), (119, "Dingell, Debbie", "")} + assert T.write_members(c, []) == 0 + assert len(T.read_members(c)) == 2 + c.close() + + +def test_malformed_payloads_are_skipped_or_refused(): + page = { + "members": [ + "nope", + { + "bioguideId": "D000001", + "district": 1, + "name": "One", + "partyName": "Democratic", + "state": "Michigan", + "terms": {"item": None}, + }, + ] + } + rows = C.members_from_api(page, 118, STATES) + assert [r.bioguide_id for r in rows] == ["D000001"] + assert rows[0].chamber == "house" + people = ["nope", *_people()] + assert [r.bioguide_id for r in C.members_from_yaml(people, [1], STATES)] == [ + "V000001", + "S000001", + ] + assert set(C.fec_ids_by_bioguide(people)) == {"D000624"} + seats = C.membership_rows( + {"HSWM02": ["nope", {"bioguide": "E000001", "party": "majority", "rank": 1}]}, + congress=119, + snapshot=date(2025, 4, 1), + sha="s", + ) + assert [r.bioguide_id for r in seats] == ["E000001"] + + def bad_json(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b"not-json") + + with pytest.raises(http.Refused, match="not JSON"): + C.fetch_members( + _client(bad_json), + _client(lambda request: httpx.Response(200, content=b"[]\n")), + [118], + STATES, + member_pacer=Pacer(), + raw_pacer=Pacer(), + ) + + def bad_yaml(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=b":\n") + + def empty_api(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"members": [], "pagination": {"count": 0}}) + + with pytest.raises(http.Refused, match="YAML"): + C.fetch_members( + _client(empty_api), + _client(bad_yaml), + [118], + STATES, + member_pacer=Pacer(), + raw_pacer=Pacer(), + ) + + def members_object(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"members": {"bioguideId": "X"}}) + + assert ( + C.fetch_members( + _client(members_object), + _client(lambda request: httpx.Response(200, content=b"[]\n")), + [118], + STATES, + member_pacer=Pacer(), + raw_pacer=Pacer(), + ) + == [] + ) + + def no_sha(request: httpx.Request) -> httpx.Response: + if request.url.host == "api.github.com": + return httpx.Response(200, json=[{"sha": ""}]) + return httpx.Response(200, content=b"{}\n") + + with pytest.raises(http.Refused, match="no commit"): + C.fetch_assignments( + _client(no_sha), + [119], + commit_pacer=Pacer(), + raw_pacer=Pacer(), + ) + + +def test_assignments_replace_one_congress_and_keep_the_other(): + import duckdb + + c = duckdb.connect(":memory:") + a = T.AssignmentRow( + 118, date(2023, 4, 1), "HSWM02", "E000001", 1, "Chair", "majority", "aaa" + ) + b = T.AssignmentRow( + 119, date(2025, 4, 1), "HSIF14", "E000001", 2, "", "minority", "bbb" + ) + T.write_assignments(c, [a, b]) + T.write_assignments(c, [a]) + assert {(r.congress, r.committee_id) for r in T.read_assignments(c)} == { + (118, "HSWM02"), + (119, "HSIF14"), + } + assert T.write_assignments(c, []) == 0 + c.close() diff --git a/tests/market/test_imports.py b/tests/market/test_imports.py index 797a007..9542bf7 100644 --- a/tests/market/test_imports.py +++ b/tests/market/test_imports.py @@ -14,6 +14,7 @@ MODULES = [ "market.tiingo", "market.lda", "market.fec", + "market.congress", ] _PROBE = """ diff --git a/tests/market/test_tables.py b/tests/market/test_tables.py index 2f1c371..a6c219b 100644 --- a/tests/market/test_tables.py +++ b/tests/market/test_tables.py @@ -46,6 +46,8 @@ def test_ensure_tables_idempotent(): "fec_committee", "fec_totals", "fec_contribution", + "congress_member", + "committee_assignment", } @@ -184,6 +186,20 @@ FEC_CONTRIBUTION = T.FecContributionRow( "", "schedule_b/C00411553/2024/1.json", ) +MEMBER = T.MemberRow( + 118, + "D000624", + "Dingell, Debbie", + "Democratic", + "MI", + "MI-12", + "house", + "H0XX00001", + "congress.gov", +) +ASSIGNMENT = T.AssignmentRow( + 119, date(2025, 4, 1), "HSWM02", "E000001", 1, "Chair", "majority", "abc123" +) SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY") ROUNDTRIP = { @@ -212,6 +228,16 @@ ROUNDTRIP = { lambda c, r: T.write_fec_contributions(c, [r]), lambda c: T.read_fec_contributions(c), ), + "congress_member": ( + MEMBER, + lambda c, r: T.write_members(c, [r]), + lambda c: T.read_members(c), + ), + "committee_assignment": ( + ASSIGNMENT, + lambda c, r: T.write_assignments(c, [r]), + lambda c: T.read_assignments(c), + ), "entity": ( ENTITY, lambda c, r: T.write_entities(c, [r]), @@ -273,6 +299,8 @@ def test_write_empty_is_noop(con): assert T.write_fec_committees(con, []) == 0 assert T.write_fec_totals(con, []) == 0 assert T.write_fec_contributions(con, []) == 0 + assert T.write_members(con, []) == 0 + assert T.write_assignments(con, []) == 0 def test_lda_readers_filter_by_entity(con):