feat(market): members of Congress by canonical district and health-committee assignments per congress

Congress.gov rosters for the 110th through the 119th, with FEC ids from
the legislators YAML. Health-committee seats are one legislators snapshot
per congress, the last commit on or before April 1. A non-voting delegate
is POSTAL-98; an at-large state stays POSTAL-00.
This commit is contained in:
kert
2026-11-16 17:47:48 -05:00
parent 1c08090147
commit 2ee82e269d
10 changed files with 1756 additions and 0 deletions

View File

@@ -20,6 +20,7 @@ from typing import Any
import httpx
import typer
from market import congress as congress_api
from market import edgar, http, tiingo
from market import fec as fec_api
from market import lda as lda_api
@@ -53,6 +54,8 @@ from market.tables import (
read_prices,
retire_links,
retire_organizations,
state_codes,
write_assignments,
write_entities,
write_fec_committees,
write_fec_contributions,
@@ -61,6 +64,7 @@ from market.tables import (
write_gap,
write_hits,
write_links,
write_members,
write_passages,
write_prices,
write_securities,
@@ -75,6 +79,8 @@ PRICES_GAP = "tiingo:prices"
FTS_GAP = "sec-edgar:fts"
LDA_GAP = "lda:lda"
FEC_GAP = "fec:fec"
MEMBERS_GAP = "congress:members"
COMMITTEES_GAP = "legislators:committees"
# ── indirections the tests monkeypatch ───────────────────────────────
@@ -1448,6 +1454,135 @@ def fec(
typer.echo(f"stored: {n_c} committees, {n_t} totals, {n_b} schedule B rows")
def _congress_range(from_congress: int, to_congress: int) -> range:
if from_congress > to_congress:
typer.echo(f"--from {from_congress} is after --to {to_congress}")
raise typer.Exit(2)
return range(from_congress, to_congress + 1)
def _member_line(rows: list[Any]) -> str:
house = sum(r.chamber == "house" for r in rows)
senate = sum(r.chamber == "senate" for r in rows)
at_large = sum(r.district_id.endswith("-00") for r in rows)
return f"{len(rows)} members ({house} house, {senate} senate, {at_large} at-large)"
@app.command()
def members(
from_congress: int = typer.Option(congress_api.CONGRESSES.start, "--from"),
to_congress: int = typer.Option(congress_api.CONGRESSES.stop - 1, "--to"),
write: bool = typer.Option(False, "--write", help="Store market.congress_member."),
) -> None:
"""Members of Congress (market.congress_member) for the 110th through the
119th. Congress.gov supplies the roster; the legislators YAML supplies
FEC candidate ids and members the API does not return. A missing key
records the gap and writes nothing else."""
congresses = _congress_range(from_congress, to_congress)
key = _require_key("congress", MEMBERS_GAP, write=write)
if key is None:
return
con = _read()
try:
states = state_codes(con)
finally:
con.close()
api = http.client("congress", key)
plain = http.client("legislators")
try:
try:
rows = congress_api.fetch_members(
api,
plain,
congresses,
states,
member_pacer=congress_api.member_pacer(),
raw_pacer=congress_api.raw_pacer(),
cache=http.hoard("congress"),
)
except http.Refused as exc:
_gap(MEMBERS_GAP, str(exc), write=write)
return
finally:
api.close()
plain.close()
counts = Counter(r.congress for r in rows)
typer.echo(_member_line(rows))
for n in congresses:
typer.echo(f"congress {n}: {counts[n]}")
if not write:
typer.echo("dry run: pass --write to store")
return
from cms.ingest_log import log_ingest
with _batch() as con:
ensure_tables(con)
n = write_members(con, rows)
clear_gap(con, MEMBERS_GAP)
log_ingest(
con,
module="market.congress",
table_name="market.congress_member",
rows=n,
source_file=str(http.hoard(http.MANIFEST)),
)
_publish()
typer.echo(f"stored: {n} members")
@app.command()
def committees(
from_congress: int = typer.Option(
congress_api.COMMITTEE_CONGRESSES.start, "--from"
),
to_congress: int = typer.Option(congress_api.COMMITTEE_CONGRESSES.stop - 1, "--to"),
write: bool = typer.Option(
False, "--write", help="Store market.committee_assignment."
),
) -> None:
"""Health-committee assignments (market.committee_assignment), one
legislators snapshot per congress from the 113th on. Keyless. The GitHub
commit lookup is paced at 61 seconds."""
congresses = _congress_range(from_congress, to_congress)
plain = http.client("legislators")
try:
try:
rows = congress_api.fetch_assignments(
plain,
congresses,
commit_pacer=congress_api.commit_pacer(),
raw_pacer=congress_api.raw_pacer(),
cache=http.hoard("congress"),
)
except http.Refused as exc:
_gap(COMMITTEES_GAP, str(exc), write=write)
return
finally:
plain.close()
counts = Counter(r.congress for r in rows)
typer.echo(f"{len(rows)} assignments")
for n in congresses:
typer.echo(f"congress {n}: {counts[n]}")
if not write:
typer.echo("dry run: pass --write to store")
return
from cms.ingest_log import log_ingest
with _batch() as con:
ensure_tables(con)
n = write_assignments(con, rows)
clear_gap(con, COMMITTEES_GAP)
log_ingest(
con,
module="market.congress",
table_name="market.committee_assignment",
rows=n,
source_file=str(http.hoard(http.MANIFEST)),
)
_publish()
typer.echo(f"stored: {n} assignments")
@app.command()
def gaps() -> None:
"""Sources that could not be read (market.source_gap)."""

550
src/market/congress.py Normal file
View File

@@ -0,0 +1,550 @@
"""Members of Congress and health-committee assignments.
Congress.gov (``X-Api-Key``, never the query string) lists who served.
``currentMember=false`` keeps members who have since left. The legislators
YAML (public domain, keyless) supplies FEC candidate ids and fills a
congress the API does not return. Committee seats are one
congress-legislators snapshot per congress: the last commit of
``committee-membership-current.yaml`` on or before April 1 of that
congress's first year (M7). ``pagination.next`` is never followed; it can
carry the key.
"""
from __future__ import annotations
import datetime as dt
import json
import os
from collections.abc import Mapping, Sequence
from dataclasses import replace
from pathlib import Path
from typing import Any
import httpx
import yaml
from market import http
from market.tables import AssignmentRow, MemberRow, district_id
CONGRESS_API = "https://api.congress.gov/v3/"
LEGISLATORS = (
"https://raw.githubusercontent.com/unitedstates/congress-legislators/{ref}/{name}"
)
GH_COMMITS = "https://api.github.com/repos/unitedstates/congress-legislators/commits"
REF_MAIN = "main"
CURRENT = "legislators-current.yaml"
HISTORICAL = "legislators-historical.yaml"
MEMBERSHIP = "committee-membership-current.yaml"
CONGRESSES = range(110, 120)
COMMITTEE_CONGRESSES = range(113, 120)
HEALTH_COMMITTEES = (
"HSWM",
"HSWM02",
"HSIF",
"HSIF14",
"SSFI",
"SSFI10",
"SSHR",
"SSHR12",
)
# Congress.gov sends district 0 for both an at-large state and a delegate.
# The canonical id for a non-voting delegate is POSTAL-98 (M7).
DELEGATES = frozenset({"AS", "DC", "GU", "MP", "PR", "VI"})
LIMIT = 250
MEMBER_PACE = 0.8
COMMIT_PACE = 61.0
RAW_PACE = 0.2
def member_pacer() -> http.Pacer:
return http.Pacer(MEMBER_PACE)
def commit_pacer() -> http.Pacer:
return http.Pacer(COMMIT_PACE)
def raw_pacer() -> http.Pacer:
return http.Pacer(RAW_PACE)
def congress_of(year: int) -> int:
"""The Congress in session during *year* (the 119th covers 2025–2026)."""
return ((year - 1789) // 2) + 1
def first_year(congress: int) -> int:
"""``1789 + 2·(congress − 1)``."""
return 1789 + 2 * (congress - 1)
def snapshot_cutoff(congress: int) -> dt.date:
"""April 1 of :func:`first_year` (the legislators snapshot for M7)."""
return dt.date(first_year(congress), 4, 1)
def congress_start(congress: int) -> dt.date:
"""The day *congress* convened. The 74th Congress (1935) is the first
to open on January 3; earlier congresses open on March 4."""
year = first_year(congress)
if congress >= 74:
return dt.date(year, 1, 3)
return dt.date(year, 3, 4)
def commit_params(congress: int) -> dict[str, Any]:
"""GitHub commits query for the snapshot on or before the cutoff."""
cutoff = snapshot_cutoff(congress)
return {
"path": MEMBERSHIP,
"until": f"{cutoff.isoformat()}T00:00:00Z",
"per_page": 1,
}
def _items(terms: Any) -> list[dict[str, Any]]:
if not isinstance(terms, dict):
return []
item = terms.get("item")
if isinstance(item, dict):
return [item]
if isinstance(item, list):
return [x for x in item if isinstance(x, dict)]
return []
def _chamber(items: Sequence[Mapping[str, Any]], congress: int) -> str:
"""The chamber of the term that overlaps *congress*. A later overlapping
term wins. No term means the House."""
y0 = first_year(congress)
y1 = y0 + 1
chosen: Mapping[str, Any] | None = None
for item in items:
start = int(item.get("startYear") or 0)
end = item.get("endYear")
if start <= y1 and (end in (None, "") or int(end) >= y0):
chosen = item
text = str((chosen or {}).get("chamber") or "")
return "senate" if "senate" in text.lower() else "house"
def house_district(postal: str, number: Any) -> str:
"""``AK-00`` for an at-large state; ``DC-98`` for a non-voting delegate."""
if postal in DELEGATES:
return f"{postal}-98"
return district_id(postal, number)
def _postal(raw: str, states: Mapping[str, tuple[str, str, str]]) -> str | None:
hit = states.get(raw)
if hit is None:
return None
return hit[1]
def members_from_api(
payload: Mapping[str, Any],
congress: int,
states: Mapping[str, tuple[str, str, str]],
) -> list[MemberRow]:
"""One Congress.gov member page. A state the reference table does not
know is skipped. The Senate has an empty ``district_id``."""
rows: list[MemberRow] = []
for member in payload.get("members") or []:
if not isinstance(member, dict):
continue
bioguide = str(member.get("bioguideId") or "")
postal = _postal(str(member.get("state") or ""), states)
if not bioguide or postal is None:
continue
chamber = _chamber(_items(member.get("terms")), congress)
district = (
""
if chamber == "senate"
else house_district(postal, member.get("district"))
)
rows.append(
MemberRow(
congress=congress,
bioguide_id=bioguide,
name=str(member.get("name") or ""),
party=str(member.get("partyName") or ""),
state=postal,
district_id=district,
chamber=chamber,
fec_ids="",
source="congress.gov",
)
)
return rows
def _fec_tuple(person: Mapping[str, Any]) -> tuple[str, ...]:
raw = (person.get("id") or {}).get("fec") or []
out: list[str] = []
for item in raw:
text = str(item).strip()
if text and text not in out:
out.append(text)
return tuple(out)
def _covers(term: Mapping[str, Any], congress: int) -> bool:
"""A legislators term is ``[start, end)``. *end* equal to the next
congress's first day does not include that congress."""
start = dt.date.fromisoformat(str(term.get("start"))[:10])
end_raw = term.get("end")
end = dt.date.fromisoformat(str(end_raw)[:10]) if end_raw else None
opened = congress_start(congress)
closed = congress_start(congress + 1)
return start < closed and (end is None or end > opened)
def _yaml_term(terms: Sequence[Mapping[str, Any]], congress: int) -> Mapping | None:
chosen: Mapping[str, Any] | None = None
for term in terms:
if not isinstance(term, dict) or not _covers(term, congress):
continue
if chosen is None or str(term.get("start")) > str(chosen.get("start")):
chosen = term
return chosen
def members_from_yaml(
docs: Sequence[Mapping[str, Any]],
congresses: Sequence[int],
states: Mapping[str, tuple[str, str, str]],
) -> list[MemberRow]:
"""One row per requested congress a term covers. ``type: sen`` has an
empty ``district_id``; a representative gets ``district_id``."""
rows: list[MemberRow] = []
for person in docs:
if not isinstance(person, dict):
continue
bioguide = str((person.get("id") or {}).get("bioguide") or "")
if not bioguide:
continue
fec = "|".join(_fec_tuple(person))
name = str((person.get("name") or {}).get("official_full") or "")
terms = person.get("terms") or []
for congress in congresses:
term = _yaml_term(terms, congress)
if term is None:
continue
postal = _postal(str(term.get("state") or ""), states)
if postal is None:
continue
chamber = "senate" if str(term.get("type") or "") == "sen" else "house"
district = (
""
if chamber == "senate"
else house_district(postal, term.get("district"))
)
rows.append(
MemberRow(
congress=congress,
bioguide_id=bioguide,
name=name,
party=str(term.get("party") or ""),
state=postal,
district_id=district,
chamber=chamber,
fec_ids=fec,
source="legislators",
)
)
return rows
def fec_ids_by_bioguide(
docs: Sequence[Mapping[str, Any]],
) -> dict[str, tuple[str, ...]]:
"""FEC candidate ids from legislators YAML, keyed by bioguide."""
out: dict[str, tuple[str, ...]] = {}
for person in docs:
if not isinstance(person, dict):
continue
bioguide = str((person.get("id") or {}).get("bioguide") or "")
ids = _fec_tuple(person)
if bioguide and ids:
out[bioguide] = ids
return out
def merge_members(
api_rows: Sequence[MemberRow],
yaml_rows: Sequence[MemberRow],
fec: Mapping[str, tuple[str, ...]],
) -> list[MemberRow]:
"""API rows win. FEC ids overlay by bioguide. A YAML row is added only
when that congress does not already have the bioguide."""
out: list[MemberRow] = []
seen: set[tuple[int, str]] = set()
for row in api_rows:
ids = fec.get(row.bioguide_id, ())
if ids:
row = replace(row, fec_ids="|".join(ids))
out.append(row)
seen.add((row.congress, row.bioguide_id))
for row in yaml_rows:
key = (row.congress, row.bioguide_id)
if key not in seen:
out.append(row)
seen.add(key)
return out
def membership_rows(
doc: Mapping[str, Any],
*,
congress: int,
snapshot: dt.date,
sha: str,
) -> list[AssignmentRow]:
"""Health subcommittees only. A missing rank is 0 and a missing title
is empty."""
rows: list[AssignmentRow] = []
for committee_id, members in doc.items():
if committee_id not in HEALTH_COMMITTEES or not isinstance(members, list):
continue
for member in members:
if not isinstance(member, dict):
continue
rank = member.get("rank")
rows.append(
AssignmentRow(
congress=congress,
snapshot=snapshot,
committee_id=str(committee_id),
bioguide_id=str(member.get("bioguide") or ""),
rank=0 if rank in (None, "") else int(rank),
title=str(member.get("title") or ""),
party=str(member.get("party") or ""),
source_sha=sha,
)
)
return rows
def _cached(path: Path | None) -> bytes | None:
if path is None:
return None
have = http.newest(path)
if have.is_file():
return have.read_bytes()
return None
def _store(path: Path | None, body: bytes, url: str) -> None:
if path is None:
return
where = http.save(path, body)
os.utime(where)
http.manifest(where, url)
def _json_value(body: bytes, source: str, status: int) -> Any:
try:
return json.loads(body)
except ValueError:
raise http.Refused(source, status, "response is not JSON") from None
def _object(body: bytes, source: str, status: int) -> dict[str, Any]:
payload = _json_value(body, source, status)
if not isinstance(payload, dict):
raise http.Refused(source, status, "response is not JSON")
return payload
def _commit_list(body: bytes, status: int) -> list[Any]:
payload = _json_value(body, "legislators", status)
if not isinstance(payload, list) or not payload:
raise http.Refused("legislators", status, "no commit on or before the cutoff")
return payload
def _yaml_doc(body: bytes, expect: type, status: int) -> Any:
try:
doc = yaml.safe_load(body)
except yaml.YAMLError:
doc = None
kind = "list" if expect is list else "map"
if not isinstance(doc, expect):
raise http.Refused("legislators", status, f"response is not a YAML {kind}")
return doc
def _download(
cl: httpx.Client,
url: str,
params: Mapping[str, Any] | None,
*,
pacer: Any,
path: Path | None,
source: str,
) -> bytes:
cached = _cached(path)
if cached is not None:
return cached
response = http.get(cl, url, params=params, pacer=pacer, source=source)
_store(path, response.content, str(response.request.url))
return response.content
def _member_path(cache: Path | None, congress: int, offset: int) -> Path | None:
if cache is None:
return None
return cache / "members" / str(congress) / f"{offset}.json"
def _leg_path(cache: Path | None, name: str) -> Path | None:
if cache is None:
return None
return cache / "legislators" / name
def _commit_path(cache: Path | None, congress: int) -> Path | None:
if cache is None:
return None
day = snapshot_cutoff(congress).isoformat()
return cache / "commits" / MEMBERSHIP / f"{day}.json"
def _membership_path(cache: Path | None, sha: str) -> Path | None:
if cache is None:
return None
return cache / "membership" / f"{sha}.yaml"
def _pages(
api: httpx.Client,
congress: int,
states: Mapping[str, tuple[str, str, str]],
*,
pacer: Any,
cache: Path | None,
) -> list[MemberRow]:
rows: list[MemberRow] = []
offset = 0
seen_first: str | None = None
got = 0
while True:
url = f"{CONGRESS_API}member/congress/{congress}"
body = _download(
api,
url,
{
"limit": LIMIT,
"offset": offset,
"format": "json",
"currentMember": "false",
},
pacer=pacer,
path=_member_path(cache, congress, offset),
source="congress",
)
payload = _object(body, "congress", 200)
members = payload.get("members") or []
if not isinstance(members, list):
members = []
first = ""
if members and isinstance(members[0], dict):
first = str(members[0].get("bioguideId") or "")
if first and first == seen_first:
break
if first:
seen_first = first
rows.extend(members_from_api(payload, congress, states))
got += len(members) if isinstance(members, list) else 0
count = (payload.get("pagination") or {}).get("count")
if (
not members
or len(members) < LIMIT
or (count is not None and got >= int(count))
):
break
offset += LIMIT
return rows
def _legislators(
plain: httpx.Client, *, pacer: Any, cache: Path | None
) -> list[Mapping[str, Any]]:
docs: list[Mapping[str, Any]] = []
for name in (CURRENT, HISTORICAL):
url = LEGISLATORS.format(ref=REF_MAIN, name=name)
body = _download(
plain,
url,
None,
pacer=pacer,
path=_leg_path(cache, name),
source="legislators",
)
doc = _yaml_doc(body, list, 200)
docs.extend(person for person in doc if isinstance(person, dict))
return docs
def fetch_members(
api: httpx.Client,
plain: httpx.Client,
congresses: Sequence[int],
states: Mapping[str, tuple[str, str, str]],
*,
member_pacer: Any,
raw_pacer: Any,
cache: Path | None = None,
) -> list[MemberRow]:
"""Congress.gov pages for *congresses*, then the legislators YAML for
FEC ids and members the API did not return."""
api_rows: list[MemberRow] = []
for congress in congresses:
api_rows.extend(_pages(api, congress, states, pacer=member_pacer, cache=cache))
docs = _legislators(plain, pacer=raw_pacer, cache=cache)
yaml_rows = members_from_yaml(docs, list(congresses), states)
return merge_members(api_rows, yaml_rows, fec_ids_by_bioguide(docs))
def fetch_assignments(
plain: httpx.Client,
congresses: Sequence[int],
*,
commit_pacer: Any,
raw_pacer: Any,
cache: Path | None = None,
) -> list[AssignmentRow]:
"""One health-committee snapshot per congress. An empty commit list is
a refusal, so a missing history is not stored as an empty snapshot."""
rows: list[AssignmentRow] = []
for congress in congresses:
body = _download(
plain,
GH_COMMITS,
commit_params(congress),
pacer=commit_pacer,
path=_commit_path(cache, congress),
source="legislators",
)
commits = _commit_list(body, 200)
sha = str(commits[0].get("sha") or "")
if not sha:
raise http.Refused("legislators", 200, "no commit on or before the cutoff")
url = LEGISLATORS.format(ref=sha, name=MEMBERSHIP)
raw = _download(
plain,
url,
None,
pacer=raw_pacer,
path=_membership_path(cache, sha),
source="legislators",
)
doc = _yaml_doc(raw, dict, 200)
rows.extend(
membership_rows(
doc, congress=congress, snapshot=snapshot_cutoff(congress), sha=sha
)
)
return rows

View File

@@ -85,6 +85,15 @@ CREATE TABLE IF NOT EXISTS market.fec_contribution (
candidate_id VARCHAR, candidate_name VARCHAR, candidate_office VARCHAR,
candidate_state VARCHAR, candidate_district VARCHAR, memo_code VARCHAR,
source_file VARCHAR
);
CREATE TABLE IF NOT EXISTS market.congress_member (
congress INTEGER, bioguide_id VARCHAR, name VARCHAR, party VARCHAR,
state VARCHAR, district_id VARCHAR, chamber VARCHAR, fec_ids VARCHAR,
source VARCHAR
);
CREATE TABLE IF NOT EXISTS market.committee_assignment (
congress INTEGER, snapshot DATE, committee_id VARCHAR, bioguide_id VARCHAR,
rank INTEGER, title VARCHAR, party VARCHAR, source_sha VARCHAR
)
"""
@@ -339,6 +348,38 @@ class FecContributionRow:
source_file: str
@dataclass(frozen=True)
class MemberRow:
"""One member of one Congress. ``fec_ids`` is a ``|``-joined list.
``district_id`` is ``MI-12`` or at-large ``AK-00`` for the House and
``""`` for the Senate. ``state`` is the postal code."""
congress: int
bioguide_id: str
name: str
party: str
state: str
district_id: str
chamber: str
fec_ids: str
source: str
@dataclass(frozen=True)
class AssignmentRow:
"""One health-committee seat in one congress's legislators snapshot.
``snapshot`` is April 1 of that congress's first year (M7)."""
congress: int
snapshot: date
committee_id: str
bioguide_id: str
rank: int
title: str
party: str
source_sha: str
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
a statement per row, far too slow for filing lists)."""
@@ -774,6 +815,53 @@ def read_fec_contributions(
)
# ── congress ─────────────────────────────────────────────────────────
def write_members(con: Any, rows: Sequence[MemberRow]) -> int:
"""Replace the members of each congress present in *rows*."""
ensure_tables(con)
if not rows:
return 0
for chunk in _chunks(sorted({r.congress for r in rows})):
con.execute(
"DELETE FROM market.congress_member WHERE congress IN "
f"({','.join('?' * len(chunk))})",
list(chunk),
)
return _insert(con, "market.congress_member", rows)
def read_members(con: Any) -> list[MemberRow]:
rows = _select(con, MemberRow, "market.congress_member")
return sorted(
(MemberRow(*r) for r in rows),
key=lambda r: (r.congress, r.bioguide_id, r.district_id),
)
def write_assignments(con: Any, rows: Sequence[AssignmentRow]) -> int:
"""Replace the assignments of each congress present in *rows*."""
ensure_tables(con)
if not rows:
return 0
for chunk in _chunks(sorted({r.congress for r in rows})):
con.execute(
"DELETE FROM market.committee_assignment WHERE congress IN "
f"({','.join('?' * len(chunk))})",
list(chunk),
)
return _insert(con, "market.committee_assignment", rows)
def read_assignments(con: Any) -> list[AssignmentRow]:
rows = _select(con, AssignmentRow, "market.committee_assignment")
return sorted(
(AssignmentRow(*r) for r in rows),
key=lambda r: (r.congress, r.committee_id, r.rank, r.bioguide_id),
)
# ── districts ────────────────────────────────────────────────────────

View File

@@ -2181,3 +2181,205 @@ def test_fec_cli_limit_keeps_the_gap_and_empty_registry(fec, monkeypatch):
assert empty.exit_code == 1 and "no tracked entities" in empty.output
blank = runner.invoke(app, ["market", "fec", "--cycles", ","])
assert blank.exit_code == 2 and "--cycles is empty" in _plain(blank.output)
# ── congress ─────────────────────────────────────────────────────────
CONGRESS_SECRET = "congress-cli-key-do-not-leak"
def _congress_states(c) -> None:
c.execute("CREATE SCHEMA IF NOT EXISTS reference_data")
c.execute(
"CREATE TABLE IF NOT EXISTS reference_data.ansi_fips_state ("
"ansi_fips_state_code VARCHAR, ansi_fips_state_abbreviation VARCHAR, "
"ansi_fips_state_name VARCHAR)"
)
c.execute(
"INSERT INTO reference_data.ansi_fips_state VALUES "
"('26', 'MI', 'Michigan'), ('02', 'AK', 'Alaska'), ('51', 'VA', 'Virginia')"
)
@pytest.fixture
def congress(env, monkeypatch):
from market import congress as congress_api
c, store, published, hoard = env
_congress_states(c)
T.ensure_tables(c)
seen: list[httpx.Request] = []
members = json.loads((FX / "congress_members_118.json").read_text())
people = (FX / "legislators_snippet.yaml").read_bytes()
membership = (FX / "membership_snippet.yaml").read_bytes()
state = {"status": 200}
for name in ("member_pacer", "raw_pacer", "commit_pacer"):
monkeypatch.setattr(congress_api, name, lambda: _Pacer())
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request)
if state["status"] != 200:
return httpx.Response(state["status"], text="no")
host = request.url.host
if host == "api.congress.gov":
return httpx.Response(200, json=members)
if host == "api.github.com":
return httpx.Response(200, json=[{"sha": "deadbeef"}])
if request.url.path.endswith("legislators-historical.yaml"):
return httpx.Response(200, content=b"[]\n")
if request.url.path.endswith("committee-membership-current.yaml"):
return httpx.Response(200, content=membership)
return httpx.Response(200, content=people)
def client(source, key=None):
headers = {"X-Api-Key": key} if key else {}
return httpx.Client(transport=httpx.MockTransport(handler), headers=headers)
monkeypatch.setattr(http, "client", client)
monkeypatch.setattr(http, "api_key", lambda source, env=None: CONGRESS_SECRET)
return c, store, published, hoard, seen, state
def test_members_cli_no_write_prints_counts(congress):
c, _, published, _, seen, _ = congress
res = runner.invoke(app, ["market", "members", "--from", "118", "--to", "118"])
assert res.exit_code == 0, res.output
assert "2 members (2 house, 0 senate, 1 at-large)" in res.output
assert "congress 118: 2" in res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.congress_member") == 0
assert published == []
assert any(r.url.host == "api.congress.gov" for r in seen)
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
assert all("api_key" not in str(r.url) for r in seen)
congress_hits = [r for r in seen if r.url.host == "api.congress.gov"]
assert congress_hits[0].headers["x-api-key"] == CONGRESS_SECRET
github_hits = [r for r in seen if r.url.host != "api.congress.gov"]
assert github_hits
assert all(r.headers.get("x-api-key") in (None, "") for r in github_hits)
def test_members_cli_write(congress):
c, _, published, hoard, seen, _ = congress
T.write_gap(c, "congress:members", "old")
res = runner.invoke(
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "stored: 2 members" in res.output
rows = {r.bioguide_id: r for r in T.read_members(c)}
assert rows["D000624"].district_id == "MI-12"
assert rows["D000624"].fec_ids == "H0XX00001"
assert rows["D000624"].source == "congress.gov"
assert rows["Y000033"].district_id == "AK-00"
assert rows["Y000033"].state == "AK"
assert T.read_gaps(c) == []
assert published == [True]
assert {
(t, n)
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
} == {("market.congress_member", 2)}
manifest = (hoard / "MANIFEST.jsonl").read_text()
assert CONGRESS_SECRET not in manifest
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
kept = T.MemberRow(
117,
"Z000001",
"Other",
"Republican",
"MI",
"MI-01",
"house",
"",
"congress.gov",
)
T.write_members(c, [kept])
assert {r.congress for r in T.read_members(c)} == {117, 118}
def test_members_cli_missing_key_gap_only(congress, monkeypatch):
c, _, published, _, seen, _ = congress
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
res = runner.invoke(
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: congress:members:" in res.output
assert "CONGRESS_API_KEY" in res.output
assert "API_DATA_GOV_KEY" in res.output
assert seen == []
assert [g.source for g in T.read_gaps(c)] == ["congress:members"]
assert _count(c, "market.congress_member") == 0
assert published == [True]
def test_members_cli_refusal_prints_gap(congress):
c, _, published, _, seen, state = congress
state["status"] = 403
res = runner.invoke(
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: congress:members: congress: HTTP 403" in res.output
assert CONGRESS_SECRET not in res.output
assert _count(c, "market.congress_member") == 0
assert len(seen) == 1
assert seen[0].url.host == "api.congress.gov"
assert seen[0].headers["x-api-key"] == CONGRESS_SECRET
assert CONGRESS_SECRET not in str(seen[0].url)
def test_members_cli_from_after_to(congress):
res = runner.invoke(app, ["market", "members", "--from", "119", "--to", "110"])
assert res.exit_code == 2
assert "--from 119 is after --to 110" in res.output
def test_committees_cli_write(congress):
c, _, published, hoard, seen, _ = congress
T.write_gap(c, "legislators:committees", "old")
dry = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "119"])
assert dry.exit_code == 0, dry.output
assert "1 assignments" in dry.output
assert "dry run: pass --write" in dry.output
assert _count(c, "market.committee_assignment") == 0
assert published == []
res = runner.invoke(
app, ["market", "committees", "--from", "119", "--to", "119", "--write"]
)
assert res.exit_code == 0, res.output
assert "stored: 1 assignments" in res.output
(row,) = T.read_assignments(c)
assert row.committee_id == "HSWM02"
assert row.bioguide_id == "E000001"
assert row.snapshot == date(2025, 4, 1)
assert row.source_sha == "deadbeef"
assert row.congress == 119
assert T.read_gaps(c) == []
assert published == [True]
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
assert all(r.headers.get("x-api-key") in (None, "") for r in seen)
manifest = (hoard / "MANIFEST.jsonl").read_text()
assert CONGRESS_SECRET not in manifest
logged = c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
assert ("market.committee_assignment", 1) in logged
def test_committees_cli_refusal_prints_gap(congress):
c, _, published, _, seen, state = congress
state["status"] = 403
res = runner.invoke(
app, ["market", "committees", "--from", "119", "--to", "119", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: legislators:committees: legislators: HTTP 403" in res.output
assert CONGRESS_SECRET not in res.output
assert _count(c, "market.committee_assignment") == 0
assert seen
assert published == [True]
def test_committees_cli_from_after_to(congress):
res = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "113"])
assert res.exit_code == 2
assert "--from 119 is after --to 113" in res.output

View File

@@ -0,0 +1,29 @@
{
"members": [
{
"bioguideId": "D000624",
"district": 12,
"name": "Dingell, Debbie",
"partyName": "Democratic",
"state": "Michigan",
"terms": {
"item": [
{"chamber": "House of Representatives", "startYear": 2023}
]
}
},
{
"bioguideId": "Y000033",
"district": null,
"name": "Young, Don",
"partyName": "Republican",
"state": "Alaska",
"terms": {
"item": [
{"chamber": "House of Representatives", "startYear": 2021}
]
}
}
],
"pagination": {"count": 2}
}

View File

@@ -0,0 +1,35 @@
- id:
bioguide: D000624
fec:
- H0XX00001
name:
official_full: Debbie Dingell
terms:
- type: rep
start: '2023-01-03'
end: '2025-01-03'
state: MI
district: 12
party: Democrat
- id:
bioguide: V000001
fec: []
name:
official_full: Virginia Rep
terms:
- type: rep
start: '1789-03-04'
end: '1791-03-03'
state: VA
district: 9
party: Federalist
- id:
bioguide: S000001
name:
official_full: Virginia Senator
terms:
- type: sen
start: '1789-03-04'
end: '1791-03-03'
state: VA
party: Federalist

View File

@@ -0,0 +1,11 @@
HSWM02:
- name: Example, Member
party: majority
rank: 1
title: Chair
bioguide: E000001
HSAG:
- name: Other, Member
party: minority
rank: 2
bioguide: X000001

View File

@@ -0,0 +1,677 @@
"""market.congress: members by district and health-committee snapshots."""
from __future__ import annotations
import json
from datetime import date
from pathlib import Path
import httpx
import pytest
import yaml
from market import congress as C
from market import http
from market import tables as T
FX = Path(__file__).parent / "fixtures"
STATES = {
"Michigan": ("26", "MI", "Michigan"),
"MI": ("26", "MI", "Michigan"),
"Alaska": ("02", "AK", "Alaska"),
"AK": ("02", "AK", "Alaska"),
"Virginia": ("51", "VA", "Virginia"),
"VA": ("51", "VA", "Virginia"),
}
def _people() -> list:
return yaml.safe_load((FX / "legislators_snippet.yaml").read_text())
def _page() -> dict:
return json.loads((FX / "congress_members_118.json").read_text())
class Pacer:
def wait(self):
pass
def _client(handler) -> httpx.Client:
return httpx.Client(transport=httpx.MockTransport(handler))
def test_congress_of_year():
assert C.congress_of(2013) == 113
assert C.congress_of(2026) == 119
assert C.congress_of(2027) == 120
assert C.first_year(119) == 2025
assert C.first_year(1) == 1789
assert C.member_pacer().min_interval == 0.8
assert C.commit_pacer().min_interval == 61
assert C.raw_pacer().min_interval == 0.2
assert C.LIMIT == 250
assert C.CONGRESSES == range(110, 120)
assert C.COMMITTEE_CONGRESSES == range(113, 120)
def test_snapshot_cutoff():
assert C.snapshot_cutoff(119) == date(2025, 4, 1)
assert C.snapshot_cutoff(113) == date(2013, 4, 1)
def test_members_from_api_page():
rows = C.members_from_api(_page(), 118, STATES)
dingell = next(r for r in rows if r.bioguide_id == "D000624")
assert dingell.state == "MI"
assert dingell.district_id == "MI-12"
assert dingell.chamber == "house"
assert dingell.party == "Democratic"
assert dingell.name == "Dingell, Debbie"
assert dingell.congress == 118
assert dingell.source == "congress.gov"
assert dingell.fec_ids == ""
def test_delegate_district_is_98_and_at_large_state_stays_00():
states = {
**STATES,
"District of Columbia": ("11", "DC", "District of Columbia"),
"DC": ("11", "DC", "District of Columbia"),
}
payload = {
"members": [
{
"bioguideId": "N000147",
"district": 0,
"name": "Norton, Eleanor Holmes",
"partyName": "Democratic",
"state": "District of Columbia",
"terms": {
"item": [{"chamber": "House of Representatives", "startYear": 1991}]
},
},
{
"bioguideId": "B001323",
"district": 0,
"name": "Begich, Nicholas J.",
"partyName": "Republican",
"state": "Alaska",
"terms": {
"item": [{"chamber": "House of Representatives", "startYear": 2025}]
},
},
]
}
rows = {r.bioguide_id: r for r in C.members_from_api(payload, 119, states)}
assert rows["N000147"].district_id == "DC-98"
assert rows["N000147"].chamber == "house"
assert rows["B001323"].district_id == "AK-00"
docs = [
{
"id": {"bioguide": "N000147"},
"name": {"official_full": "Eleanor Holmes Norton"},
"terms": [
{
"type": "rep",
"start": "2025-01-03",
"end": "2027-01-03",
"state": "DC",
"district": 0,
"party": "Democrat",
}
],
}
]
(yaml_row,) = C.members_from_yaml(docs, [119], states)
assert yaml_row.district_id == "DC-98"
def test_at_large_member_district():
rows = C.members_from_api(_page(), 118, STATES)
young = next(r for r in rows if r.bioguide_id == "Y000033")
assert young.state == "AK"
assert young.district_id == "AK-00"
assert young.chamber == "house"
def test_senate_member_has_empty_district():
payload = {
"members": [
{
"bioguideId": "S000033",
"district": None,
"name": "Stabenow, Debbie",
"partyName": "Democratic",
"state": "Michigan",
"terms": {"item": [{"chamber": "Senate", "startYear": 2001}]},
}
]
}
(row,) = C.members_from_api(payload, 118, STATES)
assert row.chamber == "senate"
assert row.district_id == ""
assert row.state == "MI"
def test_api_skips_unknown_state_and_accepts_a_single_term_dict():
payload = {
"members": [
{
"bioguideId": "Z000001",
"district": 1,
"name": "Nobody",
"partyName": "Independent",
"state": "Narnia",
"terms": {
"item": [{"chamber": "House of Representatives", "startYear": 2023}]
},
},
{
"bioguideId": "D000001",
"district": 1,
"name": "Delegate, One",
"partyName": "Democratic",
"state": "Michigan",
"terms": {
"item": {"chamber": "House of Representatives", "startYear": 2023}
},
},
{
"bioguideId": "H000001",
"district": 2,
"name": "No Terms",
"partyName": "Republican",
"state": "Michigan",
},
]
}
rows = C.members_from_api(payload, 118, STATES)
assert [r.bioguide_id for r in rows] == ["D000001", "H000001"]
assert rows[0].district_id == "MI-01"
assert rows[1].chamber == "house"
assert rows[1].district_id == "MI-02"
def test_api_chamber_follows_the_term_overlapping_the_congress():
payload = {
"members": [
{
"bioguideId": "B000001",
"district": 4,
"name": "Both, Chambers",
"partyName": "Republican",
"state": "Michigan",
"terms": {
"item": [
{
"chamber": "House of Representatives",
"startYear": 2013,
"endYear": 2019,
},
{"chamber": "Senate", "startYear": 2019},
]
},
}
]
}
house = C.members_from_api(payload, 115, STATES)[0]
senate = C.members_from_api(payload, 116, STATES)[0]
assert house.chamber == "house" and house.district_id == "MI-04"
assert senate.chamber == "senate" and senate.district_id == ""
def test_members_from_legislators_yaml():
docs = [
*_people(),
{"id": {}, "name": {"official_full": "No Id"}, "terms": []},
{
"id": {"bioguide": "N000009"},
"name": {"official_full": "Narnia"},
"terms": [
{
"type": "rep",
"start": "1789-03-04",
"end": "1791-03-03",
"state": "Narnia",
"district": 1,
}
],
},
]
rows = C.members_from_yaml(docs, [1], STATES)
by_id = {r.bioguide_id: r for r in rows}
assert set(by_id) == {"V000001", "S000001"}
assert by_id["V000001"].district_id == "VA-09"
assert by_id["V000001"].state == "VA"
assert by_id["V000001"].chamber == "house"
assert by_id["V000001"].source == "legislators"
assert by_id["V000001"].congress == 1
assert by_id["S000001"].district_id == ""
assert by_id["S000001"].chamber == "senate"
later = C.members_from_yaml(docs, [1, 2], STATES)
assert {r.congress for r in later} == {1}
modern = C.members_from_yaml(docs, [118], STATES)
assert [r.bioguide_id for r in modern] == ["D000624"]
assert modern[0].district_id == "MI-12"
assert modern[0].fec_ids == "H0XX00001"
def test_yaml_term_ending_on_the_next_congress_start_is_excluded():
docs = [
{
"id": {"bioguide": "N000001", "fec": ["H0AA00001", "H0AA00001"]},
"name": {"official_full": "Next Boundary"},
"terms": [
{
"type": "rep",
"start": "2025-01-03",
"end": "2027-01-03",
"state": "VA",
"district": 1,
"party": "Democrat",
}
],
}
]
rows = C.members_from_yaml(docs, [119, 120], STATES)
assert [(r.congress, r.district_id, r.fec_ids) for r in rows] == [
(119, "VA-01", "H0AA00001")
]
def test_fec_ids_by_bioguide():
ids = C.fec_ids_by_bioguide(_people())
assert ids["D000624"] == ("H0XX00001",)
assert set(ids) == {"D000624"}
def test_merge_members_overlays_fec_ids_and_adds_the_api_gap():
api = C.members_from_api(_page(), 118, STATES)
yaml_rows = C.members_from_yaml(_people(), [1, 118], STATES)
fec = C.fec_ids_by_bioguide(_people())
merged = C.merge_members(api, yaml_rows, fec)
by = {(r.congress, r.bioguide_id): r for r in merged}
assert by[(118, "D000624")].fec_ids == "H0XX00001"
assert by[(118, "D000624")].source == "congress.gov"
assert (118, "D000624") in by and (118, "Y000033") in by
assert (1, "V000001") in by and (1, "S000001") in by
assert (118, "V000001") not in by
assert len(merged) == len(by)
def test_membership_snapshot_rows():
doc = yaml.safe_load((FX / "membership_snippet.yaml").read_text())
rows = C.membership_rows(doc, congress=119, snapshot=date(2025, 4, 1), sha="abc123")
assert [r.committee_id for r in rows] == ["HSWM02"]
assert rows[0].bioguide_id == "E000001"
assert rows[0].rank == 1
assert rows[0].title == "Chair"
assert rows[0].party == "majority"
assert rows[0].source_sha == "abc123"
assert rows[0].snapshot == date(2025, 4, 1)
assert C.HEALTH_COMMITTEES == (
"HSWM",
"HSWM02",
"HSIF",
"HSIF14",
"SSFI",
"SSFI10",
"SSHR",
"SSHR12",
)
def test_membership_row_defaults_missing_rank_and_title():
rows = C.membership_rows(
{"HSIF14": [{"bioguide": "A000001", "party": "minority"}]},
congress=119,
snapshot=date(2025, 4, 1),
sha="s",
)
assert rows[0].rank == 0 and rows[0].title == "" and rows[0].party == "minority"
def test_github_commit_lookup_params():
params = C.commit_params(119)
assert params["until"] == "2025-04-01T00:00:00Z"
assert params["path"] == "committee-membership-current.yaml"
assert params["per_page"] == 1
assert C.commit_params(113)["until"] == "2013-04-01T00:00:00Z"
def test_fetch_assignments_uses_the_cutoff_and_keeps_the_sha(monkeypatch, tmp_path):
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
seen: list[httpx.Request] = []
membership = (FX / "membership_snippet.yaml").read_bytes()
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request)
if request.url.host == "api.github.com":
return httpx.Response(200, json=[{"sha": "deadbeef"}])
return httpx.Response(200, content=membership)
rows = C.fetch_assignments(
_client(handler),
[119],
commit_pacer=Pacer(),
raw_pacer=Pacer(),
cache=tmp_path / "congress",
)
assert seen[0].url.params["until"] == "2025-04-01T00:00:00Z"
assert seen[0].url.params["path"] == "committee-membership-current.yaml"
assert seen[0].url.params["per_page"] == "1"
assert seen[1].url.path.endswith("/deadbeef/committee-membership-current.yaml")
assert [(r.committee_id, r.source_sha, r.snapshot) for r in rows] == [
("HSWM02", "deadbeef", date(2025, 4, 1))
]
n = len(seen)
again = C.fetch_assignments(
_client(handler),
[119],
commit_pacer=Pacer(),
raw_pacer=Pacer(),
cache=tmp_path / "congress",
)
assert again == rows
assert len(seen) == n
def test_empty_commit_list_is_refused():
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json=[])
with pytest.raises(http.Refused, match="no commit"):
C.fetch_assignments(
_client(handler),
[119],
commit_pacer=Pacer(),
raw_pacer=Pacer(),
)
def test_legislators_yaml_that_is_not_a_list_is_refused():
def api(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"members": [], "pagination": {"count": 0}})
def plain(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, content=b"123\n")
with pytest.raises(http.Refused, match="YAML list"):
C.fetch_members(
_client(api),
_client(plain),
[118],
STATES,
member_pacer=Pacer(),
raw_pacer=Pacer(),
)
def test_membership_yaml_that_is_not_a_map_is_refused():
def handler(request: httpx.Request) -> httpx.Response:
if request.url.host == "api.github.com":
return httpx.Response(200, json=[{"sha": "abc"}])
return httpx.Response(200, content=b"- just\n")
with pytest.raises(http.Refused, match="YAML map"):
C.fetch_assignments(
_client(handler),
[119],
commit_pacer=Pacer(),
raw_pacer=Pacer(),
)
def test_member_pages_stop_at_count_and_never_follow_next(monkeypatch, tmp_path):
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
monkeypatch.setattr(C, "LIMIT", 1)
page = _page()
people = (FX / "legislators_snippet.yaml").read_bytes()
seen: list[httpx.Request] = []
def api(request: httpx.Request) -> httpx.Response:
seen.append(request)
offset = int(request.url.params["offset"])
member = page["members"][offset]
return httpx.Response(
200,
json={
"members": [member],
"pagination": {
"count": 2,
"next": "https://api.congress.gov/v3/member/congress/118"
"?offset=1&api_key=SECRET",
},
},
)
def plain(request: httpx.Request) -> httpx.Response:
seen.append(request)
if request.url.path.endswith("legislators-historical.yaml"):
return httpx.Response(200, content=b"[]\n")
return httpx.Response(200, content=people)
rows = C.fetch_members(
_client(api),
_client(plain),
[118],
STATES,
member_pacer=Pacer(),
raw_pacer=Pacer(),
cache=tmp_path / "congress",
)
offsets = [r.url.params["offset"] for r in seen if r.url.host == "api.congress.gov"]
assert offsets == ["0", "1"]
assert all("api_key" not in str(r.url) for r in seen)
congress_hits = [r for r in seen if r.url.host == "api.congress.gov"]
assert [r.url.params["currentMember"] for r in congress_hits] == ["false", "false"]
assert [r.url.params["format"] for r in congress_hits] == ["json", "json"]
by = {r.bioguide_id: r for r in rows}
assert by["D000624"].fec_ids == "H0XX00001"
assert by["Y000033"].district_id == "AK-00"
def test_repeated_first_bioguide_stops_paging(monkeypatch):
monkeypatch.setattr(C, "LIMIT", 1)
member = _page()["members"][0]
calls = {"n": 0}
def api(request: httpx.Request) -> httpx.Response:
calls["n"] += 1
return httpx.Response(
200,
json={"members": [member], "pagination": {"count": 50}},
)
def plain(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, content=b"[]\n")
rows = C.fetch_members(
_client(api),
_client(plain),
[118],
STATES,
member_pacer=Pacer(),
raw_pacer=Pacer(),
)
assert [r.bioguide_id for r in rows] == ["D000624"]
assert calls["n"] == 2
def test_member_json_that_is_not_an_object_is_refused():
def api(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json=["nope"])
def plain(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, content=b"[]\n")
with pytest.raises(http.Refused, match="not JSON"):
C.fetch_members(
_client(api),
_client(plain),
[118],
STATES,
member_pacer=Pacer(),
raw_pacer=Pacer(),
)
def test_members_replace_one_congress_and_keep_the_other():
import duckdb
c = duckdb.connect(":memory:")
first = T.MemberRow(
118,
"D000624",
"Dingell, Debbie",
"Democratic",
"MI",
"MI-12",
"house",
"",
"congress.gov",
)
second = T.MemberRow(
119,
"D000624",
"Dingell, Debbie",
"Democratic",
"MI",
"MI-06",
"house",
"",
"congress.gov",
)
T.write_members(c, [first, second])
T.write_members(
c,
[
T.MemberRow(
118,
"D000624",
"Renamed",
"Democratic",
"MI",
"MI-12",
"house",
"H0XX00001",
"congress.gov",
)
],
)
got = {(r.congress, r.name, r.fec_ids) for r in T.read_members(c)}
assert got == {(118, "Renamed", "H0XX00001"), (119, "Dingell, Debbie", "")}
assert T.write_members(c, []) == 0
assert len(T.read_members(c)) == 2
c.close()
def test_malformed_payloads_are_skipped_or_refused():
page = {
"members": [
"nope",
{
"bioguideId": "D000001",
"district": 1,
"name": "One",
"partyName": "Democratic",
"state": "Michigan",
"terms": {"item": None},
},
]
}
rows = C.members_from_api(page, 118, STATES)
assert [r.bioguide_id for r in rows] == ["D000001"]
assert rows[0].chamber == "house"
people = ["nope", *_people()]
assert [r.bioguide_id for r in C.members_from_yaml(people, [1], STATES)] == [
"V000001",
"S000001",
]
assert set(C.fec_ids_by_bioguide(people)) == {"D000624"}
seats = C.membership_rows(
{"HSWM02": ["nope", {"bioguide": "E000001", "party": "majority", "rank": 1}]},
congress=119,
snapshot=date(2025, 4, 1),
sha="s",
)
assert [r.bioguide_id for r in seats] == ["E000001"]
def bad_json(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, content=b"not-json")
with pytest.raises(http.Refused, match="not JSON"):
C.fetch_members(
_client(bad_json),
_client(lambda request: httpx.Response(200, content=b"[]\n")),
[118],
STATES,
member_pacer=Pacer(),
raw_pacer=Pacer(),
)
def bad_yaml(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, content=b":\n")
def empty_api(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"members": [], "pagination": {"count": 0}})
with pytest.raises(http.Refused, match="YAML"):
C.fetch_members(
_client(empty_api),
_client(bad_yaml),
[118],
STATES,
member_pacer=Pacer(),
raw_pacer=Pacer(),
)
def members_object(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"members": {"bioguideId": "X"}})
assert (
C.fetch_members(
_client(members_object),
_client(lambda request: httpx.Response(200, content=b"[]\n")),
[118],
STATES,
member_pacer=Pacer(),
raw_pacer=Pacer(),
)
== []
)
def no_sha(request: httpx.Request) -> httpx.Response:
if request.url.host == "api.github.com":
return httpx.Response(200, json=[{"sha": ""}])
return httpx.Response(200, content=b"{}\n")
with pytest.raises(http.Refused, match="no commit"):
C.fetch_assignments(
_client(no_sha),
[119],
commit_pacer=Pacer(),
raw_pacer=Pacer(),
)
def test_assignments_replace_one_congress_and_keep_the_other():
import duckdb
c = duckdb.connect(":memory:")
a = T.AssignmentRow(
118, date(2023, 4, 1), "HSWM02", "E000001", 1, "Chair", "majority", "aaa"
)
b = T.AssignmentRow(
119, date(2025, 4, 1), "HSIF14", "E000001", 2, "", "minority", "bbb"
)
T.write_assignments(c, [a, b])
T.write_assignments(c, [a])
assert {(r.congress, r.committee_id) for r in T.read_assignments(c)} == {
(118, "HSWM02"),
(119, "HSIF14"),
}
assert T.write_assignments(c, []) == 0
c.close()

View File

@@ -14,6 +14,7 @@ MODULES = [
"market.tiingo",
"market.lda",
"market.fec",
"market.congress",
]
_PROBE = """

View File

@@ -46,6 +46,8 @@ def test_ensure_tables_idempotent():
"fec_committee",
"fec_totals",
"fec_contribution",
"congress_member",
"committee_assignment",
}
@@ -184,6 +186,20 @@ FEC_CONTRIBUTION = T.FecContributionRow(
"",
"schedule_b/C00411553/2024/1.json",
)
MEMBER = T.MemberRow(
118,
"D000624",
"Dingell, Debbie",
"Democratic",
"MI",
"MI-12",
"house",
"H0XX00001",
"congress.gov",
)
ASSIGNMENT = T.AssignmentRow(
119, date(2025, 4, 1), "HSWM02", "E000001", 1, "Chair", "majority", "abc123"
)
SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY")
ROUNDTRIP = {
@@ -212,6 +228,16 @@ ROUNDTRIP = {
lambda c, r: T.write_fec_contributions(c, [r]),
lambda c: T.read_fec_contributions(c),
),
"congress_member": (
MEMBER,
lambda c, r: T.write_members(c, [r]),
lambda c: T.read_members(c),
),
"committee_assignment": (
ASSIGNMENT,
lambda c, r: T.write_assignments(c, [r]),
lambda c: T.read_assignments(c),
),
"entity": (
ENTITY,
lambda c, r: T.write_entities(c, [r]),
@@ -273,6 +299,8 @@ def test_write_empty_is_noop(con):
assert T.write_fec_committees(con, []) == 0
assert T.write_fec_totals(con, []) == 0
assert T.write_fec_contributions(con, []) == 0
assert T.write_members(con, []) == 0
assert T.write_assignments(con, []) == 0
def test_lda_readers_filter_by_entity(con):