feat(market): members of Congress by canonical district and health-committee assignments per congress
Congress.gov rosters for the 110th through the 119th, with FEC ids from the legislators YAML. Health-committee seats are one legislators snapshot per congress, the last commit on or before April 1. A non-voting delegate is POSTAL-98; an at-large state stays POSTAL-00.
This commit is contained in:
@@ -20,6 +20,7 @@ from typing import Any
|
||||
import httpx
|
||||
import typer
|
||||
|
||||
from market import congress as congress_api
|
||||
from market import edgar, http, tiingo
|
||||
from market import fec as fec_api
|
||||
from market import lda as lda_api
|
||||
@@ -53,6 +54,8 @@ from market.tables import (
|
||||
read_prices,
|
||||
retire_links,
|
||||
retire_organizations,
|
||||
state_codes,
|
||||
write_assignments,
|
||||
write_entities,
|
||||
write_fec_committees,
|
||||
write_fec_contributions,
|
||||
@@ -61,6 +64,7 @@ from market.tables import (
|
||||
write_gap,
|
||||
write_hits,
|
||||
write_links,
|
||||
write_members,
|
||||
write_passages,
|
||||
write_prices,
|
||||
write_securities,
|
||||
@@ -75,6 +79,8 @@ PRICES_GAP = "tiingo:prices"
|
||||
FTS_GAP = "sec-edgar:fts"
|
||||
LDA_GAP = "lda:lda"
|
||||
FEC_GAP = "fec:fec"
|
||||
MEMBERS_GAP = "congress:members"
|
||||
COMMITTEES_GAP = "legislators:committees"
|
||||
|
||||
|
||||
# ── indirections the tests monkeypatch ───────────────────────────────
|
||||
@@ -1448,6 +1454,135 @@ def fec(
|
||||
typer.echo(f"stored: {n_c} committees, {n_t} totals, {n_b} schedule B rows")
|
||||
|
||||
|
||||
def _congress_range(from_congress: int, to_congress: int) -> range:
|
||||
if from_congress > to_congress:
|
||||
typer.echo(f"--from {from_congress} is after --to {to_congress}")
|
||||
raise typer.Exit(2)
|
||||
return range(from_congress, to_congress + 1)
|
||||
|
||||
|
||||
def _member_line(rows: list[Any]) -> str:
|
||||
house = sum(r.chamber == "house" for r in rows)
|
||||
senate = sum(r.chamber == "senate" for r in rows)
|
||||
at_large = sum(r.district_id.endswith("-00") for r in rows)
|
||||
return f"{len(rows)} members ({house} house, {senate} senate, {at_large} at-large)"
|
||||
|
||||
|
||||
@app.command()
|
||||
def members(
|
||||
from_congress: int = typer.Option(congress_api.CONGRESSES.start, "--from"),
|
||||
to_congress: int = typer.Option(congress_api.CONGRESSES.stop - 1, "--to"),
|
||||
write: bool = typer.Option(False, "--write", help="Store market.congress_member."),
|
||||
) -> None:
|
||||
"""Members of Congress (market.congress_member) for the 110th through the
|
||||
119th. Congress.gov supplies the roster; the legislators YAML supplies
|
||||
FEC candidate ids and members the API does not return. A missing key
|
||||
records the gap and writes nothing else."""
|
||||
congresses = _congress_range(from_congress, to_congress)
|
||||
key = _require_key("congress", MEMBERS_GAP, write=write)
|
||||
if key is None:
|
||||
return
|
||||
con = _read()
|
||||
try:
|
||||
states = state_codes(con)
|
||||
finally:
|
||||
con.close()
|
||||
api = http.client("congress", key)
|
||||
plain = http.client("legislators")
|
||||
try:
|
||||
try:
|
||||
rows = congress_api.fetch_members(
|
||||
api,
|
||||
plain,
|
||||
congresses,
|
||||
states,
|
||||
member_pacer=congress_api.member_pacer(),
|
||||
raw_pacer=congress_api.raw_pacer(),
|
||||
cache=http.hoard("congress"),
|
||||
)
|
||||
except http.Refused as exc:
|
||||
_gap(MEMBERS_GAP, str(exc), write=write)
|
||||
return
|
||||
finally:
|
||||
api.close()
|
||||
plain.close()
|
||||
counts = Counter(r.congress for r in rows)
|
||||
typer.echo(_member_line(rows))
|
||||
for n in congresses:
|
||||
typer.echo(f"congress {n}: {counts[n]}")
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write to store")
|
||||
return
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
n = write_members(con, rows)
|
||||
clear_gap(con, MEMBERS_GAP)
|
||||
log_ingest(
|
||||
con,
|
||||
module="market.congress",
|
||||
table_name="market.congress_member",
|
||||
rows=n,
|
||||
source_file=str(http.hoard(http.MANIFEST)),
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: {n} members")
|
||||
|
||||
|
||||
@app.command()
|
||||
def committees(
|
||||
from_congress: int = typer.Option(
|
||||
congress_api.COMMITTEE_CONGRESSES.start, "--from"
|
||||
),
|
||||
to_congress: int = typer.Option(congress_api.COMMITTEE_CONGRESSES.stop - 1, "--to"),
|
||||
write: bool = typer.Option(
|
||||
False, "--write", help="Store market.committee_assignment."
|
||||
),
|
||||
) -> None:
|
||||
"""Health-committee assignments (market.committee_assignment), one
|
||||
legislators snapshot per congress from the 113th on. Keyless. The GitHub
|
||||
commit lookup is paced at 61 seconds."""
|
||||
congresses = _congress_range(from_congress, to_congress)
|
||||
plain = http.client("legislators")
|
||||
try:
|
||||
try:
|
||||
rows = congress_api.fetch_assignments(
|
||||
plain,
|
||||
congresses,
|
||||
commit_pacer=congress_api.commit_pacer(),
|
||||
raw_pacer=congress_api.raw_pacer(),
|
||||
cache=http.hoard("congress"),
|
||||
)
|
||||
except http.Refused as exc:
|
||||
_gap(COMMITTEES_GAP, str(exc), write=write)
|
||||
return
|
||||
finally:
|
||||
plain.close()
|
||||
counts = Counter(r.congress for r in rows)
|
||||
typer.echo(f"{len(rows)} assignments")
|
||||
for n in congresses:
|
||||
typer.echo(f"congress {n}: {counts[n]}")
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write to store")
|
||||
return
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
n = write_assignments(con, rows)
|
||||
clear_gap(con, COMMITTEES_GAP)
|
||||
log_ingest(
|
||||
con,
|
||||
module="market.congress",
|
||||
table_name="market.committee_assignment",
|
||||
rows=n,
|
||||
source_file=str(http.hoard(http.MANIFEST)),
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: {n} assignments")
|
||||
|
||||
|
||||
@app.command()
|
||||
def gaps() -> None:
|
||||
"""Sources that could not be read (market.source_gap)."""
|
||||
|
||||
550
src/market/congress.py
Normal file
550
src/market/congress.py
Normal file
@@ -0,0 +1,550 @@
|
||||
"""Members of Congress and health-committee assignments.
|
||||
|
||||
Congress.gov (``X-Api-Key``, never the query string) lists who served.
|
||||
``currentMember=false`` keeps members who have since left. The legislators
|
||||
YAML (public domain, keyless) supplies FEC candidate ids and fills a
|
||||
congress the API does not return. Committee seats are one
|
||||
congress-legislators snapshot per congress: the last commit of
|
||||
``committee-membership-current.yaml`` on or before April 1 of that
|
||||
congress's first year (M7). ``pagination.next`` is never followed; it can
|
||||
carry the key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import yaml
|
||||
|
||||
from market import http
|
||||
from market.tables import AssignmentRow, MemberRow, district_id
|
||||
|
||||
CONGRESS_API = "https://api.congress.gov/v3/"
|
||||
LEGISLATORS = (
|
||||
"https://raw.githubusercontent.com/unitedstates/congress-legislators/{ref}/{name}"
|
||||
)
|
||||
GH_COMMITS = "https://api.github.com/repos/unitedstates/congress-legislators/commits"
|
||||
REF_MAIN = "main"
|
||||
CURRENT = "legislators-current.yaml"
|
||||
HISTORICAL = "legislators-historical.yaml"
|
||||
MEMBERSHIP = "committee-membership-current.yaml"
|
||||
|
||||
CONGRESSES = range(110, 120)
|
||||
COMMITTEE_CONGRESSES = range(113, 120)
|
||||
HEALTH_COMMITTEES = (
|
||||
"HSWM",
|
||||
"HSWM02",
|
||||
"HSIF",
|
||||
"HSIF14",
|
||||
"SSFI",
|
||||
"SSFI10",
|
||||
"SSHR",
|
||||
"SSHR12",
|
||||
)
|
||||
# Congress.gov sends district 0 for both an at-large state and a delegate.
|
||||
# The canonical id for a non-voting delegate is POSTAL-98 (M7).
|
||||
DELEGATES = frozenset({"AS", "DC", "GU", "MP", "PR", "VI"})
|
||||
LIMIT = 250
|
||||
MEMBER_PACE = 0.8
|
||||
COMMIT_PACE = 61.0
|
||||
RAW_PACE = 0.2
|
||||
|
||||
|
||||
def member_pacer() -> http.Pacer:
|
||||
return http.Pacer(MEMBER_PACE)
|
||||
|
||||
|
||||
def commit_pacer() -> http.Pacer:
|
||||
return http.Pacer(COMMIT_PACE)
|
||||
|
||||
|
||||
def raw_pacer() -> http.Pacer:
|
||||
return http.Pacer(RAW_PACE)
|
||||
|
||||
|
||||
def congress_of(year: int) -> int:
|
||||
"""The Congress in session during *year* (the 119th covers 2025–2026)."""
|
||||
return ((year - 1789) // 2) + 1
|
||||
|
||||
|
||||
def first_year(congress: int) -> int:
|
||||
"""``1789 + 2·(congress − 1)``."""
|
||||
return 1789 + 2 * (congress - 1)
|
||||
|
||||
|
||||
def snapshot_cutoff(congress: int) -> dt.date:
|
||||
"""April 1 of :func:`first_year` (the legislators snapshot for M7)."""
|
||||
return dt.date(first_year(congress), 4, 1)
|
||||
|
||||
|
||||
def congress_start(congress: int) -> dt.date:
|
||||
"""The day *congress* convened. The 74th Congress (1935) is the first
|
||||
to open on January 3; earlier congresses open on March 4."""
|
||||
year = first_year(congress)
|
||||
if congress >= 74:
|
||||
return dt.date(year, 1, 3)
|
||||
return dt.date(year, 3, 4)
|
||||
|
||||
|
||||
def commit_params(congress: int) -> dict[str, Any]:
|
||||
"""GitHub commits query for the snapshot on or before the cutoff."""
|
||||
cutoff = snapshot_cutoff(congress)
|
||||
return {
|
||||
"path": MEMBERSHIP,
|
||||
"until": f"{cutoff.isoformat()}T00:00:00Z",
|
||||
"per_page": 1,
|
||||
}
|
||||
|
||||
|
||||
def _items(terms: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(terms, dict):
|
||||
return []
|
||||
item = terms.get("item")
|
||||
if isinstance(item, dict):
|
||||
return [item]
|
||||
if isinstance(item, list):
|
||||
return [x for x in item if isinstance(x, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def _chamber(items: Sequence[Mapping[str, Any]], congress: int) -> str:
|
||||
"""The chamber of the term that overlaps *congress*. A later overlapping
|
||||
term wins. No term means the House."""
|
||||
y0 = first_year(congress)
|
||||
y1 = y0 + 1
|
||||
chosen: Mapping[str, Any] | None = None
|
||||
for item in items:
|
||||
start = int(item.get("startYear") or 0)
|
||||
end = item.get("endYear")
|
||||
if start <= y1 and (end in (None, "") or int(end) >= y0):
|
||||
chosen = item
|
||||
text = str((chosen or {}).get("chamber") or "")
|
||||
return "senate" if "senate" in text.lower() else "house"
|
||||
|
||||
|
||||
def house_district(postal: str, number: Any) -> str:
|
||||
"""``AK-00`` for an at-large state; ``DC-98`` for a non-voting delegate."""
|
||||
if postal in DELEGATES:
|
||||
return f"{postal}-98"
|
||||
return district_id(postal, number)
|
||||
|
||||
|
||||
def _postal(raw: str, states: Mapping[str, tuple[str, str, str]]) -> str | None:
|
||||
hit = states.get(raw)
|
||||
if hit is None:
|
||||
return None
|
||||
return hit[1]
|
||||
|
||||
|
||||
def members_from_api(
|
||||
payload: Mapping[str, Any],
|
||||
congress: int,
|
||||
states: Mapping[str, tuple[str, str, str]],
|
||||
) -> list[MemberRow]:
|
||||
"""One Congress.gov member page. A state the reference table does not
|
||||
know is skipped. The Senate has an empty ``district_id``."""
|
||||
rows: list[MemberRow] = []
|
||||
for member in payload.get("members") or []:
|
||||
if not isinstance(member, dict):
|
||||
continue
|
||||
bioguide = str(member.get("bioguideId") or "")
|
||||
postal = _postal(str(member.get("state") or ""), states)
|
||||
if not bioguide or postal is None:
|
||||
continue
|
||||
chamber = _chamber(_items(member.get("terms")), congress)
|
||||
district = (
|
||||
""
|
||||
if chamber == "senate"
|
||||
else house_district(postal, member.get("district"))
|
||||
)
|
||||
rows.append(
|
||||
MemberRow(
|
||||
congress=congress,
|
||||
bioguide_id=bioguide,
|
||||
name=str(member.get("name") or ""),
|
||||
party=str(member.get("partyName") or ""),
|
||||
state=postal,
|
||||
district_id=district,
|
||||
chamber=chamber,
|
||||
fec_ids="",
|
||||
source="congress.gov",
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _fec_tuple(person: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
raw = (person.get("id") or {}).get("fec") or []
|
||||
out: list[str] = []
|
||||
for item in raw:
|
||||
text = str(item).strip()
|
||||
if text and text not in out:
|
||||
out.append(text)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _covers(term: Mapping[str, Any], congress: int) -> bool:
|
||||
"""A legislators term is ``[start, end)``. *end* equal to the next
|
||||
congress's first day does not include that congress."""
|
||||
start = dt.date.fromisoformat(str(term.get("start"))[:10])
|
||||
end_raw = term.get("end")
|
||||
end = dt.date.fromisoformat(str(end_raw)[:10]) if end_raw else None
|
||||
opened = congress_start(congress)
|
||||
closed = congress_start(congress + 1)
|
||||
return start < closed and (end is None or end > opened)
|
||||
|
||||
|
||||
def _yaml_term(terms: Sequence[Mapping[str, Any]], congress: int) -> Mapping | None:
|
||||
chosen: Mapping[str, Any] | None = None
|
||||
for term in terms:
|
||||
if not isinstance(term, dict) or not _covers(term, congress):
|
||||
continue
|
||||
if chosen is None or str(term.get("start")) > str(chosen.get("start")):
|
||||
chosen = term
|
||||
return chosen
|
||||
|
||||
|
||||
def members_from_yaml(
|
||||
docs: Sequence[Mapping[str, Any]],
|
||||
congresses: Sequence[int],
|
||||
states: Mapping[str, tuple[str, str, str]],
|
||||
) -> list[MemberRow]:
|
||||
"""One row per requested congress a term covers. ``type: sen`` has an
|
||||
empty ``district_id``; a representative gets ``district_id``."""
|
||||
rows: list[MemberRow] = []
|
||||
for person in docs:
|
||||
if not isinstance(person, dict):
|
||||
continue
|
||||
bioguide = str((person.get("id") or {}).get("bioguide") or "")
|
||||
if not bioguide:
|
||||
continue
|
||||
fec = "|".join(_fec_tuple(person))
|
||||
name = str((person.get("name") or {}).get("official_full") or "")
|
||||
terms = person.get("terms") or []
|
||||
for congress in congresses:
|
||||
term = _yaml_term(terms, congress)
|
||||
if term is None:
|
||||
continue
|
||||
postal = _postal(str(term.get("state") or ""), states)
|
||||
if postal is None:
|
||||
continue
|
||||
chamber = "senate" if str(term.get("type") or "") == "sen" else "house"
|
||||
district = (
|
||||
""
|
||||
if chamber == "senate"
|
||||
else house_district(postal, term.get("district"))
|
||||
)
|
||||
rows.append(
|
||||
MemberRow(
|
||||
congress=congress,
|
||||
bioguide_id=bioguide,
|
||||
name=name,
|
||||
party=str(term.get("party") or ""),
|
||||
state=postal,
|
||||
district_id=district,
|
||||
chamber=chamber,
|
||||
fec_ids=fec,
|
||||
source="legislators",
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def fec_ids_by_bioguide(
|
||||
docs: Sequence[Mapping[str, Any]],
|
||||
) -> dict[str, tuple[str, ...]]:
|
||||
"""FEC candidate ids from legislators YAML, keyed by bioguide."""
|
||||
out: dict[str, tuple[str, ...]] = {}
|
||||
for person in docs:
|
||||
if not isinstance(person, dict):
|
||||
continue
|
||||
bioguide = str((person.get("id") or {}).get("bioguide") or "")
|
||||
ids = _fec_tuple(person)
|
||||
if bioguide and ids:
|
||||
out[bioguide] = ids
|
||||
return out
|
||||
|
||||
|
||||
def merge_members(
|
||||
api_rows: Sequence[MemberRow],
|
||||
yaml_rows: Sequence[MemberRow],
|
||||
fec: Mapping[str, tuple[str, ...]],
|
||||
) -> list[MemberRow]:
|
||||
"""API rows win. FEC ids overlay by bioguide. A YAML row is added only
|
||||
when that congress does not already have the bioguide."""
|
||||
out: list[MemberRow] = []
|
||||
seen: set[tuple[int, str]] = set()
|
||||
for row in api_rows:
|
||||
ids = fec.get(row.bioguide_id, ())
|
||||
if ids:
|
||||
row = replace(row, fec_ids="|".join(ids))
|
||||
out.append(row)
|
||||
seen.add((row.congress, row.bioguide_id))
|
||||
for row in yaml_rows:
|
||||
key = (row.congress, row.bioguide_id)
|
||||
if key not in seen:
|
||||
out.append(row)
|
||||
seen.add(key)
|
||||
return out
|
||||
|
||||
|
||||
def membership_rows(
|
||||
doc: Mapping[str, Any],
|
||||
*,
|
||||
congress: int,
|
||||
snapshot: dt.date,
|
||||
sha: str,
|
||||
) -> list[AssignmentRow]:
|
||||
"""Health subcommittees only. A missing rank is 0 and a missing title
|
||||
is empty."""
|
||||
rows: list[AssignmentRow] = []
|
||||
for committee_id, members in doc.items():
|
||||
if committee_id not in HEALTH_COMMITTEES or not isinstance(members, list):
|
||||
continue
|
||||
for member in members:
|
||||
if not isinstance(member, dict):
|
||||
continue
|
||||
rank = member.get("rank")
|
||||
rows.append(
|
||||
AssignmentRow(
|
||||
congress=congress,
|
||||
snapshot=snapshot,
|
||||
committee_id=str(committee_id),
|
||||
bioguide_id=str(member.get("bioguide") or ""),
|
||||
rank=0 if rank in (None, "") else int(rank),
|
||||
title=str(member.get("title") or ""),
|
||||
party=str(member.get("party") or ""),
|
||||
source_sha=sha,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _cached(path: Path | None) -> bytes | None:
|
||||
if path is None:
|
||||
return None
|
||||
have = http.newest(path)
|
||||
if have.is_file():
|
||||
return have.read_bytes()
|
||||
return None
|
||||
|
||||
|
||||
def _store(path: Path | None, body: bytes, url: str) -> None:
|
||||
if path is None:
|
||||
return
|
||||
where = http.save(path, body)
|
||||
os.utime(where)
|
||||
http.manifest(where, url)
|
||||
|
||||
|
||||
def _json_value(body: bytes, source: str, status: int) -> Any:
|
||||
try:
|
||||
return json.loads(body)
|
||||
except ValueError:
|
||||
raise http.Refused(source, status, "response is not JSON") from None
|
||||
|
||||
|
||||
def _object(body: bytes, source: str, status: int) -> dict[str, Any]:
|
||||
payload = _json_value(body, source, status)
|
||||
if not isinstance(payload, dict):
|
||||
raise http.Refused(source, status, "response is not JSON")
|
||||
return payload
|
||||
|
||||
|
||||
def _commit_list(body: bytes, status: int) -> list[Any]:
|
||||
payload = _json_value(body, "legislators", status)
|
||||
if not isinstance(payload, list) or not payload:
|
||||
raise http.Refused("legislators", status, "no commit on or before the cutoff")
|
||||
return payload
|
||||
|
||||
|
||||
def _yaml_doc(body: bytes, expect: type, status: int) -> Any:
|
||||
try:
|
||||
doc = yaml.safe_load(body)
|
||||
except yaml.YAMLError:
|
||||
doc = None
|
||||
kind = "list" if expect is list else "map"
|
||||
if not isinstance(doc, expect):
|
||||
raise http.Refused("legislators", status, f"response is not a YAML {kind}")
|
||||
return doc
|
||||
|
||||
|
||||
def _download(
|
||||
cl: httpx.Client,
|
||||
url: str,
|
||||
params: Mapping[str, Any] | None,
|
||||
*,
|
||||
pacer: Any,
|
||||
path: Path | None,
|
||||
source: str,
|
||||
) -> bytes:
|
||||
cached = _cached(path)
|
||||
if cached is not None:
|
||||
return cached
|
||||
response = http.get(cl, url, params=params, pacer=pacer, source=source)
|
||||
_store(path, response.content, str(response.request.url))
|
||||
return response.content
|
||||
|
||||
|
||||
def _member_path(cache: Path | None, congress: int, offset: int) -> Path | None:
|
||||
if cache is None:
|
||||
return None
|
||||
return cache / "members" / str(congress) / f"{offset}.json"
|
||||
|
||||
|
||||
def _leg_path(cache: Path | None, name: str) -> Path | None:
|
||||
if cache is None:
|
||||
return None
|
||||
return cache / "legislators" / name
|
||||
|
||||
|
||||
def _commit_path(cache: Path | None, congress: int) -> Path | None:
|
||||
if cache is None:
|
||||
return None
|
||||
day = snapshot_cutoff(congress).isoformat()
|
||||
return cache / "commits" / MEMBERSHIP / f"{day}.json"
|
||||
|
||||
|
||||
def _membership_path(cache: Path | None, sha: str) -> Path | None:
|
||||
if cache is None:
|
||||
return None
|
||||
return cache / "membership" / f"{sha}.yaml"
|
||||
|
||||
|
||||
def _pages(
|
||||
api: httpx.Client,
|
||||
congress: int,
|
||||
states: Mapping[str, tuple[str, str, str]],
|
||||
*,
|
||||
pacer: Any,
|
||||
cache: Path | None,
|
||||
) -> list[MemberRow]:
|
||||
rows: list[MemberRow] = []
|
||||
offset = 0
|
||||
seen_first: str | None = None
|
||||
got = 0
|
||||
while True:
|
||||
url = f"{CONGRESS_API}member/congress/{congress}"
|
||||
body = _download(
|
||||
api,
|
||||
url,
|
||||
{
|
||||
"limit": LIMIT,
|
||||
"offset": offset,
|
||||
"format": "json",
|
||||
"currentMember": "false",
|
||||
},
|
||||
pacer=pacer,
|
||||
path=_member_path(cache, congress, offset),
|
||||
source="congress",
|
||||
)
|
||||
payload = _object(body, "congress", 200)
|
||||
members = payload.get("members") or []
|
||||
if not isinstance(members, list):
|
||||
members = []
|
||||
first = ""
|
||||
if members and isinstance(members[0], dict):
|
||||
first = str(members[0].get("bioguideId") or "")
|
||||
if first and first == seen_first:
|
||||
break
|
||||
if first:
|
||||
seen_first = first
|
||||
rows.extend(members_from_api(payload, congress, states))
|
||||
got += len(members) if isinstance(members, list) else 0
|
||||
count = (payload.get("pagination") or {}).get("count")
|
||||
if (
|
||||
not members
|
||||
or len(members) < LIMIT
|
||||
or (count is not None and got >= int(count))
|
||||
):
|
||||
break
|
||||
offset += LIMIT
|
||||
return rows
|
||||
|
||||
|
||||
def _legislators(
|
||||
plain: httpx.Client, *, pacer: Any, cache: Path | None
|
||||
) -> list[Mapping[str, Any]]:
|
||||
docs: list[Mapping[str, Any]] = []
|
||||
for name in (CURRENT, HISTORICAL):
|
||||
url = LEGISLATORS.format(ref=REF_MAIN, name=name)
|
||||
body = _download(
|
||||
plain,
|
||||
url,
|
||||
None,
|
||||
pacer=pacer,
|
||||
path=_leg_path(cache, name),
|
||||
source="legislators",
|
||||
)
|
||||
doc = _yaml_doc(body, list, 200)
|
||||
docs.extend(person for person in doc if isinstance(person, dict))
|
||||
return docs
|
||||
|
||||
|
||||
def fetch_members(
|
||||
api: httpx.Client,
|
||||
plain: httpx.Client,
|
||||
congresses: Sequence[int],
|
||||
states: Mapping[str, tuple[str, str, str]],
|
||||
*,
|
||||
member_pacer: Any,
|
||||
raw_pacer: Any,
|
||||
cache: Path | None = None,
|
||||
) -> list[MemberRow]:
|
||||
"""Congress.gov pages for *congresses*, then the legislators YAML for
|
||||
FEC ids and members the API did not return."""
|
||||
api_rows: list[MemberRow] = []
|
||||
for congress in congresses:
|
||||
api_rows.extend(_pages(api, congress, states, pacer=member_pacer, cache=cache))
|
||||
docs = _legislators(plain, pacer=raw_pacer, cache=cache)
|
||||
yaml_rows = members_from_yaml(docs, list(congresses), states)
|
||||
return merge_members(api_rows, yaml_rows, fec_ids_by_bioguide(docs))
|
||||
|
||||
|
||||
def fetch_assignments(
|
||||
plain: httpx.Client,
|
||||
congresses: Sequence[int],
|
||||
*,
|
||||
commit_pacer: Any,
|
||||
raw_pacer: Any,
|
||||
cache: Path | None = None,
|
||||
) -> list[AssignmentRow]:
|
||||
"""One health-committee snapshot per congress. An empty commit list is
|
||||
a refusal, so a missing history is not stored as an empty snapshot."""
|
||||
rows: list[AssignmentRow] = []
|
||||
for congress in congresses:
|
||||
body = _download(
|
||||
plain,
|
||||
GH_COMMITS,
|
||||
commit_params(congress),
|
||||
pacer=commit_pacer,
|
||||
path=_commit_path(cache, congress),
|
||||
source="legislators",
|
||||
)
|
||||
commits = _commit_list(body, 200)
|
||||
sha = str(commits[0].get("sha") or "")
|
||||
if not sha:
|
||||
raise http.Refused("legislators", 200, "no commit on or before the cutoff")
|
||||
url = LEGISLATORS.format(ref=sha, name=MEMBERSHIP)
|
||||
raw = _download(
|
||||
plain,
|
||||
url,
|
||||
None,
|
||||
pacer=raw_pacer,
|
||||
path=_membership_path(cache, sha),
|
||||
source="legislators",
|
||||
)
|
||||
doc = _yaml_doc(raw, dict, 200)
|
||||
rows.extend(
|
||||
membership_rows(
|
||||
doc, congress=congress, snapshot=snapshot_cutoff(congress), sha=sha
|
||||
)
|
||||
)
|
||||
return rows
|
||||
@@ -85,6 +85,15 @@ CREATE TABLE IF NOT EXISTS market.fec_contribution (
|
||||
candidate_id VARCHAR, candidate_name VARCHAR, candidate_office VARCHAR,
|
||||
candidate_state VARCHAR, candidate_district VARCHAR, memo_code VARCHAR,
|
||||
source_file VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.congress_member (
|
||||
congress INTEGER, bioguide_id VARCHAR, name VARCHAR, party VARCHAR,
|
||||
state VARCHAR, district_id VARCHAR, chamber VARCHAR, fec_ids VARCHAR,
|
||||
source VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.committee_assignment (
|
||||
congress INTEGER, snapshot DATE, committee_id VARCHAR, bioguide_id VARCHAR,
|
||||
rank INTEGER, title VARCHAR, party VARCHAR, source_sha VARCHAR
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -339,6 +348,38 @@ class FecContributionRow:
|
||||
source_file: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MemberRow:
|
||||
"""One member of one Congress. ``fec_ids`` is a ``|``-joined list.
|
||||
``district_id`` is ``MI-12`` or at-large ``AK-00`` for the House and
|
||||
``""`` for the Senate. ``state`` is the postal code."""
|
||||
|
||||
congress: int
|
||||
bioguide_id: str
|
||||
name: str
|
||||
party: str
|
||||
state: str
|
||||
district_id: str
|
||||
chamber: str
|
||||
fec_ids: str
|
||||
source: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AssignmentRow:
|
||||
"""One health-committee seat in one congress's legislators snapshot.
|
||||
``snapshot`` is April 1 of that congress's first year (M7)."""
|
||||
|
||||
congress: int
|
||||
snapshot: date
|
||||
committee_id: str
|
||||
bioguide_id: str
|
||||
rank: int
|
||||
title: str
|
||||
party: str
|
||||
source_sha: str
|
||||
|
||||
|
||||
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
|
||||
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
|
||||
a statement per row, far too slow for filing lists)."""
|
||||
@@ -774,6 +815,53 @@ def read_fec_contributions(
|
||||
)
|
||||
|
||||
|
||||
# ── congress ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def write_members(con: Any, rows: Sequence[MemberRow]) -> int:
|
||||
"""Replace the members of each congress present in *rows*."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
for chunk in _chunks(sorted({r.congress for r in rows})):
|
||||
con.execute(
|
||||
"DELETE FROM market.congress_member WHERE congress IN "
|
||||
f"({','.join('?' * len(chunk))})",
|
||||
list(chunk),
|
||||
)
|
||||
return _insert(con, "market.congress_member", rows)
|
||||
|
||||
|
||||
def read_members(con: Any) -> list[MemberRow]:
|
||||
rows = _select(con, MemberRow, "market.congress_member")
|
||||
return sorted(
|
||||
(MemberRow(*r) for r in rows),
|
||||
key=lambda r: (r.congress, r.bioguide_id, r.district_id),
|
||||
)
|
||||
|
||||
|
||||
def write_assignments(con: Any, rows: Sequence[AssignmentRow]) -> int:
|
||||
"""Replace the assignments of each congress present in *rows*."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
for chunk in _chunks(sorted({r.congress for r in rows})):
|
||||
con.execute(
|
||||
"DELETE FROM market.committee_assignment WHERE congress IN "
|
||||
f"({','.join('?' * len(chunk))})",
|
||||
list(chunk),
|
||||
)
|
||||
return _insert(con, "market.committee_assignment", rows)
|
||||
|
||||
|
||||
def read_assignments(con: Any) -> list[AssignmentRow]:
|
||||
rows = _select(con, AssignmentRow, "market.committee_assignment")
|
||||
return sorted(
|
||||
(AssignmentRow(*r) for r in rows),
|
||||
key=lambda r: (r.congress, r.committee_id, r.rank, r.bioguide_id),
|
||||
)
|
||||
|
||||
|
||||
# ── districts ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -2181,3 +2181,205 @@ def test_fec_cli_limit_keeps_the_gap_and_empty_registry(fec, monkeypatch):
|
||||
assert empty.exit_code == 1 and "no tracked entities" in empty.output
|
||||
blank = runner.invoke(app, ["market", "fec", "--cycles", ","])
|
||||
assert blank.exit_code == 2 and "--cycles is empty" in _plain(blank.output)
|
||||
|
||||
|
||||
# ── congress ─────────────────────────────────────────────────────────
|
||||
|
||||
CONGRESS_SECRET = "congress-cli-key-do-not-leak"
|
||||
|
||||
|
||||
def _congress_states(c) -> None:
|
||||
c.execute("CREATE SCHEMA IF NOT EXISTS reference_data")
|
||||
c.execute(
|
||||
"CREATE TABLE IF NOT EXISTS reference_data.ansi_fips_state ("
|
||||
"ansi_fips_state_code VARCHAR, ansi_fips_state_abbreviation VARCHAR, "
|
||||
"ansi_fips_state_name VARCHAR)"
|
||||
)
|
||||
c.execute(
|
||||
"INSERT INTO reference_data.ansi_fips_state VALUES "
|
||||
"('26', 'MI', 'Michigan'), ('02', 'AK', 'Alaska'), ('51', 'VA', 'Virginia')"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def congress(env, monkeypatch):
|
||||
from market import congress as congress_api
|
||||
|
||||
c, store, published, hoard = env
|
||||
_congress_states(c)
|
||||
T.ensure_tables(c)
|
||||
seen: list[httpx.Request] = []
|
||||
members = json.loads((FX / "congress_members_118.json").read_text())
|
||||
people = (FX / "legislators_snippet.yaml").read_bytes()
|
||||
membership = (FX / "membership_snippet.yaml").read_bytes()
|
||||
state = {"status": 200}
|
||||
for name in ("member_pacer", "raw_pacer", "commit_pacer"):
|
||||
monkeypatch.setattr(congress_api, name, lambda: _Pacer())
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request)
|
||||
if state["status"] != 200:
|
||||
return httpx.Response(state["status"], text="no")
|
||||
host = request.url.host
|
||||
if host == "api.congress.gov":
|
||||
return httpx.Response(200, json=members)
|
||||
if host == "api.github.com":
|
||||
return httpx.Response(200, json=[{"sha": "deadbeef"}])
|
||||
if request.url.path.endswith("legislators-historical.yaml"):
|
||||
return httpx.Response(200, content=b"[]\n")
|
||||
if request.url.path.endswith("committee-membership-current.yaml"):
|
||||
return httpx.Response(200, content=membership)
|
||||
return httpx.Response(200, content=people)
|
||||
|
||||
def client(source, key=None):
|
||||
headers = {"X-Api-Key": key} if key else {}
|
||||
return httpx.Client(transport=httpx.MockTransport(handler), headers=headers)
|
||||
|
||||
monkeypatch.setattr(http, "client", client)
|
||||
monkeypatch.setattr(http, "api_key", lambda source, env=None: CONGRESS_SECRET)
|
||||
return c, store, published, hoard, seen, state
|
||||
|
||||
|
||||
def test_members_cli_no_write_prints_counts(congress):
|
||||
c, _, published, _, seen, _ = congress
|
||||
res = runner.invoke(app, ["market", "members", "--from", "118", "--to", "118"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "2 members (2 house, 0 senate, 1 at-large)" in res.output
|
||||
assert "congress 118: 2" in res.output
|
||||
assert "dry run: pass --write" in res.output
|
||||
assert _count(c, "market.congress_member") == 0
|
||||
assert published == []
|
||||
assert any(r.url.host == "api.congress.gov" for r in seen)
|
||||
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
|
||||
assert all("api_key" not in str(r.url) for r in seen)
|
||||
congress_hits = [r for r in seen if r.url.host == "api.congress.gov"]
|
||||
assert congress_hits[0].headers["x-api-key"] == CONGRESS_SECRET
|
||||
github_hits = [r for r in seen if r.url.host != "api.congress.gov"]
|
||||
assert github_hits
|
||||
assert all(r.headers.get("x-api-key") in (None, "") for r in github_hits)
|
||||
|
||||
|
||||
def test_members_cli_write(congress):
|
||||
c, _, published, hoard, seen, _ = congress
|
||||
T.write_gap(c, "congress:members", "old")
|
||||
res = runner.invoke(
|
||||
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stored: 2 members" in res.output
|
||||
rows = {r.bioguide_id: r for r in T.read_members(c)}
|
||||
assert rows["D000624"].district_id == "MI-12"
|
||||
assert rows["D000624"].fec_ids == "H0XX00001"
|
||||
assert rows["D000624"].source == "congress.gov"
|
||||
assert rows["Y000033"].district_id == "AK-00"
|
||||
assert rows["Y000033"].state == "AK"
|
||||
assert T.read_gaps(c) == []
|
||||
assert published == [True]
|
||||
assert {
|
||||
(t, n)
|
||||
for t, n in c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||||
} == {("market.congress_member", 2)}
|
||||
manifest = (hoard / "MANIFEST.jsonl").read_text()
|
||||
assert CONGRESS_SECRET not in manifest
|
||||
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
|
||||
kept = T.MemberRow(
|
||||
117,
|
||||
"Z000001",
|
||||
"Other",
|
||||
"Republican",
|
||||
"MI",
|
||||
"MI-01",
|
||||
"house",
|
||||
"",
|
||||
"congress.gov",
|
||||
)
|
||||
T.write_members(c, [kept])
|
||||
assert {r.congress for r in T.read_members(c)} == {117, 118}
|
||||
|
||||
|
||||
def test_members_cli_missing_key_gap_only(congress, monkeypatch):
|
||||
c, _, published, _, seen, _ = congress
|
||||
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
|
||||
res = runner.invoke(
|
||||
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: congress:members:" in res.output
|
||||
assert "CONGRESS_API_KEY" in res.output
|
||||
assert "API_DATA_GOV_KEY" in res.output
|
||||
assert seen == []
|
||||
assert [g.source for g in T.read_gaps(c)] == ["congress:members"]
|
||||
assert _count(c, "market.congress_member") == 0
|
||||
assert published == [True]
|
||||
|
||||
|
||||
def test_members_cli_refusal_prints_gap(congress):
|
||||
c, _, published, _, seen, state = congress
|
||||
state["status"] = 403
|
||||
res = runner.invoke(
|
||||
app, ["market", "members", "--from", "118", "--to", "118", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: congress:members: congress: HTTP 403" in res.output
|
||||
assert CONGRESS_SECRET not in res.output
|
||||
assert _count(c, "market.congress_member") == 0
|
||||
assert len(seen) == 1
|
||||
assert seen[0].url.host == "api.congress.gov"
|
||||
assert seen[0].headers["x-api-key"] == CONGRESS_SECRET
|
||||
assert CONGRESS_SECRET not in str(seen[0].url)
|
||||
|
||||
|
||||
def test_members_cli_from_after_to(congress):
|
||||
res = runner.invoke(app, ["market", "members", "--from", "119", "--to", "110"])
|
||||
assert res.exit_code == 2
|
||||
assert "--from 119 is after --to 110" in res.output
|
||||
|
||||
|
||||
def test_committees_cli_write(congress):
|
||||
c, _, published, hoard, seen, _ = congress
|
||||
T.write_gap(c, "legislators:committees", "old")
|
||||
dry = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "119"])
|
||||
assert dry.exit_code == 0, dry.output
|
||||
assert "1 assignments" in dry.output
|
||||
assert "dry run: pass --write" in dry.output
|
||||
assert _count(c, "market.committee_assignment") == 0
|
||||
assert published == []
|
||||
res = runner.invoke(
|
||||
app, ["market", "committees", "--from", "119", "--to", "119", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stored: 1 assignments" in res.output
|
||||
(row,) = T.read_assignments(c)
|
||||
assert row.committee_id == "HSWM02"
|
||||
assert row.bioguide_id == "E000001"
|
||||
assert row.snapshot == date(2025, 4, 1)
|
||||
assert row.source_sha == "deadbeef"
|
||||
assert row.congress == 119
|
||||
assert T.read_gaps(c) == []
|
||||
assert published == [True]
|
||||
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
|
||||
assert all(r.headers.get("x-api-key") in (None, "") for r in seen)
|
||||
manifest = (hoard / "MANIFEST.jsonl").read_text()
|
||||
assert CONGRESS_SECRET not in manifest
|
||||
logged = c.execute("SELECT table_name, rows FROM cms.ingest_log").fetchall()
|
||||
assert ("market.committee_assignment", 1) in logged
|
||||
|
||||
|
||||
def test_committees_cli_refusal_prints_gap(congress):
|
||||
c, _, published, _, seen, state = congress
|
||||
state["status"] = 403
|
||||
res = runner.invoke(
|
||||
app, ["market", "committees", "--from", "119", "--to", "119", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: legislators:committees: legislators: HTTP 403" in res.output
|
||||
assert CONGRESS_SECRET not in res.output
|
||||
assert _count(c, "market.committee_assignment") == 0
|
||||
assert seen
|
||||
assert published == [True]
|
||||
|
||||
|
||||
def test_committees_cli_from_after_to(congress):
|
||||
res = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "113"])
|
||||
assert res.exit_code == 2
|
||||
assert "--from 119 is after --to 113" in res.output
|
||||
|
||||
29
tests/market/fixtures/congress_members_118.json
Normal file
29
tests/market/fixtures/congress_members_118.json
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"members": [
|
||||
{
|
||||
"bioguideId": "D000624",
|
||||
"district": 12,
|
||||
"name": "Dingell, Debbie",
|
||||
"partyName": "Democratic",
|
||||
"state": "Michigan",
|
||||
"terms": {
|
||||
"item": [
|
||||
{"chamber": "House of Representatives", "startYear": 2023}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"bioguideId": "Y000033",
|
||||
"district": null,
|
||||
"name": "Young, Don",
|
||||
"partyName": "Republican",
|
||||
"state": "Alaska",
|
||||
"terms": {
|
||||
"item": [
|
||||
{"chamber": "House of Representatives", "startYear": 2021}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"pagination": {"count": 2}
|
||||
}
|
||||
35
tests/market/fixtures/legislators_snippet.yaml
Normal file
35
tests/market/fixtures/legislators_snippet.yaml
Normal file
@@ -0,0 +1,35 @@
|
||||
- id:
|
||||
bioguide: D000624
|
||||
fec:
|
||||
- H0XX00001
|
||||
name:
|
||||
official_full: Debbie Dingell
|
||||
terms:
|
||||
- type: rep
|
||||
start: '2023-01-03'
|
||||
end: '2025-01-03'
|
||||
state: MI
|
||||
district: 12
|
||||
party: Democrat
|
||||
- id:
|
||||
bioguide: V000001
|
||||
fec: []
|
||||
name:
|
||||
official_full: Virginia Rep
|
||||
terms:
|
||||
- type: rep
|
||||
start: '1789-03-04'
|
||||
end: '1791-03-03'
|
||||
state: VA
|
||||
district: 9
|
||||
party: Federalist
|
||||
- id:
|
||||
bioguide: S000001
|
||||
name:
|
||||
official_full: Virginia Senator
|
||||
terms:
|
||||
- type: sen
|
||||
start: '1789-03-04'
|
||||
end: '1791-03-03'
|
||||
state: VA
|
||||
party: Federalist
|
||||
11
tests/market/fixtures/membership_snippet.yaml
Normal file
11
tests/market/fixtures/membership_snippet.yaml
Normal file
@@ -0,0 +1,11 @@
|
||||
HSWM02:
|
||||
- name: Example, Member
|
||||
party: majority
|
||||
rank: 1
|
||||
title: Chair
|
||||
bioguide: E000001
|
||||
HSAG:
|
||||
- name: Other, Member
|
||||
party: minority
|
||||
rank: 2
|
||||
bioguide: X000001
|
||||
677
tests/market/test_congress.py
Normal file
677
tests/market/test_congress.py
Normal file
@@ -0,0 +1,677 @@
|
||||
"""market.congress: members by district and health-committee snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from market import congress as C
|
||||
from market import http
|
||||
from market import tables as T
|
||||
|
||||
FX = Path(__file__).parent / "fixtures"
|
||||
STATES = {
|
||||
"Michigan": ("26", "MI", "Michigan"),
|
||||
"MI": ("26", "MI", "Michigan"),
|
||||
"Alaska": ("02", "AK", "Alaska"),
|
||||
"AK": ("02", "AK", "Alaska"),
|
||||
"Virginia": ("51", "VA", "Virginia"),
|
||||
"VA": ("51", "VA", "Virginia"),
|
||||
}
|
||||
|
||||
|
||||
def _people() -> list:
|
||||
return yaml.safe_load((FX / "legislators_snippet.yaml").read_text())
|
||||
|
||||
|
||||
def _page() -> dict:
|
||||
return json.loads((FX / "congress_members_118.json").read_text())
|
||||
|
||||
|
||||
class Pacer:
|
||||
def wait(self):
|
||||
pass
|
||||
|
||||
|
||||
def _client(handler) -> httpx.Client:
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
def test_congress_of_year():
|
||||
assert C.congress_of(2013) == 113
|
||||
assert C.congress_of(2026) == 119
|
||||
assert C.congress_of(2027) == 120
|
||||
assert C.first_year(119) == 2025
|
||||
assert C.first_year(1) == 1789
|
||||
assert C.member_pacer().min_interval == 0.8
|
||||
assert C.commit_pacer().min_interval == 61
|
||||
assert C.raw_pacer().min_interval == 0.2
|
||||
assert C.LIMIT == 250
|
||||
assert C.CONGRESSES == range(110, 120)
|
||||
assert C.COMMITTEE_CONGRESSES == range(113, 120)
|
||||
|
||||
|
||||
def test_snapshot_cutoff():
|
||||
assert C.snapshot_cutoff(119) == date(2025, 4, 1)
|
||||
assert C.snapshot_cutoff(113) == date(2013, 4, 1)
|
||||
|
||||
|
||||
def test_members_from_api_page():
|
||||
rows = C.members_from_api(_page(), 118, STATES)
|
||||
dingell = next(r for r in rows if r.bioguide_id == "D000624")
|
||||
assert dingell.state == "MI"
|
||||
assert dingell.district_id == "MI-12"
|
||||
assert dingell.chamber == "house"
|
||||
assert dingell.party == "Democratic"
|
||||
assert dingell.name == "Dingell, Debbie"
|
||||
assert dingell.congress == 118
|
||||
assert dingell.source == "congress.gov"
|
||||
assert dingell.fec_ids == ""
|
||||
|
||||
|
||||
def test_delegate_district_is_98_and_at_large_state_stays_00():
|
||||
states = {
|
||||
**STATES,
|
||||
"District of Columbia": ("11", "DC", "District of Columbia"),
|
||||
"DC": ("11", "DC", "District of Columbia"),
|
||||
}
|
||||
payload = {
|
||||
"members": [
|
||||
{
|
||||
"bioguideId": "N000147",
|
||||
"district": 0,
|
||||
"name": "Norton, Eleanor Holmes",
|
||||
"partyName": "Democratic",
|
||||
"state": "District of Columbia",
|
||||
"terms": {
|
||||
"item": [{"chamber": "House of Representatives", "startYear": 1991}]
|
||||
},
|
||||
},
|
||||
{
|
||||
"bioguideId": "B001323",
|
||||
"district": 0,
|
||||
"name": "Begich, Nicholas J.",
|
||||
"partyName": "Republican",
|
||||
"state": "Alaska",
|
||||
"terms": {
|
||||
"item": [{"chamber": "House of Representatives", "startYear": 2025}]
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
rows = {r.bioguide_id: r for r in C.members_from_api(payload, 119, states)}
|
||||
assert rows["N000147"].district_id == "DC-98"
|
||||
assert rows["N000147"].chamber == "house"
|
||||
assert rows["B001323"].district_id == "AK-00"
|
||||
docs = [
|
||||
{
|
||||
"id": {"bioguide": "N000147"},
|
||||
"name": {"official_full": "Eleanor Holmes Norton"},
|
||||
"terms": [
|
||||
{
|
||||
"type": "rep",
|
||||
"start": "2025-01-03",
|
||||
"end": "2027-01-03",
|
||||
"state": "DC",
|
||||
"district": 0,
|
||||
"party": "Democrat",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
(yaml_row,) = C.members_from_yaml(docs, [119], states)
|
||||
assert yaml_row.district_id == "DC-98"
|
||||
|
||||
|
||||
def test_at_large_member_district():
|
||||
rows = C.members_from_api(_page(), 118, STATES)
|
||||
young = next(r for r in rows if r.bioguide_id == "Y000033")
|
||||
assert young.state == "AK"
|
||||
assert young.district_id == "AK-00"
|
||||
assert young.chamber == "house"
|
||||
|
||||
|
||||
def test_senate_member_has_empty_district():
|
||||
payload = {
|
||||
"members": [
|
||||
{
|
||||
"bioguideId": "S000033",
|
||||
"district": None,
|
||||
"name": "Stabenow, Debbie",
|
||||
"partyName": "Democratic",
|
||||
"state": "Michigan",
|
||||
"terms": {"item": [{"chamber": "Senate", "startYear": 2001}]},
|
||||
}
|
||||
]
|
||||
}
|
||||
(row,) = C.members_from_api(payload, 118, STATES)
|
||||
assert row.chamber == "senate"
|
||||
assert row.district_id == ""
|
||||
assert row.state == "MI"
|
||||
|
||||
|
||||
def test_api_skips_unknown_state_and_accepts_a_single_term_dict():
|
||||
payload = {
|
||||
"members": [
|
||||
{
|
||||
"bioguideId": "Z000001",
|
||||
"district": 1,
|
||||
"name": "Nobody",
|
||||
"partyName": "Independent",
|
||||
"state": "Narnia",
|
||||
"terms": {
|
||||
"item": [{"chamber": "House of Representatives", "startYear": 2023}]
|
||||
},
|
||||
},
|
||||
{
|
||||
"bioguideId": "D000001",
|
||||
"district": 1,
|
||||
"name": "Delegate, One",
|
||||
"partyName": "Democratic",
|
||||
"state": "Michigan",
|
||||
"terms": {
|
||||
"item": {"chamber": "House of Representatives", "startYear": 2023}
|
||||
},
|
||||
},
|
||||
{
|
||||
"bioguideId": "H000001",
|
||||
"district": 2,
|
||||
"name": "No Terms",
|
||||
"partyName": "Republican",
|
||||
"state": "Michigan",
|
||||
},
|
||||
]
|
||||
}
|
||||
rows = C.members_from_api(payload, 118, STATES)
|
||||
assert [r.bioguide_id for r in rows] == ["D000001", "H000001"]
|
||||
assert rows[0].district_id == "MI-01"
|
||||
assert rows[1].chamber == "house"
|
||||
assert rows[1].district_id == "MI-02"
|
||||
|
||||
|
||||
def test_api_chamber_follows_the_term_overlapping_the_congress():
|
||||
payload = {
|
||||
"members": [
|
||||
{
|
||||
"bioguideId": "B000001",
|
||||
"district": 4,
|
||||
"name": "Both, Chambers",
|
||||
"partyName": "Republican",
|
||||
"state": "Michigan",
|
||||
"terms": {
|
||||
"item": [
|
||||
{
|
||||
"chamber": "House of Representatives",
|
||||
"startYear": 2013,
|
||||
"endYear": 2019,
|
||||
},
|
||||
{"chamber": "Senate", "startYear": 2019},
|
||||
]
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
house = C.members_from_api(payload, 115, STATES)[0]
|
||||
senate = C.members_from_api(payload, 116, STATES)[0]
|
||||
assert house.chamber == "house" and house.district_id == "MI-04"
|
||||
assert senate.chamber == "senate" and senate.district_id == ""
|
||||
|
||||
|
||||
def test_members_from_legislators_yaml():
|
||||
docs = [
|
||||
*_people(),
|
||||
{"id": {}, "name": {"official_full": "No Id"}, "terms": []},
|
||||
{
|
||||
"id": {"bioguide": "N000009"},
|
||||
"name": {"official_full": "Narnia"},
|
||||
"terms": [
|
||||
{
|
||||
"type": "rep",
|
||||
"start": "1789-03-04",
|
||||
"end": "1791-03-03",
|
||||
"state": "Narnia",
|
||||
"district": 1,
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
rows = C.members_from_yaml(docs, [1], STATES)
|
||||
by_id = {r.bioguide_id: r for r in rows}
|
||||
assert set(by_id) == {"V000001", "S000001"}
|
||||
assert by_id["V000001"].district_id == "VA-09"
|
||||
assert by_id["V000001"].state == "VA"
|
||||
assert by_id["V000001"].chamber == "house"
|
||||
assert by_id["V000001"].source == "legislators"
|
||||
assert by_id["V000001"].congress == 1
|
||||
assert by_id["S000001"].district_id == ""
|
||||
assert by_id["S000001"].chamber == "senate"
|
||||
later = C.members_from_yaml(docs, [1, 2], STATES)
|
||||
assert {r.congress for r in later} == {1}
|
||||
modern = C.members_from_yaml(docs, [118], STATES)
|
||||
assert [r.bioguide_id for r in modern] == ["D000624"]
|
||||
assert modern[0].district_id == "MI-12"
|
||||
assert modern[0].fec_ids == "H0XX00001"
|
||||
|
||||
|
||||
def test_yaml_term_ending_on_the_next_congress_start_is_excluded():
|
||||
docs = [
|
||||
{
|
||||
"id": {"bioguide": "N000001", "fec": ["H0AA00001", "H0AA00001"]},
|
||||
"name": {"official_full": "Next Boundary"},
|
||||
"terms": [
|
||||
{
|
||||
"type": "rep",
|
||||
"start": "2025-01-03",
|
||||
"end": "2027-01-03",
|
||||
"state": "VA",
|
||||
"district": 1,
|
||||
"party": "Democrat",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
rows = C.members_from_yaml(docs, [119, 120], STATES)
|
||||
assert [(r.congress, r.district_id, r.fec_ids) for r in rows] == [
|
||||
(119, "VA-01", "H0AA00001")
|
||||
]
|
||||
|
||||
|
||||
def test_fec_ids_by_bioguide():
|
||||
ids = C.fec_ids_by_bioguide(_people())
|
||||
assert ids["D000624"] == ("H0XX00001",)
|
||||
assert set(ids) == {"D000624"}
|
||||
|
||||
|
||||
def test_merge_members_overlays_fec_ids_and_adds_the_api_gap():
|
||||
api = C.members_from_api(_page(), 118, STATES)
|
||||
yaml_rows = C.members_from_yaml(_people(), [1, 118], STATES)
|
||||
fec = C.fec_ids_by_bioguide(_people())
|
||||
merged = C.merge_members(api, yaml_rows, fec)
|
||||
by = {(r.congress, r.bioguide_id): r for r in merged}
|
||||
assert by[(118, "D000624")].fec_ids == "H0XX00001"
|
||||
assert by[(118, "D000624")].source == "congress.gov"
|
||||
assert (118, "D000624") in by and (118, "Y000033") in by
|
||||
assert (1, "V000001") in by and (1, "S000001") in by
|
||||
assert (118, "V000001") not in by
|
||||
assert len(merged) == len(by)
|
||||
|
||||
|
||||
def test_membership_snapshot_rows():
|
||||
doc = yaml.safe_load((FX / "membership_snippet.yaml").read_text())
|
||||
rows = C.membership_rows(doc, congress=119, snapshot=date(2025, 4, 1), sha="abc123")
|
||||
assert [r.committee_id for r in rows] == ["HSWM02"]
|
||||
assert rows[0].bioguide_id == "E000001"
|
||||
assert rows[0].rank == 1
|
||||
assert rows[0].title == "Chair"
|
||||
assert rows[0].party == "majority"
|
||||
assert rows[0].source_sha == "abc123"
|
||||
assert rows[0].snapshot == date(2025, 4, 1)
|
||||
assert C.HEALTH_COMMITTEES == (
|
||||
"HSWM",
|
||||
"HSWM02",
|
||||
"HSIF",
|
||||
"HSIF14",
|
||||
"SSFI",
|
||||
"SSFI10",
|
||||
"SSHR",
|
||||
"SSHR12",
|
||||
)
|
||||
|
||||
|
||||
def test_membership_row_defaults_missing_rank_and_title():
|
||||
rows = C.membership_rows(
|
||||
{"HSIF14": [{"bioguide": "A000001", "party": "minority"}]},
|
||||
congress=119,
|
||||
snapshot=date(2025, 4, 1),
|
||||
sha="s",
|
||||
)
|
||||
assert rows[0].rank == 0 and rows[0].title == "" and rows[0].party == "minority"
|
||||
|
||||
|
||||
def test_github_commit_lookup_params():
|
||||
params = C.commit_params(119)
|
||||
assert params["until"] == "2025-04-01T00:00:00Z"
|
||||
assert params["path"] == "committee-membership-current.yaml"
|
||||
assert params["per_page"] == 1
|
||||
assert C.commit_params(113)["until"] == "2013-04-01T00:00:00Z"
|
||||
|
||||
|
||||
def test_fetch_assignments_uses_the_cutoff_and_keeps_the_sha(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
|
||||
seen: list[httpx.Request] = []
|
||||
membership = (FX / "membership_snippet.yaml").read_bytes()
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request)
|
||||
if request.url.host == "api.github.com":
|
||||
return httpx.Response(200, json=[{"sha": "deadbeef"}])
|
||||
return httpx.Response(200, content=membership)
|
||||
|
||||
rows = C.fetch_assignments(
|
||||
_client(handler),
|
||||
[119],
|
||||
commit_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
cache=tmp_path / "congress",
|
||||
)
|
||||
assert seen[0].url.params["until"] == "2025-04-01T00:00:00Z"
|
||||
assert seen[0].url.params["path"] == "committee-membership-current.yaml"
|
||||
assert seen[0].url.params["per_page"] == "1"
|
||||
assert seen[1].url.path.endswith("/deadbeef/committee-membership-current.yaml")
|
||||
assert [(r.committee_id, r.source_sha, r.snapshot) for r in rows] == [
|
||||
("HSWM02", "deadbeef", date(2025, 4, 1))
|
||||
]
|
||||
n = len(seen)
|
||||
again = C.fetch_assignments(
|
||||
_client(handler),
|
||||
[119],
|
||||
commit_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
cache=tmp_path / "congress",
|
||||
)
|
||||
assert again == rows
|
||||
assert len(seen) == n
|
||||
|
||||
|
||||
def test_empty_commit_list_is_refused():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=[])
|
||||
|
||||
with pytest.raises(http.Refused, match="no commit"):
|
||||
C.fetch_assignments(
|
||||
_client(handler),
|
||||
[119],
|
||||
commit_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
)
|
||||
|
||||
|
||||
def test_legislators_yaml_that_is_not_a_list_is_refused():
|
||||
def api(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"members": [], "pagination": {"count": 0}})
|
||||
|
||||
def plain(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, content=b"123\n")
|
||||
|
||||
with pytest.raises(http.Refused, match="YAML list"):
|
||||
C.fetch_members(
|
||||
_client(api),
|
||||
_client(plain),
|
||||
[118],
|
||||
STATES,
|
||||
member_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
)
|
||||
|
||||
|
||||
def test_membership_yaml_that_is_not_a_map_is_refused():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == "api.github.com":
|
||||
return httpx.Response(200, json=[{"sha": "abc"}])
|
||||
return httpx.Response(200, content=b"- just\n")
|
||||
|
||||
with pytest.raises(http.Refused, match="YAML map"):
|
||||
C.fetch_assignments(
|
||||
_client(handler),
|
||||
[119],
|
||||
commit_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
)
|
||||
|
||||
|
||||
def test_member_pages_stop_at_count_and_never_follow_next(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
|
||||
monkeypatch.setattr(C, "LIMIT", 1)
|
||||
page = _page()
|
||||
people = (FX / "legislators_snippet.yaml").read_bytes()
|
||||
seen: list[httpx.Request] = []
|
||||
|
||||
def api(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request)
|
||||
offset = int(request.url.params["offset"])
|
||||
member = page["members"][offset]
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"members": [member],
|
||||
"pagination": {
|
||||
"count": 2,
|
||||
"next": "https://api.congress.gov/v3/member/congress/118"
|
||||
"?offset=1&api_key=SECRET",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def plain(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request)
|
||||
if request.url.path.endswith("legislators-historical.yaml"):
|
||||
return httpx.Response(200, content=b"[]\n")
|
||||
return httpx.Response(200, content=people)
|
||||
|
||||
rows = C.fetch_members(
|
||||
_client(api),
|
||||
_client(plain),
|
||||
[118],
|
||||
STATES,
|
||||
member_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
cache=tmp_path / "congress",
|
||||
)
|
||||
offsets = [r.url.params["offset"] for r in seen if r.url.host == "api.congress.gov"]
|
||||
assert offsets == ["0", "1"]
|
||||
assert all("api_key" not in str(r.url) for r in seen)
|
||||
congress_hits = [r for r in seen if r.url.host == "api.congress.gov"]
|
||||
assert [r.url.params["currentMember"] for r in congress_hits] == ["false", "false"]
|
||||
assert [r.url.params["format"] for r in congress_hits] == ["json", "json"]
|
||||
by = {r.bioguide_id: r for r in rows}
|
||||
assert by["D000624"].fec_ids == "H0XX00001"
|
||||
assert by["Y000033"].district_id == "AK-00"
|
||||
|
||||
|
||||
def test_repeated_first_bioguide_stops_paging(monkeypatch):
|
||||
monkeypatch.setattr(C, "LIMIT", 1)
|
||||
member = _page()["members"][0]
|
||||
calls = {"n": 0}
|
||||
|
||||
def api(request: httpx.Request) -> httpx.Response:
|
||||
calls["n"] += 1
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"members": [member], "pagination": {"count": 50}},
|
||||
)
|
||||
|
||||
def plain(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, content=b"[]\n")
|
||||
|
||||
rows = C.fetch_members(
|
||||
_client(api),
|
||||
_client(plain),
|
||||
[118],
|
||||
STATES,
|
||||
member_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
)
|
||||
assert [r.bioguide_id for r in rows] == ["D000624"]
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
def test_member_json_that_is_not_an_object_is_refused():
|
||||
def api(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=["nope"])
|
||||
|
||||
def plain(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, content=b"[]\n")
|
||||
|
||||
with pytest.raises(http.Refused, match="not JSON"):
|
||||
C.fetch_members(
|
||||
_client(api),
|
||||
_client(plain),
|
||||
[118],
|
||||
STATES,
|
||||
member_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
)
|
||||
|
||||
|
||||
def test_members_replace_one_congress_and_keep_the_other():
|
||||
import duckdb
|
||||
|
||||
c = duckdb.connect(":memory:")
|
||||
first = T.MemberRow(
|
||||
118,
|
||||
"D000624",
|
||||
"Dingell, Debbie",
|
||||
"Democratic",
|
||||
"MI",
|
||||
"MI-12",
|
||||
"house",
|
||||
"",
|
||||
"congress.gov",
|
||||
)
|
||||
second = T.MemberRow(
|
||||
119,
|
||||
"D000624",
|
||||
"Dingell, Debbie",
|
||||
"Democratic",
|
||||
"MI",
|
||||
"MI-06",
|
||||
"house",
|
||||
"",
|
||||
"congress.gov",
|
||||
)
|
||||
T.write_members(c, [first, second])
|
||||
T.write_members(
|
||||
c,
|
||||
[
|
||||
T.MemberRow(
|
||||
118,
|
||||
"D000624",
|
||||
"Renamed",
|
||||
"Democratic",
|
||||
"MI",
|
||||
"MI-12",
|
||||
"house",
|
||||
"H0XX00001",
|
||||
"congress.gov",
|
||||
)
|
||||
],
|
||||
)
|
||||
got = {(r.congress, r.name, r.fec_ids) for r in T.read_members(c)}
|
||||
assert got == {(118, "Renamed", "H0XX00001"), (119, "Dingell, Debbie", "")}
|
||||
assert T.write_members(c, []) == 0
|
||||
assert len(T.read_members(c)) == 2
|
||||
c.close()
|
||||
|
||||
|
||||
def test_malformed_payloads_are_skipped_or_refused():
|
||||
page = {
|
||||
"members": [
|
||||
"nope",
|
||||
{
|
||||
"bioguideId": "D000001",
|
||||
"district": 1,
|
||||
"name": "One",
|
||||
"partyName": "Democratic",
|
||||
"state": "Michigan",
|
||||
"terms": {"item": None},
|
||||
},
|
||||
]
|
||||
}
|
||||
rows = C.members_from_api(page, 118, STATES)
|
||||
assert [r.bioguide_id for r in rows] == ["D000001"]
|
||||
assert rows[0].chamber == "house"
|
||||
people = ["nope", *_people()]
|
||||
assert [r.bioguide_id for r in C.members_from_yaml(people, [1], STATES)] == [
|
||||
"V000001",
|
||||
"S000001",
|
||||
]
|
||||
assert set(C.fec_ids_by_bioguide(people)) == {"D000624"}
|
||||
seats = C.membership_rows(
|
||||
{"HSWM02": ["nope", {"bioguide": "E000001", "party": "majority", "rank": 1}]},
|
||||
congress=119,
|
||||
snapshot=date(2025, 4, 1),
|
||||
sha="s",
|
||||
)
|
||||
assert [r.bioguide_id for r in seats] == ["E000001"]
|
||||
|
||||
def bad_json(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, content=b"not-json")
|
||||
|
||||
with pytest.raises(http.Refused, match="not JSON"):
|
||||
C.fetch_members(
|
||||
_client(bad_json),
|
||||
_client(lambda request: httpx.Response(200, content=b"[]\n")),
|
||||
[118],
|
||||
STATES,
|
||||
member_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
)
|
||||
|
||||
def bad_yaml(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, content=b":\n")
|
||||
|
||||
def empty_api(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"members": [], "pagination": {"count": 0}})
|
||||
|
||||
with pytest.raises(http.Refused, match="YAML"):
|
||||
C.fetch_members(
|
||||
_client(empty_api),
|
||||
_client(bad_yaml),
|
||||
[118],
|
||||
STATES,
|
||||
member_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
)
|
||||
|
||||
def members_object(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"members": {"bioguideId": "X"}})
|
||||
|
||||
assert (
|
||||
C.fetch_members(
|
||||
_client(members_object),
|
||||
_client(lambda request: httpx.Response(200, content=b"[]\n")),
|
||||
[118],
|
||||
STATES,
|
||||
member_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
)
|
||||
== []
|
||||
)
|
||||
|
||||
def no_sha(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == "api.github.com":
|
||||
return httpx.Response(200, json=[{"sha": ""}])
|
||||
return httpx.Response(200, content=b"{}\n")
|
||||
|
||||
with pytest.raises(http.Refused, match="no commit"):
|
||||
C.fetch_assignments(
|
||||
_client(no_sha),
|
||||
[119],
|
||||
commit_pacer=Pacer(),
|
||||
raw_pacer=Pacer(),
|
||||
)
|
||||
|
||||
|
||||
def test_assignments_replace_one_congress_and_keep_the_other():
|
||||
import duckdb
|
||||
|
||||
c = duckdb.connect(":memory:")
|
||||
a = T.AssignmentRow(
|
||||
118, date(2023, 4, 1), "HSWM02", "E000001", 1, "Chair", "majority", "aaa"
|
||||
)
|
||||
b = T.AssignmentRow(
|
||||
119, date(2025, 4, 1), "HSIF14", "E000001", 2, "", "minority", "bbb"
|
||||
)
|
||||
T.write_assignments(c, [a, b])
|
||||
T.write_assignments(c, [a])
|
||||
assert {(r.congress, r.committee_id) for r in T.read_assignments(c)} == {
|
||||
(118, "HSWM02"),
|
||||
(119, "HSIF14"),
|
||||
}
|
||||
assert T.write_assignments(c, []) == 0
|
||||
c.close()
|
||||
@@ -14,6 +14,7 @@ MODULES = [
|
||||
"market.tiingo",
|
||||
"market.lda",
|
||||
"market.fec",
|
||||
"market.congress",
|
||||
]
|
||||
|
||||
_PROBE = """
|
||||
|
||||
@@ -46,6 +46,8 @@ def test_ensure_tables_idempotent():
|
||||
"fec_committee",
|
||||
"fec_totals",
|
||||
"fec_contribution",
|
||||
"congress_member",
|
||||
"committee_assignment",
|
||||
}
|
||||
|
||||
|
||||
@@ -184,6 +186,20 @@ FEC_CONTRIBUTION = T.FecContributionRow(
|
||||
"",
|
||||
"schedule_b/C00411553/2024/1.json",
|
||||
)
|
||||
MEMBER = T.MemberRow(
|
||||
118,
|
||||
"D000624",
|
||||
"Dingell, Debbie",
|
||||
"Democratic",
|
||||
"MI",
|
||||
"MI-12",
|
||||
"house",
|
||||
"H0XX00001",
|
||||
"congress.gov",
|
||||
)
|
||||
ASSIGNMENT = T.AssignmentRow(
|
||||
119, date(2025, 4, 1), "HSWM02", "E000001", 1, "Chair", "majority", "abc123"
|
||||
)
|
||||
SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY")
|
||||
|
||||
ROUNDTRIP = {
|
||||
@@ -212,6 +228,16 @@ ROUNDTRIP = {
|
||||
lambda c, r: T.write_fec_contributions(c, [r]),
|
||||
lambda c: T.read_fec_contributions(c),
|
||||
),
|
||||
"congress_member": (
|
||||
MEMBER,
|
||||
lambda c, r: T.write_members(c, [r]),
|
||||
lambda c: T.read_members(c),
|
||||
),
|
||||
"committee_assignment": (
|
||||
ASSIGNMENT,
|
||||
lambda c, r: T.write_assignments(c, [r]),
|
||||
lambda c: T.read_assignments(c),
|
||||
),
|
||||
"entity": (
|
||||
ENTITY,
|
||||
lambda c, r: T.write_entities(c, [r]),
|
||||
@@ -273,6 +299,8 @@ def test_write_empty_is_noop(con):
|
||||
assert T.write_fec_committees(con, []) == 0
|
||||
assert T.write_fec_totals(con, []) == 0
|
||||
assert T.write_fec_contributions(con, []) == 0
|
||||
assert T.write_members(con, []) == 0
|
||||
assert T.write_assignments(con, []) == 0
|
||||
|
||||
|
||||
def test_lda_readers_filter_by_entity(con):
|
||||
|
||||
Reference in New Issue
Block a user