feat(market): Medicare physician-payment bills with actions, sponsors and cosponsors (Congress.gov)

This commit is contained in:
kert
2026-11-16 19:16:26 -05:00
parent 2ee82e269d
commit 6164945e61
10 changed files with 1187 additions and 11 deletions

View File

@@ -56,6 +56,9 @@ from market.tables import (
retire_organizations,
state_codes,
write_assignments,
write_bill_actions,
write_bill_members,
write_bills,
write_entities,
write_fec_committees,
write_fec_contributions,
@@ -81,6 +84,7 @@ LDA_GAP = "lda:lda"
FEC_GAP = "fec:fec"
MEMBERS_GAP = "congress:members"
COMMITTEES_GAP = "legislators:committees"
BILLS_GAP = "congress:bills"
# ── indirections the tests monkeypatch ───────────────────────────────
@@ -1583,6 +1587,74 @@ def committees(
typer.echo(f"stored: {n} assignments")
@app.command()
def bills(
from_congress: int = typer.Option(congress_api.CONGRESSES.start, "--from"),
to_congress: int = typer.Option(congress_api.CONGRESSES.stop - 1, "--to"),
limit: int = typer.Option(
0, "--limit", help="Stop after N uncached requests (0 = no limit)."
),
write: bool = typer.Option(False, "--write", help="Store market.bill."),
) -> None:
"""Medicare physician-payment bills (market.bill) for the 110th through
the 119th. A title match is kept only when the bill's policy area is
Health. A request limit stores nothing: a partial congress would replace
that congress's bills."""
congresses = _congress_range(from_congress, to_congress)
key = _require_key("congress", BILLS_GAP, write=write)
if key is None:
return
api = http.client("congress", key)
try:
try:
found, actions, sponsors, stopped = congress_api.fetch_bills(
api,
congresses,
pacer=congress_api.member_pacer(),
cache=http.hoard("congress"),
limit=limit,
)
except http.Refused as exc:
_gap(BILLS_GAP, str(exc), write=write)
return
finally:
api.close()
counts = Counter(row.congress for row in found)
for row in found:
typer.echo(f"{row.congress} {row.bill_type} {row.number} {row.matched_term}")
typer.echo(f"{len(found)} bills")
for n in congresses:
typer.echo(f"congress {n}: {counts[n]}")
if stopped:
typer.echo("stopped: request limit")
return
if not write:
typer.echo("dry run: pass --write")
return
store = _store()
cited = [
dataclasses.replace(row, item_key=congress_api.cite_bill(store, row))
for row in found
]
from cms.ingest_log import log_ingest
with _batch() as con:
ensure_tables(con)
n = write_bills(con, cited)
write_bill_actions(con, actions)
write_bill_members(con, sponsors)
clear_gap(con, BILLS_GAP)
log_ingest(
con,
module="market.congress",
table_name="market.bill",
rows=n,
source_file=str(http.hoard(http.MANIFEST)),
)
_publish()
typer.echo(f"stored: {n} bills")
@app.command()
def gaps() -> None:
"""Sources that could not be read (market.source_gap)."""

View File

@@ -1,13 +1,14 @@
"""Members of Congress and health-committee assignments.
"""Members of Congress, health-committee assignments, and physician-payment bills.
Congress.gov (``X-Api-Key``, never the query string) lists who served.
``currentMember=false`` keeps members who have since left. The legislators
YAML (public domain, keyless) supplies FEC candidate ids and fills a
congress the API does not return. Committee seats are one
congress-legislators snapshot per congress: the last commit of
``committee-membership-current.yaml`` on or before April 1 of that
congress's first year (M7). ``pagination.next`` is never followed; it can
carry the key.
Congress.gov (``X-Api-Key``, never the query string) lists who served and
which bills were introduced. ``currentMember=false`` keeps members who have
since left. The legislators YAML (public domain, keyless) supplies FEC
candidate ids and fills a congress the API does not return. Committee seats
are one congress-legislators snapshot per congress: the last commit of
``committee-membership-current.yaml`` on or before April 1 of that congress's
first year (M7). A bill is kept when its title matches a physician-payment
term and its detail says the policy area is Health. ``pagination.next`` is
never followed; it can carry the key.
"""
from __future__ import annotations
@@ -15,6 +16,7 @@ from __future__ import annotations
import datetime as dt
import json
import os
import re
from collections.abc import Mapping, Sequence
from dataclasses import replace
from pathlib import Path
@@ -24,7 +26,14 @@ import httpx
import yaml
from market import http
from market.tables import AssignmentRow, MemberRow, district_id
from market.tables import (
AssignmentRow,
BillActionRow,
BillMemberRow,
BillRow,
MemberRow,
district_id,
)
CONGRESS_API = "https://api.congress.gov/v3/"
LEGISLATORS = (
@@ -55,6 +64,21 @@ LIMIT = 250
MEMBER_PACE = 0.8
COMMIT_PACE = 61.0
RAW_PACE = 0.2
BILL_RE = re.compile(
r"(?i)\b(physician fee schedule|physician payment|conversion factor|"
r"sustainable growth rate|relative value|evaluation and management|"
r"telehealth|home-based|house call|care management|primary care)\b"
)
_BILL_SLUG = {
"HR": "house-bill",
"S": "senate-bill",
"HRES": "house-resolution",
"SRES": "senate-resolution",
"HJRES": "house-joint-resolution",
"SJRES": "senate-joint-resolution",
"HCONRES": "house-concurrent-resolution",
"SCONRES": "senate-concurrent-resolution",
}
def member_pacer() -> http.Pacer:
@@ -69,6 +93,181 @@ def raw_pacer() -> http.Pacer:
return http.Pacer(RAW_PACE)
def match_title(title: str) -> str | None:
"""The physician-payment term in *title*, lowercased, or ``None``."""
found = BILL_RE.search(title or "")
if found is None:
return None
return found.group(1).casefold()
def ordinal(n: int) -> str:
"""``111th``, ``121st``, ``122nd``, ``123rd``. 11–13 stay ``th``."""
if 10 <= (n % 100) <= 20:
suffix = "th"
else:
suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th")
return f"{n}{suffix}"
def bill_url(congress: int, bill_type: str, number: int) -> str:
"""The public Congress.gov bill page (no API key)."""
slug = _BILL_SLUG[bill_type.upper()]
return f"https://www.congress.gov/bill/{ordinal(congress)}-congress/{slug}/{number}"
def _day(value: Any) -> dt.date | None:
if not value:
return None
try:
return dt.date.fromisoformat(str(value)[:10])
except ValueError:
return None
def bill_from_detail(payload: Mapping[str, Any]) -> BillRow | None:
"""One kept bill, or ``None`` when the title misses or the policy area
is not Health."""
bill = payload.get("bill") if isinstance(payload, Mapping) else None
if not isinstance(bill, dict):
return None
title = str(bill.get("title") or "")
term = match_title(title)
if term is None:
return None
area = bill.get("policyArea")
name = area.get("name") if isinstance(area, dict) else None
if name != "Health":
return None
congress = int(bill["congress"])
bill_type = str(bill.get("type") or "").upper()
number = int(bill["number"])
latest = bill.get("latestAction")
if not isinstance(latest, dict):
latest = {}
return BillRow(
congress=congress,
bill_type=bill_type,
number=number,
title=title,
introduced=_day(bill.get("introducedDate")),
policy_area="Health",
latest_action_date=_day(latest.get("actionDate")),
latest_action=str(latest.get("text") or ""),
matched_term=term,
url=bill_url(congress, bill_type, number),
item_key="",
)
def action_rows(
payload: Mapping[str, Any], congress: int, bill_type: str, number: int
) -> list[BillActionRow]:
"""Dated actions. A non-object, or an action with no parseable date, is
skipped."""
raw = payload.get("actions") if isinstance(payload, Mapping) else None
if not isinstance(raw, list):
return []
rows: list[BillActionRow] = []
for item in raw:
if not isinstance(item, dict):
continue
day = _day(item.get("actionDate"))
if day is None:
continue
codes: list[str] = []
committees = item.get("committees") or []
if isinstance(committees, list):
for committee in committees:
if isinstance(committee, dict) and committee.get("systemCode"):
codes.append(str(committee["systemCode"]))
rows.append(
BillActionRow(
congress,
bill_type,
number,
day,
str(item.get("text") or ""),
str(item.get("actionCode") or ""),
"|".join(codes),
)
)
return rows
def member_rows(
detail: Mapping[str, Any],
cosponsors: Mapping[str, Any],
congress: int,
bill_type: str,
number: int,
) -> list[BillMemberRow]:
"""Sponsors from the detail (``since`` is the introduced date) and
cosponsors from the cosponsors payload."""
bill = detail.get("bill") if isinstance(detail, Mapping) else None
if not isinstance(bill, dict):
bill = {}
introduced = _day(bill.get("introducedDate"))
rows: list[BillMemberRow] = []
sponsors = bill.get("sponsors") or []
if isinstance(sponsors, list):
for sponsor in sponsors:
if isinstance(sponsor, dict) and sponsor.get("bioguideId"):
rows.append(
BillMemberRow(
congress,
bill_type,
number,
str(sponsor["bioguideId"]),
"sponsor",
introduced,
)
)
raw = cosponsors.get("cosponsors") if isinstance(cosponsors, Mapping) else None
if not isinstance(raw, list):
return rows
for person in raw:
if not isinstance(person, dict) or not person.get("bioguideId"):
continue
rows.append(
BillMemberRow(
congress,
bill_type,
number,
str(person["bioguideId"]),
"cosponsor",
_day(person.get("sponsorshipDate")),
)
)
return rows
def cite_bill(store: Any, row: BillRow) -> str:
"""A bib ``source`` item for one kept bill. Returns its item key."""
from bib.item import Source
from bib.tag import Tag
item = Source(
title=row.title,
url=row.url,
doc_type="bill",
abstract=(
f"{row.bill_type} {row.number} in the {ordinal(row.congress)} "
f"Congress, policy area {row.policy_area}, matched on "
f"{row.matched_term}."
),
access_date=dt.date.today().isoformat(),
)
return store.upsert(
item,
tags=[
Tag.module("market"),
Tag.table("market.bill"),
Tag.source("congress"),
],
)
def congress_of(year: int) -> int:
"""The Congress in session during *year* (the 119th covers 2025–2026)."""
return ((year - 1789) // 2) + 1
@@ -376,6 +575,20 @@ def _yaml_doc(body: bytes, expect: type, status: int) -> Any:
return doc
class _BillBudget:
"""At most *limit* uncached requests (0 = unlimited)."""
def __init__(self, limit: int) -> None:
self.limit = limit
self.used = 0
def allow(self) -> bool:
if self.limit and self.used >= self.limit:
return False
self.used += 1
return True
def _download(
cl: httpx.Client,
url: str,
@@ -384,10 +597,13 @@ def _download(
pacer: Any,
path: Path | None,
source: str,
) -> bytes:
budget: _BillBudget | None = None,
) -> bytes | None:
cached = _cached(path)
if cached is not None:
return cached
if budget is not None and not budget.allow():
return None
response = http.get(cl, url, params=params, pacer=pacer, source=source)
_store(path, response.content, str(response.request.url))
return response.content
@@ -548,3 +764,216 @@ def fetch_assignments(
)
)
return rows
def _bill_cache(cache: Path | None, congress: int, *parts: str) -> Path | None:
if cache is None:
return None
return cache.joinpath("bills", str(congress), *parts)
def _offset_path(cache: Path | None, congress: int, *prefix: str) -> Any:
def path_for(offset: int) -> Path | None:
return _bill_cache(cache, congress, *prefix, f"{offset}.json")
return path_for
def _first_stamp(kind: str, item: Mapping[str, Any]) -> str:
if kind == "bills":
return f"{item.get('type')}:{item.get('number')}"
if kind == "cosponsors":
return str(item.get("bioguideId") or "")
return f"{item.get('actionDate')}|{item.get('text')}"
def _page_items(
api: httpx.Client,
url: str,
kind: str,
*,
pacer: Any,
path_for: Any,
budget: _BillBudget | None,
) -> list[dict[str, Any]] | None:
"""Pages of *kind* (``bills``, ``actions``, or ``cosponsors``). ``None``
when the request budget stops the pull. A repeated first row ends the
page walk; ``pagination.next`` is never followed."""
items: list[dict[str, Any]] = []
offset = 0
seen_first: str | None = None
got = 0
while True:
body = _download(
api,
url,
{"limit": LIMIT, "offset": offset, "format": "json"},
pacer=pacer,
path=path_for(offset),
source="congress",
budget=budget,
)
if body is None:
return None
payload = _object(body, "congress", 200)
raw = payload.get(kind) or []
if not isinstance(raw, list):
raw = []
first = ""
if raw and isinstance(raw[0], dict):
first = _first_stamp(kind, raw[0])
if first and first == seen_first:
break
if first:
seen_first = first
items.extend(row for row in raw if isinstance(row, dict))
got += len(raw)
page = payload.get("pagination")
count = page.get("count") if isinstance(page, dict) else None
done = not raw or len(raw) < LIMIT
if not done and count is not None:
done = got >= int(count)
if done:
break
offset += LIMIT
return items
def _bill_detail(
api: httpx.Client,
congress: int,
bill_type: str,
number: int,
*,
pacer: Any,
cache: Path | None,
budget: _BillBudget | None,
) -> dict[str, Any] | None:
url = f"{CONGRESS_API}bill/{congress}/{bill_type.lower()}/{number}"
body = _download(
api,
url,
{"format": "json"},
pacer=pacer,
path=_bill_cache(cache, congress, bill_type, str(number), "detail.json"),
source="congress",
budget=budget,
)
if body is None:
return None
return _object(body, "congress", 200)
def fetch_bills(
api: httpx.Client,
congresses: Sequence[int],
*,
pacer: Any,
cache: Path | None = None,
limit: int = 0,
) -> tuple[list[BillRow], list[BillActionRow], list[BillMemberRow], bool]:
"""Title matches in *congresses* whose detail policy area is Health,
with their actions, sponsors and cosponsors. The fourth value is true
when *limit* uncached requests were spent before the pull finished.
A stopped pull returns what it already confirmed and nothing past the
request that the budget refused."""
budget = _BillBudget(limit)
bills: list[BillRow] = []
actions: list[BillActionRow] = []
members: list[BillMemberRow] = []
for congress in congresses:
listed = _page_items(
api,
f"{CONGRESS_API}bill/{congress}",
"bills",
pacer=pacer,
path_for=_offset_path(cache, congress),
budget=budget,
)
if listed is None:
return bills, actions, members, True
stopped = _keep_matches(
api,
congress,
listed,
pacer=pacer,
cache=cache,
budget=budget,
bills=bills,
actions=actions,
members=members,
)
if stopped:
return bills, actions, members, True
return bills, actions, members, False
def _keep_matches(
api: httpx.Client,
congress: int,
listed: Sequence[Mapping[str, Any]],
*,
pacer: Any,
cache: Path | None,
budget: _BillBudget,
bills: list[BillRow],
actions: list[BillActionRow],
members: list[BillMemberRow],
) -> bool:
"""Append Health title-matches from one congress. True when the budget
stops the pull before this congress is finished."""
for item in listed:
if match_title(str(item.get("title") or "")) is None:
continue
bill_type = str(item.get("type") or "").upper()
try:
number = int(item["number"])
except (KeyError, TypeError, ValueError):
continue
if bill_type not in _BILL_SLUG:
continue
detail = _bill_detail(
api, congress, bill_type, number, pacer=pacer, cache=cache, budget=budget
)
if detail is None:
return True
row = bill_from_detail(detail)
if row is None:
continue
stem = f"{CONGRESS_API}bill/{congress}/{bill_type.lower()}/{number}"
act = _page_items(
api,
f"{stem}/actions",
"actions",
pacer=pacer,
path_for=_offset_path(cache, congress, bill_type, str(number), "actions"),
budget=budget,
)
if act is None:
return True
cos = _page_items(
api,
f"{stem}/cosponsors",
"cosponsors",
pacer=pacer,
path_for=_offset_path(
cache, congress, bill_type, str(number), "cosponsors"
),
budget=budget,
)
if cos is None:
return True
bills.append(row)
actions.extend(
action_rows({"actions": act}, row.congress, row.bill_type, row.number)
)
members.extend(
member_rows(
detail,
{"cosponsors": cos},
row.congress,
row.bill_type,
row.number,
)
)
return False

View File

@@ -94,6 +94,19 @@ CREATE TABLE IF NOT EXISTS market.congress_member (
CREATE TABLE IF NOT EXISTS market.committee_assignment (
congress INTEGER, snapshot DATE, committee_id VARCHAR, bioguide_id VARCHAR,
rank INTEGER, title VARCHAR, party VARCHAR, source_sha VARCHAR
);
CREATE TABLE IF NOT EXISTS market.bill (
congress INTEGER, bill_type VARCHAR, number INTEGER, title VARCHAR,
introduced DATE, policy_area VARCHAR, latest_action_date DATE,
latest_action VARCHAR, matched_term VARCHAR, url VARCHAR, item_key VARCHAR
);
CREATE TABLE IF NOT EXISTS market.bill_action (
congress INTEGER, bill_type VARCHAR, number INTEGER, action_date DATE,
text VARCHAR, action_code VARCHAR, committee_codes VARCHAR
);
CREATE TABLE IF NOT EXISTS market.bill_member (
congress INTEGER, bill_type VARCHAR, number INTEGER, bioguide_id VARCHAR,
role VARCHAR, since DATE
)
"""
@@ -380,6 +393,51 @@ class AssignmentRow:
source_sha: str
@dataclass(frozen=True)
class BillRow:
"""One Congress.gov bill kept because its title matched a physician-payment
term and its policy area is Health. ``item_key`` is empty until cited."""
congress: int
bill_type: str
number: int
title: str
introduced: date | None
policy_area: str
latest_action_date: date | None
latest_action: str
matched_term: str
url: str
item_key: str = ""
@dataclass(frozen=True)
class BillActionRow:
"""One dated action. ``committee_codes`` is a ``|``-joined list of
Congress.gov ``systemCode`` values, in source order."""
congress: int
bill_type: str
number: int
action_date: date
text: str
action_code: str
committee_codes: str
@dataclass(frozen=True)
class BillMemberRow:
"""A sponsor (``since`` is the introduced date) or a cosponsor (``since``
is the sponsorship date)."""
congress: int
bill_type: str
number: int
bioguide_id: str
role: str
since: date | None
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
a statement per row, far too slow for filing lists)."""
@@ -862,6 +920,65 @@ def read_assignments(con: Any) -> list[AssignmentRow]:
)
def _replace_congresses(con: Any, table: str, rows: Sequence[Any]) -> None:
for chunk in _chunks(sorted({r.congress for r in rows})):
con.execute(
f"DELETE FROM {table} WHERE congress IN ({','.join('?' * len(chunk))})",
list(chunk),
)
def write_bills(con: Any, rows: Sequence[BillRow]) -> int:
"""Replace the bills of each congress present in *rows*."""
ensure_tables(con)
if not rows:
return 0
_replace_congresses(con, "market.bill", rows)
return _insert(con, "market.bill", rows)
def read_bills(con: Any) -> list[BillRow]:
rows = _select(con, BillRow, "market.bill")
return sorted(
(BillRow(*r) for r in rows),
key=lambda r: (r.congress, r.bill_type, r.number),
)
def write_bill_actions(con: Any, rows: Sequence[BillActionRow]) -> int:
"""Replace the actions of each congress present in *rows*."""
ensure_tables(con)
if not rows:
return 0
_replace_congresses(con, "market.bill_action", rows)
return _insert(con, "market.bill_action", rows)
def read_bill_actions(con: Any) -> list[BillActionRow]:
rows = _select(con, BillActionRow, "market.bill_action")
return sorted(
(BillActionRow(*r) for r in rows),
key=lambda r: (r.congress, r.bill_type, r.number, r.action_date, r.text),
)
def write_bill_members(con: Any, rows: Sequence[BillMemberRow]) -> int:
"""Replace the sponsors and cosponsors of each congress present in *rows*."""
ensure_tables(con)
if not rows:
return 0
_replace_congresses(con, "market.bill_member", rows)
return _insert(con, "market.bill_member", rows)
def read_bill_members(con: Any) -> list[BillMemberRow]:
rows = _select(con, BillMemberRow, "market.bill_member")
return sorted(
(BillMemberRow(*r) for r in rows),
key=lambda r: (r.congress, r.bill_type, r.number, r.role, r.bioguide_id),
)
# ── districts ────────────────────────────────────────────────────────

View File

@@ -2210,6 +2210,10 @@ def congress(env, monkeypatch):
T.ensure_tables(c)
seen: list[httpx.Request] = []
members = json.loads((FX / "congress_members_118.json").read_text())
bills = json.loads((FX / "congress_bills_page.json").read_text())
detail = json.loads((FX / "congress_bill_detail.json").read_text())
actions = json.loads((FX / "congress_bill_actions.json").read_text())
cosponsors = json.loads((FX / "congress_bill_cosponsors.json").read_text())
people = (FX / "legislators_snippet.yaml").read_bytes()
membership = (FX / "membership_snippet.yaml").read_bytes()
state = {"status": 200}
@@ -2222,6 +2226,16 @@ def congress(env, monkeypatch):
return httpx.Response(state["status"], text="no")
host = request.url.host
if host == "api.congress.gov":
path = request.url.path
if path.endswith("/actions"):
return httpx.Response(200, json=actions)
if path.endswith("/cosponsors"):
return httpx.Response(200, json=cosponsors)
parts = [part for part in path.split("/") if part]
if "bill" in parts and len(parts) >= 5:
return httpx.Response(200, json=detail)
if "bill" in parts:
return httpx.Response(200, json=bills)
return httpx.Response(200, json=members)
if host == "api.github.com":
return httpx.Response(200, json=[{"sha": "deadbeef"}])
@@ -2365,6 +2379,139 @@ def test_committees_cli_write(congress):
assert ("market.committee_assignment", 1) in logged
def test_bills_cli_no_write_lists_matches(congress):
c, _, published, _, seen, _ = congress
res = runner.invoke(app, ["market", "bills", "--from", "118", "--to", "118"])
assert res.exit_code == 0, res.output
assert "118 HR 1234 physician fee schedule" in res.output
assert "1 bills" in res.output
assert "congress 118: 1" in res.output
assert "Farm Bill" not in res.output
assert "dry run: pass --write" in res.output
assert _count(c, "market.bill") == 0
assert published == []
hits = [r for r in seen if r.url.host == "api.congress.gov"]
assert hits
assert all(CONGRESS_SECRET not in str(r.url) for r in hits)
assert all("api_key" not in str(r.url) for r in hits)
assert hits[0].headers["x-api-key"] == CONGRESS_SECRET
def test_bills_cli_offline_write(congress):
c, store, published, hoard, seen, _ = congress
T.write_gap(c, "congress:bills", "old")
res = runner.invoke(
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "stored: 1 bills" in res.output
(bill,) = T.read_bills(c)
assert bill.bill_type == "HR"
assert bill.number == 1234
assert bill.matched_term == "physician fee schedule"
assert bill.policy_area == "Health"
assert bill.introduced == date(2023, 1, 9)
assert bill.item_key
assert len(T.read_bill_actions(c)) == 3
roles = sorted((m.role, m.bioguide_id) for m in T.read_bill_members(c))
assert roles == [
("cosponsor", "S000001"),
("cosponsor", "Y000033"),
("sponsor", "D000624"),
]
assert T.read_gaps(c) == []
assert published == [True]
assert any(
(i.url or "").endswith("/118th-congress/house-bill/1234")
for i in store.list_items()
)
assert ("market.bill", 1) in c.execute(
"SELECT table_name, rows FROM cms.ingest_log"
).fetchall()
manifest = (hoard / "MANIFEST.jsonl").read_text()
assert CONGRESS_SECRET not in manifest
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
kept = T.BillRow(
117,
"S",
1,
"Telehealth Modernization Act",
None,
"Health",
None,
"",
"telehealth",
"https://www.congress.gov/bill/117th-congress/senate-bill/1",
"",
)
T.write_bills(c, [kept])
assert {r.congress for r in T.read_bills(c)} == {117, 118}
def test_bills_cli_refusal_prints_gap(congress):
c, _, published, _, seen, state = congress
state["status"] = 403
res = runner.invoke(
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: congress:bills: congress: HTTP 403" in res.output
assert CONGRESS_SECRET not in res.output
assert _count(c, "market.bill") == 0
assert len(seen) == 1
assert seen[0].url.host == "api.congress.gov"
assert seen[0].headers["x-api-key"] == CONGRESS_SECRET
assert CONGRESS_SECRET not in str(seen[0].url)
assert published == [True]
def test_bills_cli_from_after_to(congress):
res = runner.invoke(app, ["market", "bills", "--from", "119", "--to", "110"])
assert res.exit_code == 2
assert "--from 119 is after --to 110" in res.output
def test_bills_cli_missing_key_gap_only(congress, monkeypatch):
c, _, published, _, seen, _ = congress
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
res = runner.invoke(
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
)
assert res.exit_code == 0, res.output
assert "gap: congress:bills:" in res.output
assert "CONGRESS_API_KEY" in res.output
assert seen == []
assert [g.source for g in T.read_gaps(c)] == ["congress:bills"]
assert _count(c, "market.bill") == 0
assert published == [True]
def test_bills_cli_limit_stores_nothing_and_keeps_the_gap(congress):
c, _, published, _, seen, _ = congress
T.write_gap(c, "congress:bills", "old")
res = runner.invoke(
app,
[
"market",
"bills",
"--from",
"118",
"--to",
"119",
"--limit",
"1",
"--write",
],
)
assert res.exit_code == 0, res.output
assert "stopped: request limit" in res.output
assert "stored:" not in res.output
assert [g.note for g in T.read_gaps(c)] == ["old"]
assert _count(c, "market.bill") == 0
assert published == []
assert [r.url.path for r in seen] == ["/v3/bill/118"]
def test_committees_cli_refusal_prints_gap(congress):
c, _, published, _, seen, state = congress
state["status"] = 403

View File

@@ -0,0 +1,21 @@
{
"actions": [
{
"actionDate": "2023-01-09",
"text": "Introduced in House",
"actionCode": "1000"
},
{
"actionDate": "2023-01-10",
"text": "Referred to Ways and Means",
"actionCode": "H11100",
"committees": [{"systemCode": "hswm"}]
},
{
"actionDate": "2023-02-01",
"text": "Referred to Health",
"actionCode": "H30000",
"committees": [{"systemCode": "hsif"}, {"systemCode": "hsif14"}]
}
]
}

View File

@@ -0,0 +1,6 @@
{
"cosponsors": [
{"bioguideId": "S000001", "sponsorshipDate": "2023-01-11"},
{"bioguideId": "Y000033", "sponsorshipDate": "2023-01-12"}
]
}

View File

@@ -0,0 +1,15 @@
{
"bill": {
"congress": 118,
"type": "HR",
"number": "1234",
"title": "Medicare Physician Fee Schedule Update Act",
"introducedDate": "2023-01-09",
"policyArea": {"name": "Health"},
"latestAction": {
"actionDate": "2023-02-01",
"text": "Referred to the Committee on Ways and Means."
},
"sponsors": [{"bioguideId": "D000624"}]
}
}

View File

@@ -0,0 +1,21 @@
{
"bills": [
{
"congress": 118,
"type": "HR",
"number": "1234",
"title": "Medicare Physician Fee Schedule Update Act",
"latestAction": {
"actionDate": "2023-02-01",
"text": "Referred to the Committee on Ways and Means."
}
},
{
"congress": 118,
"type": "S",
"number": "9",
"title": "Farm Bill"
}
],
"pagination": {"count": 2}
}

View File

@@ -675,3 +675,311 @@ def test_assignments_replace_one_congress_and_keep_the_other():
}
assert T.write_assignments(c, []) == 0
c.close()
def _bill_detail() -> dict:
return json.loads((FX / "congress_bill_detail.json").read_text())
def _bill_actions() -> dict:
return json.loads((FX / "congress_bill_actions.json").read_text())
def _bill_cosponsors() -> dict:
return json.loads((FX / "congress_bill_cosponsors.json").read_text())
def test_bill_title_filter():
assert (
C.match_title("Medicare Physician Fee Schedule Update Act")
== "physician fee schedule"
)
assert C.match_title("Telehealth Modernization Act") == "telehealth"
assert C.match_title("Farm Bill") is None
def test_bill_kept_only_health_policy_area():
assert C.bill_from_detail([]) is None
assert C.bill_from_detail({}) is None
assert C.bill_from_detail({"bill": "nope"}) is None
detail = _bill_detail()
detail["bill"]["title"] = "Farm Bill"
assert C.bill_from_detail(detail) is None
detail = _bill_detail()
detail["bill"]["policyArea"] = {"name": "Taxation"}
assert C.bill_from_detail(detail) is None
detail = _bill_detail()
detail["bill"]["policyArea"] = "Health"
assert C.bill_from_detail(detail) is None
detail = _bill_detail()
detail["bill"]["latestAction"] = None
detail["bill"]["introducedDate"] = ""
row = C.bill_from_detail(detail)
assert row is not None
assert row.introduced is None
assert row.latest_action_date is None
assert row.latest_action == ""
def test_ordinal():
assert C.ordinal(111) == "111th"
assert C.ordinal(112) == "112th"
assert C.ordinal(121) == "121st"
assert C.ordinal(122) == "122nd"
assert C.ordinal(123) == "123rd"
assert C.ordinal(124) == "124th"
def test_bill_url():
assert (
C.bill_url(118, "HR", 1234)
== "https://www.congress.gov/bill/118th-congress/house-bill/1234"
)
def test_actions_and_cosponsors_rows():
payload = _bill_actions()
payload["actions"].extend(
["nope", {"text": "undated"}, {"actionDate": "not-a-date", "text": "bad"}]
)
actions = C.action_rows(payload, 118, "HR", 1234)
assert C.action_rows({"actions": "nope"}, 118, "HR", 1234) == []
assert C.action_rows([], 118, "HR", 1234) == []
assert [a.action_date for a in actions] == [
date(2023, 1, 9),
date(2023, 1, 10),
date(2023, 2, 1),
]
assert actions[0].text == "Introduced in House"
assert actions[0].action_code == "1000"
assert actions[0].committee_codes == ""
assert actions[1].committee_codes == "hswm"
assert actions[2].committee_codes == "hsif|hsif14"
cosponsors = _bill_cosponsors()
cosponsors["cosponsors"].extend(["nope", {"name": "nobody"}, {"bioguideId": ""}])
members = C.member_rows(_bill_detail(), cosponsors, 118, "HR", 1234)
sponsors = [m for m in members if m.role == "sponsor"]
cosponsors = [m for m in members if m.role == "cosponsor"]
assert len(sponsors) == 1
assert sponsors[0].bioguide_id == "D000624"
assert sponsors[0].since == date(2023, 1, 9)
assert [(m.bioguide_id, m.since) for m in cosponsors] == [
("S000001", date(2023, 1, 11)),
("Y000033", date(2023, 1, 12)),
]
bare = _bill_detail()
bare["bill"]["sponsors"] = "nope"
assert C.member_rows(bare, {"cosponsors": "nope"}, 118, "HR", 1234) == []
assert C.member_rows([], {"cosponsors": []}, 118, "HR", 1234) == []
row = C.bill_from_detail(_bill_detail())
assert row is not None
assert row.congress == 118
assert row.bill_type == "HR"
assert row.number == 1234
assert row.matched_term == "physician fee schedule"
assert row.policy_area == "Health"
assert row.introduced == date(2023, 1, 9)
assert row.latest_action_date == date(2023, 2, 1)
assert row.url == C.bill_url(118, "HR", 1234)
def _route_bills(request: httpx.Request, page: dict | None = None) -> httpx.Response:
path = request.url.path
if path.endswith("/actions"):
return httpx.Response(200, json=_bill_actions())
if path.endswith("/cosponsors"):
return httpx.Response(200, json=_bill_cosponsors())
parts = [part for part in path.split("/") if part]
if "bill" in parts and len(parts) >= 5:
detail = _bill_detail()
if parts[-2:] == ["s", "9"]:
detail["bill"]["type"] = "S"
detail["bill"]["number"] = "9"
detail["bill"]["title"] = "Telehealth Tax Act"
detail["bill"]["policyArea"] = {"name": "Taxation"}
return httpx.Response(200, json=detail)
body = (
page
if page is not None
else json.loads((FX / "congress_bills_page.json").read_text())
)
return httpx.Response(200, json=body)
def test_fetch_bills_pages_and_drops_a_non_health_match(monkeypatch):
monkeypatch.setattr(C, "LIMIT", 1)
pages = {
0: {
"bills": [
{
"type": "HR",
"number": "1234",
"title": "Medicare Physician Fee Schedule Update Act",
}
],
"pagination": {"count": 99},
},
1: {
"bills": [
{
"type": "S",
"number": "9",
"title": "Telehealth Tax Act",
},
{"type": "FOO", "number": "1", "title": "Primary Care Act"},
{"type": "HR", "number": "nope", "title": "Primary Care Act"},
],
"pagination": {"count": 99},
},
2: {
"bills": [{"type": "S", "number": "9", "title": "Telehealth Tax Act"}],
"pagination": {"count": 99},
},
}
def handler(request: httpx.Request) -> httpx.Response:
if request.url.path.rstrip("/").endswith("/bill/118"):
offset = int(request.url.params.get("offset", "0"))
return httpx.Response(200, json=pages[offset])
return _route_bills(request)
bills, actions, members, stopped = C.fetch_bills(
_client(handler), [118], pacer=Pacer()
)
assert stopped is False
assert [b.number for b in bills] == [1234]
assert len(actions) == 3
assert len(members) == 3
def test_fetch_bills_stops_at_the_request_limit_and_ignores_a_non_list():
calls: list[str] = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(request.url.path)
if request.url.path.rstrip("/").endswith("/bill/119"):
return httpx.Response(200, json={"bills": {"nope": True}})
return _route_bills(request)
bills, _, _, stopped = C.fetch_bills(
_client(handler), [118, 119], pacer=Pacer(), limit=1
)
assert stopped is True
assert bills == []
assert calls == ["/v3/bill/118"]
empty, _, _, stopped = C.fetch_bills(
_client(lambda request: httpx.Response(200, json={"bills": {"nope": True}})),
[118],
pacer=Pacer(),
)
assert empty == [] and stopped is False
def test_cached_bill_pages_do_not_spend_the_limit(tmp_path, monkeypatch):
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
seen: list[str] = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request.url.path)
return _route_bills(request)
first, _, _, stopped = C.fetch_bills(
_client(handler), [118], pacer=Pacer(), cache=tmp_path
)
assert len(first) == 1 and stopped is False
assert seen
seen.clear()
second, _, _, stopped = C.fetch_bills(
_client(handler), [118], pacer=Pacer(), cache=tmp_path, limit=1
)
assert seen == []
assert stopped is False
assert [b.number for b in second] == [1234]
def test_bill_budget_stops_before_the_next_congress_list():
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"bills": [{"type": "S", "number": "1", "title": "Farm Bill"}],
"pagination": {"count": 1},
},
)
bills, _, _, stopped = C.fetch_bills(
_client(handler), [118, 119], pacer=Pacer(), limit=1
)
assert bills == [] and stopped is True
def test_bill_budget_stops_on_actions_and_on_cosponsors(tmp_path, monkeypatch):
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
def handler(request: httpx.Request) -> httpx.Response:
return _route_bills(request)
C.fetch_bills(_client(handler), [118], pacer=Pacer(), cache=tmp_path)
root = tmp_path / "bills" / "118" / "HR" / "1234"
for path in (root / "actions").glob("*.json"):
path.unlink()
for path in (root / "cosponsors").glob("*.json"):
path.unlink()
_, _, _, stopped = C.fetch_bills(
_client(handler), [118], pacer=Pacer(), cache=tmp_path, limit=1
)
assert stopped is True
for path in (root / "detail.json", *root.joinpath("actions").glob("*.json")):
path.unlink()
_, _, _, stopped = C.fetch_bills(
_client(handler), [118], pacer=Pacer(), cache=tmp_path, limit=1
)
assert stopped is True
def test_bill_fetch_refuses_a_non_json_page():
with pytest.raises(http.Refused, match="not JSON"):
C.fetch_bills(
_client(lambda request: httpx.Response(200, content=b"nope")),
[118],
pacer=Pacer(),
)
def test_bills_replace_one_congress_and_keep_the_other():
import duckdb
c = duckdb.connect(":memory:")
older = T.BillRow(
117,
"S",
1,
"Telehealth Modernization Act",
None,
"Health",
None,
"",
"telehealth",
"https://www.congress.gov/bill/117th-congress/senate-bill/1",
"",
)
newer = T.BillRow(
118,
"HR",
1234,
"Medicare Physician Fee Schedule Update Act",
date(2023, 1, 9),
"Health",
date(2023, 2, 1),
"Referred to the Committee on Ways and Means.",
"physician fee schedule",
"https://www.congress.gov/bill/118th-congress/house-bill/1234",
"ITEM",
)
T.write_bills(c, [older, newer])
T.write_bills(c, [newer])
assert {r.congress for r in T.read_bills(c)} == {117, 118}
assert T.write_bills(c, []) == 0
assert T.write_bill_actions(c, []) == 0
assert T.write_bill_members(c, []) == 0
c.close()

View File

@@ -48,6 +48,9 @@ def test_ensure_tables_idempotent():
"fec_contribution",
"congress_member",
"committee_assignment",
"bill",
"bill_action",
"bill_member",
}
@@ -200,6 +203,29 @@ MEMBER = T.MemberRow(
ASSIGNMENT = T.AssignmentRow(
119, date(2025, 4, 1), "HSWM02", "E000001", 1, "Chair", "majority", "abc123"
)
BILL = T.BillRow(
118,
"HR",
1234,
"Medicare Physician Fee Schedule Update Act",
date(2023, 1, 9),
"Health",
date(2023, 2, 1),
"Referred to the Committee on Ways and Means.",
"physician fee schedule",
"https://www.congress.gov/bill/118th-congress/house-bill/1234",
"ITEM",
)
BILL_ACTION = T.BillActionRow(
118,
"HR",
1234,
date(2023, 1, 9),
"Introduced in House",
"1000",
"",
)
BILL_MEMBER = T.BillMemberRow(118, "HR", 1234, "D000624", "sponsor", date(2023, 1, 9))
SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY")
ROUNDTRIP = {
@@ -238,6 +264,17 @@ ROUNDTRIP = {
lambda c, r: T.write_assignments(c, [r]),
lambda c: T.read_assignments(c),
),
"bill": (BILL, lambda c, r: T.write_bills(c, [r]), lambda c: T.read_bills(c)),
"bill_action": (
BILL_ACTION,
lambda c, r: T.write_bill_actions(c, [r]),
lambda c: T.read_bill_actions(c),
),
"bill_member": (
BILL_MEMBER,
lambda c, r: T.write_bill_members(c, [r]),
lambda c: T.read_bill_members(c),
),
"entity": (
ENTITY,
lambda c, r: T.write_entities(c, [r]),
@@ -301,6 +338,9 @@ def test_write_empty_is_noop(con):
assert T.write_fec_contributions(con, []) == 0
assert T.write_members(con, []) == 0
assert T.write_assignments(con, []) == 0
assert T.write_bills(con, []) == 0
assert T.write_bill_actions(con, []) == 0
assert T.write_bill_members(con, []) == 0
def test_lda_readers_filter_by_entity(con):