diff --git a/src/cli/market.py b/src/cli/market.py index 52680cc..c8210b4 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -56,6 +56,9 @@ from market.tables import ( retire_organizations, state_codes, write_assignments, + write_bill_actions, + write_bill_members, + write_bills, write_entities, write_fec_committees, write_fec_contributions, @@ -81,6 +84,7 @@ LDA_GAP = "lda:lda" FEC_GAP = "fec:fec" MEMBERS_GAP = "congress:members" COMMITTEES_GAP = "legislators:committees" +BILLS_GAP = "congress:bills" # ── indirections the tests monkeypatch ─────────────────────────────── @@ -1583,6 +1587,74 @@ def committees( typer.echo(f"stored: {n} assignments") +@app.command() +def bills( + from_congress: int = typer.Option(congress_api.CONGRESSES.start, "--from"), + to_congress: int = typer.Option(congress_api.CONGRESSES.stop - 1, "--to"), + limit: int = typer.Option( + 0, "--limit", help="Stop after N uncached requests (0 = no limit)." + ), + write: bool = typer.Option(False, "--write", help="Store market.bill."), +) -> None: + """Medicare physician-payment bills (market.bill) for the 110th through + the 119th. A title match is kept only when the bill's policy area is + Health. A request limit stores nothing: a partial congress would replace + that congress's bills.""" + congresses = _congress_range(from_congress, to_congress) + key = _require_key("congress", BILLS_GAP, write=write) + if key is None: + return + api = http.client("congress", key) + try: + try: + found, actions, sponsors, stopped = congress_api.fetch_bills( + api, + congresses, + pacer=congress_api.member_pacer(), + cache=http.hoard("congress"), + limit=limit, + ) + except http.Refused as exc: + _gap(BILLS_GAP, str(exc), write=write) + return + finally: + api.close() + counts = Counter(row.congress for row in found) + for row in found: + typer.echo(f"{row.congress} {row.bill_type} {row.number} {row.matched_term}") + typer.echo(f"{len(found)} bills") + for n in congresses: + typer.echo(f"congress {n}: {counts[n]}") + if stopped: + typer.echo("stopped: request limit") + return + if not write: + typer.echo("dry run: pass --write") + return + store = _store() + cited = [ + dataclasses.replace(row, item_key=congress_api.cite_bill(store, row)) + for row in found + ] + from cms.ingest_log import log_ingest + + with _batch() as con: + ensure_tables(con) + n = write_bills(con, cited) + write_bill_actions(con, actions) + write_bill_members(con, sponsors) + clear_gap(con, BILLS_GAP) + log_ingest( + con, + module="market.congress", + table_name="market.bill", + rows=n, + source_file=str(http.hoard(http.MANIFEST)), + ) + _publish() + typer.echo(f"stored: {n} bills") + + @app.command() def gaps() -> None: """Sources that could not be read (market.source_gap).""" diff --git a/src/market/congress.py b/src/market/congress.py index c9aa531..73823b0 100644 --- a/src/market/congress.py +++ b/src/market/congress.py @@ -1,13 +1,14 @@ -"""Members of Congress and health-committee assignments. +"""Members of Congress, health-committee assignments, and physician-payment bills. -Congress.gov (``X-Api-Key``, never the query string) lists who served. -``currentMember=false`` keeps members who have since left. The legislators -YAML (public domain, keyless) supplies FEC candidate ids and fills a -congress the API does not return. Committee seats are one -congress-legislators snapshot per congress: the last commit of -``committee-membership-current.yaml`` on or before April 1 of that -congress's first year (M7). ``pagination.next`` is never followed; it can -carry the key. +Congress.gov (``X-Api-Key``, never the query string) lists who served and +which bills were introduced. ``currentMember=false`` keeps members who have +since left. The legislators YAML (public domain, keyless) supplies FEC +candidate ids and fills a congress the API does not return. Committee seats +are one congress-legislators snapshot per congress: the last commit of +``committee-membership-current.yaml`` on or before April 1 of that congress's +first year (M7). A bill is kept when its title matches a physician-payment +term and its detail says the policy area is Health. ``pagination.next`` is +never followed; it can carry the key. """ from __future__ import annotations @@ -15,6 +16,7 @@ from __future__ import annotations import datetime as dt import json import os +import re from collections.abc import Mapping, Sequence from dataclasses import replace from pathlib import Path @@ -24,7 +26,14 @@ import httpx import yaml from market import http -from market.tables import AssignmentRow, MemberRow, district_id +from market.tables import ( + AssignmentRow, + BillActionRow, + BillMemberRow, + BillRow, + MemberRow, + district_id, +) CONGRESS_API = "https://api.congress.gov/v3/" LEGISLATORS = ( @@ -55,6 +64,21 @@ LIMIT = 250 MEMBER_PACE = 0.8 COMMIT_PACE = 61.0 RAW_PACE = 0.2 +BILL_RE = re.compile( + r"(?i)\b(physician fee schedule|physician payment|conversion factor|" + r"sustainable growth rate|relative value|evaluation and management|" + r"telehealth|home-based|house call|care management|primary care)\b" +) +_BILL_SLUG = { + "HR": "house-bill", + "S": "senate-bill", + "HRES": "house-resolution", + "SRES": "senate-resolution", + "HJRES": "house-joint-resolution", + "SJRES": "senate-joint-resolution", + "HCONRES": "house-concurrent-resolution", + "SCONRES": "senate-concurrent-resolution", +} def member_pacer() -> http.Pacer: @@ -69,6 +93,181 @@ def raw_pacer() -> http.Pacer: return http.Pacer(RAW_PACE) +def match_title(title: str) -> str | None: + """The physician-payment term in *title*, lowercased, or ``None``.""" + found = BILL_RE.search(title or "") + if found is None: + return None + return found.group(1).casefold() + + +def ordinal(n: int) -> str: + """``111th``, ``121st``, ``122nd``, ``123rd``. 11–13 stay ``th``.""" + if 10 <= (n % 100) <= 20: + suffix = "th" + else: + suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th") + return f"{n}{suffix}" + + +def bill_url(congress: int, bill_type: str, number: int) -> str: + """The public Congress.gov bill page (no API key).""" + slug = _BILL_SLUG[bill_type.upper()] + return f"https://www.congress.gov/bill/{ordinal(congress)}-congress/{slug}/{number}" + + +def _day(value: Any) -> dt.date | None: + if not value: + return None + try: + return dt.date.fromisoformat(str(value)[:10]) + except ValueError: + return None + + +def bill_from_detail(payload: Mapping[str, Any]) -> BillRow | None: + """One kept bill, or ``None`` when the title misses or the policy area + is not Health.""" + bill = payload.get("bill") if isinstance(payload, Mapping) else None + if not isinstance(bill, dict): + return None + title = str(bill.get("title") or "") + term = match_title(title) + if term is None: + return None + area = bill.get("policyArea") + name = area.get("name") if isinstance(area, dict) else None + if name != "Health": + return None + congress = int(bill["congress"]) + bill_type = str(bill.get("type") or "").upper() + number = int(bill["number"]) + latest = bill.get("latestAction") + if not isinstance(latest, dict): + latest = {} + return BillRow( + congress=congress, + bill_type=bill_type, + number=number, + title=title, + introduced=_day(bill.get("introducedDate")), + policy_area="Health", + latest_action_date=_day(latest.get("actionDate")), + latest_action=str(latest.get("text") or ""), + matched_term=term, + url=bill_url(congress, bill_type, number), + item_key="", + ) + + +def action_rows( + payload: Mapping[str, Any], congress: int, bill_type: str, number: int +) -> list[BillActionRow]: + """Dated actions. A non-object, or an action with no parseable date, is + skipped.""" + raw = payload.get("actions") if isinstance(payload, Mapping) else None + if not isinstance(raw, list): + return [] + rows: list[BillActionRow] = [] + for item in raw: + if not isinstance(item, dict): + continue + day = _day(item.get("actionDate")) + if day is None: + continue + codes: list[str] = [] + committees = item.get("committees") or [] + if isinstance(committees, list): + for committee in committees: + if isinstance(committee, dict) and committee.get("systemCode"): + codes.append(str(committee["systemCode"])) + rows.append( + BillActionRow( + congress, + bill_type, + number, + day, + str(item.get("text") or ""), + str(item.get("actionCode") or ""), + "|".join(codes), + ) + ) + return rows + + +def member_rows( + detail: Mapping[str, Any], + cosponsors: Mapping[str, Any], + congress: int, + bill_type: str, + number: int, +) -> list[BillMemberRow]: + """Sponsors from the detail (``since`` is the introduced date) and + cosponsors from the cosponsors payload.""" + bill = detail.get("bill") if isinstance(detail, Mapping) else None + if not isinstance(bill, dict): + bill = {} + introduced = _day(bill.get("introducedDate")) + rows: list[BillMemberRow] = [] + sponsors = bill.get("sponsors") or [] + if isinstance(sponsors, list): + for sponsor in sponsors: + if isinstance(sponsor, dict) and sponsor.get("bioguideId"): + rows.append( + BillMemberRow( + congress, + bill_type, + number, + str(sponsor["bioguideId"]), + "sponsor", + introduced, + ) + ) + raw = cosponsors.get("cosponsors") if isinstance(cosponsors, Mapping) else None + if not isinstance(raw, list): + return rows + for person in raw: + if not isinstance(person, dict) or not person.get("bioguideId"): + continue + rows.append( + BillMemberRow( + congress, + bill_type, + number, + str(person["bioguideId"]), + "cosponsor", + _day(person.get("sponsorshipDate")), + ) + ) + return rows + + +def cite_bill(store: Any, row: BillRow) -> str: + """A bib ``source`` item for one kept bill. Returns its item key.""" + from bib.item import Source + from bib.tag import Tag + + item = Source( + title=row.title, + url=row.url, + doc_type="bill", + abstract=( + f"{row.bill_type} {row.number} in the {ordinal(row.congress)} " + f"Congress, policy area {row.policy_area}, matched on " + f"{row.matched_term}." + ), + access_date=dt.date.today().isoformat(), + ) + return store.upsert( + item, + tags=[ + Tag.module("market"), + Tag.table("market.bill"), + Tag.source("congress"), + ], + ) + + def congress_of(year: int) -> int: """The Congress in session during *year* (the 119th covers 2025–2026).""" return ((year - 1789) // 2) + 1 @@ -376,6 +575,20 @@ def _yaml_doc(body: bytes, expect: type, status: int) -> Any: return doc +class _BillBudget: + """At most *limit* uncached requests (0 = unlimited).""" + + def __init__(self, limit: int) -> None: + self.limit = limit + self.used = 0 + + def allow(self) -> bool: + if self.limit and self.used >= self.limit: + return False + self.used += 1 + return True + + def _download( cl: httpx.Client, url: str, @@ -384,10 +597,13 @@ def _download( pacer: Any, path: Path | None, source: str, -) -> bytes: + budget: _BillBudget | None = None, +) -> bytes | None: cached = _cached(path) if cached is not None: return cached + if budget is not None and not budget.allow(): + return None response = http.get(cl, url, params=params, pacer=pacer, source=source) _store(path, response.content, str(response.request.url)) return response.content @@ -548,3 +764,216 @@ def fetch_assignments( ) ) return rows + + +def _bill_cache(cache: Path | None, congress: int, *parts: str) -> Path | None: + if cache is None: + return None + return cache.joinpath("bills", str(congress), *parts) + + +def _offset_path(cache: Path | None, congress: int, *prefix: str) -> Any: + def path_for(offset: int) -> Path | None: + return _bill_cache(cache, congress, *prefix, f"{offset}.json") + + return path_for + + +def _first_stamp(kind: str, item: Mapping[str, Any]) -> str: + if kind == "bills": + return f"{item.get('type')}:{item.get('number')}" + if kind == "cosponsors": + return str(item.get("bioguideId") or "") + return f"{item.get('actionDate')}|{item.get('text')}" + + +def _page_items( + api: httpx.Client, + url: str, + kind: str, + *, + pacer: Any, + path_for: Any, + budget: _BillBudget | None, +) -> list[dict[str, Any]] | None: + """Pages of *kind* (``bills``, ``actions``, or ``cosponsors``). ``None`` + when the request budget stops the pull. A repeated first row ends the + page walk; ``pagination.next`` is never followed.""" + items: list[dict[str, Any]] = [] + offset = 0 + seen_first: str | None = None + got = 0 + while True: + body = _download( + api, + url, + {"limit": LIMIT, "offset": offset, "format": "json"}, + pacer=pacer, + path=path_for(offset), + source="congress", + budget=budget, + ) + if body is None: + return None + payload = _object(body, "congress", 200) + raw = payload.get(kind) or [] + if not isinstance(raw, list): + raw = [] + first = "" + if raw and isinstance(raw[0], dict): + first = _first_stamp(kind, raw[0]) + if first and first == seen_first: + break + if first: + seen_first = first + items.extend(row for row in raw if isinstance(row, dict)) + got += len(raw) + page = payload.get("pagination") + count = page.get("count") if isinstance(page, dict) else None + done = not raw or len(raw) < LIMIT + if not done and count is not None: + done = got >= int(count) + if done: + break + offset += LIMIT + return items + + +def _bill_detail( + api: httpx.Client, + congress: int, + bill_type: str, + number: int, + *, + pacer: Any, + cache: Path | None, + budget: _BillBudget | None, +) -> dict[str, Any] | None: + url = f"{CONGRESS_API}bill/{congress}/{bill_type.lower()}/{number}" + body = _download( + api, + url, + {"format": "json"}, + pacer=pacer, + path=_bill_cache(cache, congress, bill_type, str(number), "detail.json"), + source="congress", + budget=budget, + ) + if body is None: + return None + return _object(body, "congress", 200) + + +def fetch_bills( + api: httpx.Client, + congresses: Sequence[int], + *, + pacer: Any, + cache: Path | None = None, + limit: int = 0, +) -> tuple[list[BillRow], list[BillActionRow], list[BillMemberRow], bool]: + """Title matches in *congresses* whose detail policy area is Health, + with their actions, sponsors and cosponsors. The fourth value is true + when *limit* uncached requests were spent before the pull finished. + A stopped pull returns what it already confirmed and nothing past the + request that the budget refused.""" + budget = _BillBudget(limit) + bills: list[BillRow] = [] + actions: list[BillActionRow] = [] + members: list[BillMemberRow] = [] + for congress in congresses: + listed = _page_items( + api, + f"{CONGRESS_API}bill/{congress}", + "bills", + pacer=pacer, + path_for=_offset_path(cache, congress), + budget=budget, + ) + if listed is None: + return bills, actions, members, True + stopped = _keep_matches( + api, + congress, + listed, + pacer=pacer, + cache=cache, + budget=budget, + bills=bills, + actions=actions, + members=members, + ) + if stopped: + return bills, actions, members, True + return bills, actions, members, False + + +def _keep_matches( + api: httpx.Client, + congress: int, + listed: Sequence[Mapping[str, Any]], + *, + pacer: Any, + cache: Path | None, + budget: _BillBudget, + bills: list[BillRow], + actions: list[BillActionRow], + members: list[BillMemberRow], +) -> bool: + """Append Health title-matches from one congress. True when the budget + stops the pull before this congress is finished.""" + for item in listed: + if match_title(str(item.get("title") or "")) is None: + continue + bill_type = str(item.get("type") or "").upper() + try: + number = int(item["number"]) + except (KeyError, TypeError, ValueError): + continue + if bill_type not in _BILL_SLUG: + continue + detail = _bill_detail( + api, congress, bill_type, number, pacer=pacer, cache=cache, budget=budget + ) + if detail is None: + return True + row = bill_from_detail(detail) + if row is None: + continue + stem = f"{CONGRESS_API}bill/{congress}/{bill_type.lower()}/{number}" + act = _page_items( + api, + f"{stem}/actions", + "actions", + pacer=pacer, + path_for=_offset_path(cache, congress, bill_type, str(number), "actions"), + budget=budget, + ) + if act is None: + return True + cos = _page_items( + api, + f"{stem}/cosponsors", + "cosponsors", + pacer=pacer, + path_for=_offset_path( + cache, congress, bill_type, str(number), "cosponsors" + ), + budget=budget, + ) + if cos is None: + return True + bills.append(row) + actions.extend( + action_rows({"actions": act}, row.congress, row.bill_type, row.number) + ) + members.extend( + member_rows( + detail, + {"cosponsors": cos}, + row.congress, + row.bill_type, + row.number, + ) + ) + return False diff --git a/src/market/tables.py b/src/market/tables.py index e735a23..5469d30 100644 --- a/src/market/tables.py +++ b/src/market/tables.py @@ -94,6 +94,19 @@ CREATE TABLE IF NOT EXISTS market.congress_member ( CREATE TABLE IF NOT EXISTS market.committee_assignment ( congress INTEGER, snapshot DATE, committee_id VARCHAR, bioguide_id VARCHAR, rank INTEGER, title VARCHAR, party VARCHAR, source_sha VARCHAR +); +CREATE TABLE IF NOT EXISTS market.bill ( + congress INTEGER, bill_type VARCHAR, number INTEGER, title VARCHAR, + introduced DATE, policy_area VARCHAR, latest_action_date DATE, + latest_action VARCHAR, matched_term VARCHAR, url VARCHAR, item_key VARCHAR +); +CREATE TABLE IF NOT EXISTS market.bill_action ( + congress INTEGER, bill_type VARCHAR, number INTEGER, action_date DATE, + text VARCHAR, action_code VARCHAR, committee_codes VARCHAR +); +CREATE TABLE IF NOT EXISTS market.bill_member ( + congress INTEGER, bill_type VARCHAR, number INTEGER, bioguide_id VARCHAR, + role VARCHAR, since DATE ) """ @@ -380,6 +393,51 @@ class AssignmentRow: source_sha: str +@dataclass(frozen=True) +class BillRow: + """One Congress.gov bill kept because its title matched a physician-payment + term and its policy area is Health. ``item_key`` is empty until cited.""" + + congress: int + bill_type: str + number: int + title: str + introduced: date | None + policy_area: str + latest_action_date: date | None + latest_action: str + matched_term: str + url: str + item_key: str = "" + + +@dataclass(frozen=True) +class BillActionRow: + """One dated action. ``committee_codes`` is a ``|``-joined list of + Congress.gov ``systemCode`` values, in source order.""" + + congress: int + bill_type: str + number: int + action_date: date + text: str + action_code: str + committee_codes: str + + +@dataclass(frozen=True) +class BillMemberRow: + """A sponsor (``since`` is the introduced date) or a cosponsor (``since`` + is the sponsorship date).""" + + congress: int + bill_type: str + number: int + bioguide_id: str + role: str + since: date | None + + def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int: """Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is a statement per row, far too slow for filing lists).""" @@ -862,6 +920,65 @@ def read_assignments(con: Any) -> list[AssignmentRow]: ) +def _replace_congresses(con: Any, table: str, rows: Sequence[Any]) -> None: + for chunk in _chunks(sorted({r.congress for r in rows})): + con.execute( + f"DELETE FROM {table} WHERE congress IN ({','.join('?' * len(chunk))})", + list(chunk), + ) + + +def write_bills(con: Any, rows: Sequence[BillRow]) -> int: + """Replace the bills of each congress present in *rows*.""" + ensure_tables(con) + if not rows: + return 0 + _replace_congresses(con, "market.bill", rows) + return _insert(con, "market.bill", rows) + + +def read_bills(con: Any) -> list[BillRow]: + rows = _select(con, BillRow, "market.bill") + return sorted( + (BillRow(*r) for r in rows), + key=lambda r: (r.congress, r.bill_type, r.number), + ) + + +def write_bill_actions(con: Any, rows: Sequence[BillActionRow]) -> int: + """Replace the actions of each congress present in *rows*.""" + ensure_tables(con) + if not rows: + return 0 + _replace_congresses(con, "market.bill_action", rows) + return _insert(con, "market.bill_action", rows) + + +def read_bill_actions(con: Any) -> list[BillActionRow]: + rows = _select(con, BillActionRow, "market.bill_action") + return sorted( + (BillActionRow(*r) for r in rows), + key=lambda r: (r.congress, r.bill_type, r.number, r.action_date, r.text), + ) + + +def write_bill_members(con: Any, rows: Sequence[BillMemberRow]) -> int: + """Replace the sponsors and cosponsors of each congress present in *rows*.""" + ensure_tables(con) + if not rows: + return 0 + _replace_congresses(con, "market.bill_member", rows) + return _insert(con, "market.bill_member", rows) + + +def read_bill_members(con: Any) -> list[BillMemberRow]: + rows = _select(con, BillMemberRow, "market.bill_member") + return sorted( + (BillMemberRow(*r) for r in rows), + key=lambda r: (r.congress, r.bill_type, r.number, r.role, r.bioguide_id), + ) + + # ── districts ──────────────────────────────────────────────────────── diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index a4201f1..393dcbd 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -2210,6 +2210,10 @@ def congress(env, monkeypatch): T.ensure_tables(c) seen: list[httpx.Request] = [] members = json.loads((FX / "congress_members_118.json").read_text()) + bills = json.loads((FX / "congress_bills_page.json").read_text()) + detail = json.loads((FX / "congress_bill_detail.json").read_text()) + actions = json.loads((FX / "congress_bill_actions.json").read_text()) + cosponsors = json.loads((FX / "congress_bill_cosponsors.json").read_text()) people = (FX / "legislators_snippet.yaml").read_bytes() membership = (FX / "membership_snippet.yaml").read_bytes() state = {"status": 200} @@ -2222,6 +2226,16 @@ def congress(env, monkeypatch): return httpx.Response(state["status"], text="no") host = request.url.host if host == "api.congress.gov": + path = request.url.path + if path.endswith("/actions"): + return httpx.Response(200, json=actions) + if path.endswith("/cosponsors"): + return httpx.Response(200, json=cosponsors) + parts = [part for part in path.split("/") if part] + if "bill" in parts and len(parts) >= 5: + return httpx.Response(200, json=detail) + if "bill" in parts: + return httpx.Response(200, json=bills) return httpx.Response(200, json=members) if host == "api.github.com": return httpx.Response(200, json=[{"sha": "deadbeef"}]) @@ -2365,6 +2379,139 @@ def test_committees_cli_write(congress): assert ("market.committee_assignment", 1) in logged +def test_bills_cli_no_write_lists_matches(congress): + c, _, published, _, seen, _ = congress + res = runner.invoke(app, ["market", "bills", "--from", "118", "--to", "118"]) + assert res.exit_code == 0, res.output + assert "118 HR 1234 physician fee schedule" in res.output + assert "1 bills" in res.output + assert "congress 118: 1" in res.output + assert "Farm Bill" not in res.output + assert "dry run: pass --write" in res.output + assert _count(c, "market.bill") == 0 + assert published == [] + hits = [r for r in seen if r.url.host == "api.congress.gov"] + assert hits + assert all(CONGRESS_SECRET not in str(r.url) for r in hits) + assert all("api_key" not in str(r.url) for r in hits) + assert hits[0].headers["x-api-key"] == CONGRESS_SECRET + + +def test_bills_cli_offline_write(congress): + c, store, published, hoard, seen, _ = congress + T.write_gap(c, "congress:bills", "old") + res = runner.invoke( + app, ["market", "bills", "--from", "118", "--to", "118", "--write"] + ) + assert res.exit_code == 0, res.output + assert "stored: 1 bills" in res.output + (bill,) = T.read_bills(c) + assert bill.bill_type == "HR" + assert bill.number == 1234 + assert bill.matched_term == "physician fee schedule" + assert bill.policy_area == "Health" + assert bill.introduced == date(2023, 1, 9) + assert bill.item_key + assert len(T.read_bill_actions(c)) == 3 + roles = sorted((m.role, m.bioguide_id) for m in T.read_bill_members(c)) + assert roles == [ + ("cosponsor", "S000001"), + ("cosponsor", "Y000033"), + ("sponsor", "D000624"), + ] + assert T.read_gaps(c) == [] + assert published == [True] + assert any( + (i.url or "").endswith("/118th-congress/house-bill/1234") + for i in store.list_items() + ) + assert ("market.bill", 1) in c.execute( + "SELECT table_name, rows FROM cms.ingest_log" + ).fetchall() + manifest = (hoard / "MANIFEST.jsonl").read_text() + assert CONGRESS_SECRET not in manifest + assert all(CONGRESS_SECRET not in str(r.url) for r in seen) + kept = T.BillRow( + 117, + "S", + 1, + "Telehealth Modernization Act", + None, + "Health", + None, + "", + "telehealth", + "https://www.congress.gov/bill/117th-congress/senate-bill/1", + "", + ) + T.write_bills(c, [kept]) + assert {r.congress for r in T.read_bills(c)} == {117, 118} + + +def test_bills_cli_refusal_prints_gap(congress): + c, _, published, _, seen, state = congress + state["status"] = 403 + res = runner.invoke( + app, ["market", "bills", "--from", "118", "--to", "118", "--write"] + ) + assert res.exit_code == 0, res.output + assert "gap: congress:bills: congress: HTTP 403" in res.output + assert CONGRESS_SECRET not in res.output + assert _count(c, "market.bill") == 0 + assert len(seen) == 1 + assert seen[0].url.host == "api.congress.gov" + assert seen[0].headers["x-api-key"] == CONGRESS_SECRET + assert CONGRESS_SECRET not in str(seen[0].url) + assert published == [True] + + +def test_bills_cli_from_after_to(congress): + res = runner.invoke(app, ["market", "bills", "--from", "119", "--to", "110"]) + assert res.exit_code == 2 + assert "--from 119 is after --to 110" in res.output + + +def test_bills_cli_missing_key_gap_only(congress, monkeypatch): + c, _, published, _, seen, _ = congress + monkeypatch.setattr(http, "api_key", lambda source, env=None: None) + res = runner.invoke( + app, ["market", "bills", "--from", "118", "--to", "118", "--write"] + ) + assert res.exit_code == 0, res.output + assert "gap: congress:bills:" in res.output + assert "CONGRESS_API_KEY" in res.output + assert seen == [] + assert [g.source for g in T.read_gaps(c)] == ["congress:bills"] + assert _count(c, "market.bill") == 0 + assert published == [True] + + +def test_bills_cli_limit_stores_nothing_and_keeps_the_gap(congress): + c, _, published, _, seen, _ = congress + T.write_gap(c, "congress:bills", "old") + res = runner.invoke( + app, + [ + "market", + "bills", + "--from", + "118", + "--to", + "119", + "--limit", + "1", + "--write", + ], + ) + assert res.exit_code == 0, res.output + assert "stopped: request limit" in res.output + assert "stored:" not in res.output + assert [g.note for g in T.read_gaps(c)] == ["old"] + assert _count(c, "market.bill") == 0 + assert published == [] + assert [r.url.path for r in seen] == ["/v3/bill/118"] + + def test_committees_cli_refusal_prints_gap(congress): c, _, published, _, seen, state = congress state["status"] = 403 diff --git a/tests/market/fixtures/congress_bill_actions.json b/tests/market/fixtures/congress_bill_actions.json new file mode 100644 index 0000000..3772414 --- /dev/null +++ b/tests/market/fixtures/congress_bill_actions.json @@ -0,0 +1,21 @@ +{ + "actions": [ + { + "actionDate": "2023-01-09", + "text": "Introduced in House", + "actionCode": "1000" + }, + { + "actionDate": "2023-01-10", + "text": "Referred to Ways and Means", + "actionCode": "H11100", + "committees": [{"systemCode": "hswm"}] + }, + { + "actionDate": "2023-02-01", + "text": "Referred to Health", + "actionCode": "H30000", + "committees": [{"systemCode": "hsif"}, {"systemCode": "hsif14"}] + } + ] +} diff --git a/tests/market/fixtures/congress_bill_cosponsors.json b/tests/market/fixtures/congress_bill_cosponsors.json new file mode 100644 index 0000000..a8231d9 --- /dev/null +++ b/tests/market/fixtures/congress_bill_cosponsors.json @@ -0,0 +1,6 @@ +{ + "cosponsors": [ + {"bioguideId": "S000001", "sponsorshipDate": "2023-01-11"}, + {"bioguideId": "Y000033", "sponsorshipDate": "2023-01-12"} + ] +} diff --git a/tests/market/fixtures/congress_bill_detail.json b/tests/market/fixtures/congress_bill_detail.json new file mode 100644 index 0000000..cda1772 --- /dev/null +++ b/tests/market/fixtures/congress_bill_detail.json @@ -0,0 +1,15 @@ +{ + "bill": { + "congress": 118, + "type": "HR", + "number": "1234", + "title": "Medicare Physician Fee Schedule Update Act", + "introducedDate": "2023-01-09", + "policyArea": {"name": "Health"}, + "latestAction": { + "actionDate": "2023-02-01", + "text": "Referred to the Committee on Ways and Means." + }, + "sponsors": [{"bioguideId": "D000624"}] + } +} diff --git a/tests/market/fixtures/congress_bills_page.json b/tests/market/fixtures/congress_bills_page.json new file mode 100644 index 0000000..a2352b8 --- /dev/null +++ b/tests/market/fixtures/congress_bills_page.json @@ -0,0 +1,21 @@ +{ + "bills": [ + { + "congress": 118, + "type": "HR", + "number": "1234", + "title": "Medicare Physician Fee Schedule Update Act", + "latestAction": { + "actionDate": "2023-02-01", + "text": "Referred to the Committee on Ways and Means." + } + }, + { + "congress": 118, + "type": "S", + "number": "9", + "title": "Farm Bill" + } + ], + "pagination": {"count": 2} +} diff --git a/tests/market/test_congress.py b/tests/market/test_congress.py index 7d39958..12703fa 100644 --- a/tests/market/test_congress.py +++ b/tests/market/test_congress.py @@ -675,3 +675,311 @@ def test_assignments_replace_one_congress_and_keep_the_other(): } assert T.write_assignments(c, []) == 0 c.close() + + +def _bill_detail() -> dict: + return json.loads((FX / "congress_bill_detail.json").read_text()) + + +def _bill_actions() -> dict: + return json.loads((FX / "congress_bill_actions.json").read_text()) + + +def _bill_cosponsors() -> dict: + return json.loads((FX / "congress_bill_cosponsors.json").read_text()) + + +def test_bill_title_filter(): + assert ( + C.match_title("Medicare Physician Fee Schedule Update Act") + == "physician fee schedule" + ) + assert C.match_title("Telehealth Modernization Act") == "telehealth" + assert C.match_title("Farm Bill") is None + + +def test_bill_kept_only_health_policy_area(): + assert C.bill_from_detail([]) is None + assert C.bill_from_detail({}) is None + assert C.bill_from_detail({"bill": "nope"}) is None + detail = _bill_detail() + detail["bill"]["title"] = "Farm Bill" + assert C.bill_from_detail(detail) is None + detail = _bill_detail() + detail["bill"]["policyArea"] = {"name": "Taxation"} + assert C.bill_from_detail(detail) is None + detail = _bill_detail() + detail["bill"]["policyArea"] = "Health" + assert C.bill_from_detail(detail) is None + detail = _bill_detail() + detail["bill"]["latestAction"] = None + detail["bill"]["introducedDate"] = "" + row = C.bill_from_detail(detail) + assert row is not None + assert row.introduced is None + assert row.latest_action_date is None + assert row.latest_action == "" + + +def test_ordinal(): + assert C.ordinal(111) == "111th" + assert C.ordinal(112) == "112th" + assert C.ordinal(121) == "121st" + assert C.ordinal(122) == "122nd" + assert C.ordinal(123) == "123rd" + assert C.ordinal(124) == "124th" + + +def test_bill_url(): + assert ( + C.bill_url(118, "HR", 1234) + == "https://www.congress.gov/bill/118th-congress/house-bill/1234" + ) + + +def test_actions_and_cosponsors_rows(): + payload = _bill_actions() + payload["actions"].extend( + ["nope", {"text": "undated"}, {"actionDate": "not-a-date", "text": "bad"}] + ) + actions = C.action_rows(payload, 118, "HR", 1234) + assert C.action_rows({"actions": "nope"}, 118, "HR", 1234) == [] + assert C.action_rows([], 118, "HR", 1234) == [] + assert [a.action_date for a in actions] == [ + date(2023, 1, 9), + date(2023, 1, 10), + date(2023, 2, 1), + ] + assert actions[0].text == "Introduced in House" + assert actions[0].action_code == "1000" + assert actions[0].committee_codes == "" + assert actions[1].committee_codes == "hswm" + assert actions[2].committee_codes == "hsif|hsif14" + cosponsors = _bill_cosponsors() + cosponsors["cosponsors"].extend(["nope", {"name": "nobody"}, {"bioguideId": ""}]) + members = C.member_rows(_bill_detail(), cosponsors, 118, "HR", 1234) + sponsors = [m for m in members if m.role == "sponsor"] + cosponsors = [m for m in members if m.role == "cosponsor"] + assert len(sponsors) == 1 + assert sponsors[0].bioguide_id == "D000624" + assert sponsors[0].since == date(2023, 1, 9) + assert [(m.bioguide_id, m.since) for m in cosponsors] == [ + ("S000001", date(2023, 1, 11)), + ("Y000033", date(2023, 1, 12)), + ] + bare = _bill_detail() + bare["bill"]["sponsors"] = "nope" + assert C.member_rows(bare, {"cosponsors": "nope"}, 118, "HR", 1234) == [] + assert C.member_rows([], {"cosponsors": []}, 118, "HR", 1234) == [] + row = C.bill_from_detail(_bill_detail()) + assert row is not None + assert row.congress == 118 + assert row.bill_type == "HR" + assert row.number == 1234 + assert row.matched_term == "physician fee schedule" + assert row.policy_area == "Health" + assert row.introduced == date(2023, 1, 9) + assert row.latest_action_date == date(2023, 2, 1) + assert row.url == C.bill_url(118, "HR", 1234) + + +def _route_bills(request: httpx.Request, page: dict | None = None) -> httpx.Response: + path = request.url.path + if path.endswith("/actions"): + return httpx.Response(200, json=_bill_actions()) + if path.endswith("/cosponsors"): + return httpx.Response(200, json=_bill_cosponsors()) + parts = [part for part in path.split("/") if part] + if "bill" in parts and len(parts) >= 5: + detail = _bill_detail() + if parts[-2:] == ["s", "9"]: + detail["bill"]["type"] = "S" + detail["bill"]["number"] = "9" + detail["bill"]["title"] = "Telehealth Tax Act" + detail["bill"]["policyArea"] = {"name": "Taxation"} + return httpx.Response(200, json=detail) + body = ( + page + if page is not None + else json.loads((FX / "congress_bills_page.json").read_text()) + ) + return httpx.Response(200, json=body) + + +def test_fetch_bills_pages_and_drops_a_non_health_match(monkeypatch): + monkeypatch.setattr(C, "LIMIT", 1) + pages = { + 0: { + "bills": [ + { + "type": "HR", + "number": "1234", + "title": "Medicare Physician Fee Schedule Update Act", + } + ], + "pagination": {"count": 99}, + }, + 1: { + "bills": [ + { + "type": "S", + "number": "9", + "title": "Telehealth Tax Act", + }, + {"type": "FOO", "number": "1", "title": "Primary Care Act"}, + {"type": "HR", "number": "nope", "title": "Primary Care Act"}, + ], + "pagination": {"count": 99}, + }, + 2: { + "bills": [{"type": "S", "number": "9", "title": "Telehealth Tax Act"}], + "pagination": {"count": 99}, + }, + } + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.rstrip("/").endswith("/bill/118"): + offset = int(request.url.params.get("offset", "0")) + return httpx.Response(200, json=pages[offset]) + return _route_bills(request) + + bills, actions, members, stopped = C.fetch_bills( + _client(handler), [118], pacer=Pacer() + ) + assert stopped is False + assert [b.number for b in bills] == [1234] + assert len(actions) == 3 + assert len(members) == 3 + + +def test_fetch_bills_stops_at_the_request_limit_and_ignores_a_non_list(): + calls: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(request.url.path) + if request.url.path.rstrip("/").endswith("/bill/119"): + return httpx.Response(200, json={"bills": {"nope": True}}) + return _route_bills(request) + + bills, _, _, stopped = C.fetch_bills( + _client(handler), [118, 119], pacer=Pacer(), limit=1 + ) + assert stopped is True + assert bills == [] + assert calls == ["/v3/bill/118"] + empty, _, _, stopped = C.fetch_bills( + _client(lambda request: httpx.Response(200, json={"bills": {"nope": True}})), + [118], + pacer=Pacer(), + ) + assert empty == [] and stopped is False + + +def test_cached_bill_pages_do_not_spend_the_limit(tmp_path, monkeypatch): + monkeypatch.setattr(http, "hoard_root", lambda: tmp_path) + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.url.path) + return _route_bills(request) + + first, _, _, stopped = C.fetch_bills( + _client(handler), [118], pacer=Pacer(), cache=tmp_path + ) + assert len(first) == 1 and stopped is False + assert seen + seen.clear() + second, _, _, stopped = C.fetch_bills( + _client(handler), [118], pacer=Pacer(), cache=tmp_path, limit=1 + ) + assert seen == [] + assert stopped is False + assert [b.number for b in second] == [1234] + + +def test_bill_budget_stops_before_the_next_congress_list(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "bills": [{"type": "S", "number": "1", "title": "Farm Bill"}], + "pagination": {"count": 1}, + }, + ) + + bills, _, _, stopped = C.fetch_bills( + _client(handler), [118, 119], pacer=Pacer(), limit=1 + ) + assert bills == [] and stopped is True + + +def test_bill_budget_stops_on_actions_and_on_cosponsors(tmp_path, monkeypatch): + monkeypatch.setattr(http, "hoard_root", lambda: tmp_path) + + def handler(request: httpx.Request) -> httpx.Response: + return _route_bills(request) + + C.fetch_bills(_client(handler), [118], pacer=Pacer(), cache=tmp_path) + root = tmp_path / "bills" / "118" / "HR" / "1234" + for path in (root / "actions").glob("*.json"): + path.unlink() + for path in (root / "cosponsors").glob("*.json"): + path.unlink() + _, _, _, stopped = C.fetch_bills( + _client(handler), [118], pacer=Pacer(), cache=tmp_path, limit=1 + ) + assert stopped is True + for path in (root / "detail.json", *root.joinpath("actions").glob("*.json")): + path.unlink() + _, _, _, stopped = C.fetch_bills( + _client(handler), [118], pacer=Pacer(), cache=tmp_path, limit=1 + ) + assert stopped is True + + +def test_bill_fetch_refuses_a_non_json_page(): + with pytest.raises(http.Refused, match="not JSON"): + C.fetch_bills( + _client(lambda request: httpx.Response(200, content=b"nope")), + [118], + pacer=Pacer(), + ) + + +def test_bills_replace_one_congress_and_keep_the_other(): + import duckdb + + c = duckdb.connect(":memory:") + older = T.BillRow( + 117, + "S", + 1, + "Telehealth Modernization Act", + None, + "Health", + None, + "", + "telehealth", + "https://www.congress.gov/bill/117th-congress/senate-bill/1", + "", + ) + newer = T.BillRow( + 118, + "HR", + 1234, + "Medicare Physician Fee Schedule Update Act", + date(2023, 1, 9), + "Health", + date(2023, 2, 1), + "Referred to the Committee on Ways and Means.", + "physician fee schedule", + "https://www.congress.gov/bill/118th-congress/house-bill/1234", + "ITEM", + ) + T.write_bills(c, [older, newer]) + T.write_bills(c, [newer]) + assert {r.congress for r in T.read_bills(c)} == {117, 118} + assert T.write_bills(c, []) == 0 + assert T.write_bill_actions(c, []) == 0 + assert T.write_bill_members(c, []) == 0 + c.close() diff --git a/tests/market/test_tables.py b/tests/market/test_tables.py index a6c219b..93331bb 100644 --- a/tests/market/test_tables.py +++ b/tests/market/test_tables.py @@ -48,6 +48,9 @@ def test_ensure_tables_idempotent(): "fec_contribution", "congress_member", "committee_assignment", + "bill", + "bill_action", + "bill_member", } @@ -200,6 +203,29 @@ MEMBER = T.MemberRow( ASSIGNMENT = T.AssignmentRow( 119, date(2025, 4, 1), "HSWM02", "E000001", 1, "Chair", "majority", "abc123" ) +BILL = T.BillRow( + 118, + "HR", + 1234, + "Medicare Physician Fee Schedule Update Act", + date(2023, 1, 9), + "Health", + date(2023, 2, 1), + "Referred to the Committee on Ways and Means.", + "physician fee schedule", + "https://www.congress.gov/bill/118th-congress/house-bill/1234", + "ITEM", +) +BILL_ACTION = T.BillActionRow( + 118, + "HR", + 1234, + date(2023, 1, 9), + "Introduced in House", + "1000", + "", +) +BILL_MEMBER = T.BillMemberRow(118, "HR", 1234, "D000624", "sponsor", date(2023, 1, 9)) SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY") ROUNDTRIP = { @@ -238,6 +264,17 @@ ROUNDTRIP = { lambda c, r: T.write_assignments(c, [r]), lambda c: T.read_assignments(c), ), + "bill": (BILL, lambda c, r: T.write_bills(c, [r]), lambda c: T.read_bills(c)), + "bill_action": ( + BILL_ACTION, + lambda c, r: T.write_bill_actions(c, [r]), + lambda c: T.read_bill_actions(c), + ), + "bill_member": ( + BILL_MEMBER, + lambda c, r: T.write_bill_members(c, [r]), + lambda c: T.read_bill_members(c), + ), "entity": ( ENTITY, lambda c, r: T.write_entities(c, [r]), @@ -301,6 +338,9 @@ def test_write_empty_is_noop(con): assert T.write_fec_contributions(con, []) == 0 assert T.write_members(con, []) == 0 assert T.write_assignments(con, []) == 0 + assert T.write_bills(con, []) == 0 + assert T.write_bill_actions(con, []) == 0 + assert T.write_bill_members(con, []) == 0 def test_lda_readers_filter_by_entity(con):