feat(market): Medicare physician-payment bills with actions, sponsors and cosponsors (Congress.gov)
This commit is contained in:
@@ -56,6 +56,9 @@ from market.tables import (
|
||||
retire_organizations,
|
||||
state_codes,
|
||||
write_assignments,
|
||||
write_bill_actions,
|
||||
write_bill_members,
|
||||
write_bills,
|
||||
write_entities,
|
||||
write_fec_committees,
|
||||
write_fec_contributions,
|
||||
@@ -81,6 +84,7 @@ LDA_GAP = "lda:lda"
|
||||
FEC_GAP = "fec:fec"
|
||||
MEMBERS_GAP = "congress:members"
|
||||
COMMITTEES_GAP = "legislators:committees"
|
||||
BILLS_GAP = "congress:bills"
|
||||
|
||||
|
||||
# ── indirections the tests monkeypatch ───────────────────────────────
|
||||
@@ -1583,6 +1587,74 @@ def committees(
|
||||
typer.echo(f"stored: {n} assignments")
|
||||
|
||||
|
||||
@app.command()
|
||||
def bills(
|
||||
from_congress: int = typer.Option(congress_api.CONGRESSES.start, "--from"),
|
||||
to_congress: int = typer.Option(congress_api.CONGRESSES.stop - 1, "--to"),
|
||||
limit: int = typer.Option(
|
||||
0, "--limit", help="Stop after N uncached requests (0 = no limit)."
|
||||
),
|
||||
write: bool = typer.Option(False, "--write", help="Store market.bill."),
|
||||
) -> None:
|
||||
"""Medicare physician-payment bills (market.bill) for the 110th through
|
||||
the 119th. A title match is kept only when the bill's policy area is
|
||||
Health. A request limit stores nothing: a partial congress would replace
|
||||
that congress's bills."""
|
||||
congresses = _congress_range(from_congress, to_congress)
|
||||
key = _require_key("congress", BILLS_GAP, write=write)
|
||||
if key is None:
|
||||
return
|
||||
api = http.client("congress", key)
|
||||
try:
|
||||
try:
|
||||
found, actions, sponsors, stopped = congress_api.fetch_bills(
|
||||
api,
|
||||
congresses,
|
||||
pacer=congress_api.member_pacer(),
|
||||
cache=http.hoard("congress"),
|
||||
limit=limit,
|
||||
)
|
||||
except http.Refused as exc:
|
||||
_gap(BILLS_GAP, str(exc), write=write)
|
||||
return
|
||||
finally:
|
||||
api.close()
|
||||
counts = Counter(row.congress for row in found)
|
||||
for row in found:
|
||||
typer.echo(f"{row.congress} {row.bill_type} {row.number} {row.matched_term}")
|
||||
typer.echo(f"{len(found)} bills")
|
||||
for n in congresses:
|
||||
typer.echo(f"congress {n}: {counts[n]}")
|
||||
if stopped:
|
||||
typer.echo("stopped: request limit")
|
||||
return
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write")
|
||||
return
|
||||
store = _store()
|
||||
cited = [
|
||||
dataclasses.replace(row, item_key=congress_api.cite_bill(store, row))
|
||||
for row in found
|
||||
]
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
n = write_bills(con, cited)
|
||||
write_bill_actions(con, actions)
|
||||
write_bill_members(con, sponsors)
|
||||
clear_gap(con, BILLS_GAP)
|
||||
log_ingest(
|
||||
con,
|
||||
module="market.congress",
|
||||
table_name="market.bill",
|
||||
rows=n,
|
||||
source_file=str(http.hoard(http.MANIFEST)),
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: {n} bills")
|
||||
|
||||
|
||||
@app.command()
|
||||
def gaps() -> None:
|
||||
"""Sources that could not be read (market.source_gap)."""
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
"""Members of Congress and health-committee assignments.
|
||||
"""Members of Congress, health-committee assignments, and physician-payment bills.
|
||||
|
||||
Congress.gov (``X-Api-Key``, never the query string) lists who served.
|
||||
``currentMember=false`` keeps members who have since left. The legislators
|
||||
YAML (public domain, keyless) supplies FEC candidate ids and fills a
|
||||
congress the API does not return. Committee seats are one
|
||||
congress-legislators snapshot per congress: the last commit of
|
||||
``committee-membership-current.yaml`` on or before April 1 of that
|
||||
congress's first year (M7). ``pagination.next`` is never followed; it can
|
||||
carry the key.
|
||||
Congress.gov (``X-Api-Key``, never the query string) lists who served and
|
||||
which bills were introduced. ``currentMember=false`` keeps members who have
|
||||
since left. The legislators YAML (public domain, keyless) supplies FEC
|
||||
candidate ids and fills a congress the API does not return. Committee seats
|
||||
are one congress-legislators snapshot per congress: the last commit of
|
||||
``committee-membership-current.yaml`` on or before April 1 of that congress's
|
||||
first year (M7). A bill is kept when its title matches a physician-payment
|
||||
term and its detail says the policy area is Health. ``pagination.next`` is
|
||||
never followed; it can carry the key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -15,6 +16,7 @@ from __future__ import annotations
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
@@ -24,7 +26,14 @@ import httpx
|
||||
import yaml
|
||||
|
||||
from market import http
|
||||
from market.tables import AssignmentRow, MemberRow, district_id
|
||||
from market.tables import (
|
||||
AssignmentRow,
|
||||
BillActionRow,
|
||||
BillMemberRow,
|
||||
BillRow,
|
||||
MemberRow,
|
||||
district_id,
|
||||
)
|
||||
|
||||
CONGRESS_API = "https://api.congress.gov/v3/"
|
||||
LEGISLATORS = (
|
||||
@@ -55,6 +64,21 @@ LIMIT = 250
|
||||
MEMBER_PACE = 0.8
|
||||
COMMIT_PACE = 61.0
|
||||
RAW_PACE = 0.2
|
||||
BILL_RE = re.compile(
|
||||
r"(?i)\b(physician fee schedule|physician payment|conversion factor|"
|
||||
r"sustainable growth rate|relative value|evaluation and management|"
|
||||
r"telehealth|home-based|house call|care management|primary care)\b"
|
||||
)
|
||||
_BILL_SLUG = {
|
||||
"HR": "house-bill",
|
||||
"S": "senate-bill",
|
||||
"HRES": "house-resolution",
|
||||
"SRES": "senate-resolution",
|
||||
"HJRES": "house-joint-resolution",
|
||||
"SJRES": "senate-joint-resolution",
|
||||
"HCONRES": "house-concurrent-resolution",
|
||||
"SCONRES": "senate-concurrent-resolution",
|
||||
}
|
||||
|
||||
|
||||
def member_pacer() -> http.Pacer:
|
||||
@@ -69,6 +93,181 @@ def raw_pacer() -> http.Pacer:
|
||||
return http.Pacer(RAW_PACE)
|
||||
|
||||
|
||||
def match_title(title: str) -> str | None:
|
||||
"""The physician-payment term in *title*, lowercased, or ``None``."""
|
||||
found = BILL_RE.search(title or "")
|
||||
if found is None:
|
||||
return None
|
||||
return found.group(1).casefold()
|
||||
|
||||
|
||||
def ordinal(n: int) -> str:
|
||||
"""``111th``, ``121st``, ``122nd``, ``123rd``. 11–13 stay ``th``."""
|
||||
if 10 <= (n % 100) <= 20:
|
||||
suffix = "th"
|
||||
else:
|
||||
suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th")
|
||||
return f"{n}{suffix}"
|
||||
|
||||
|
||||
def bill_url(congress: int, bill_type: str, number: int) -> str:
|
||||
"""The public Congress.gov bill page (no API key)."""
|
||||
slug = _BILL_SLUG[bill_type.upper()]
|
||||
return f"https://www.congress.gov/bill/{ordinal(congress)}-congress/{slug}/{number}"
|
||||
|
||||
|
||||
def _day(value: Any) -> dt.date | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return dt.date.fromisoformat(str(value)[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def bill_from_detail(payload: Mapping[str, Any]) -> BillRow | None:
|
||||
"""One kept bill, or ``None`` when the title misses or the policy area
|
||||
is not Health."""
|
||||
bill = payload.get("bill") if isinstance(payload, Mapping) else None
|
||||
if not isinstance(bill, dict):
|
||||
return None
|
||||
title = str(bill.get("title") or "")
|
||||
term = match_title(title)
|
||||
if term is None:
|
||||
return None
|
||||
area = bill.get("policyArea")
|
||||
name = area.get("name") if isinstance(area, dict) else None
|
||||
if name != "Health":
|
||||
return None
|
||||
congress = int(bill["congress"])
|
||||
bill_type = str(bill.get("type") or "").upper()
|
||||
number = int(bill["number"])
|
||||
latest = bill.get("latestAction")
|
||||
if not isinstance(latest, dict):
|
||||
latest = {}
|
||||
return BillRow(
|
||||
congress=congress,
|
||||
bill_type=bill_type,
|
||||
number=number,
|
||||
title=title,
|
||||
introduced=_day(bill.get("introducedDate")),
|
||||
policy_area="Health",
|
||||
latest_action_date=_day(latest.get("actionDate")),
|
||||
latest_action=str(latest.get("text") or ""),
|
||||
matched_term=term,
|
||||
url=bill_url(congress, bill_type, number),
|
||||
item_key="",
|
||||
)
|
||||
|
||||
|
||||
def action_rows(
|
||||
payload: Mapping[str, Any], congress: int, bill_type: str, number: int
|
||||
) -> list[BillActionRow]:
|
||||
"""Dated actions. A non-object, or an action with no parseable date, is
|
||||
skipped."""
|
||||
raw = payload.get("actions") if isinstance(payload, Mapping) else None
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
rows: list[BillActionRow] = []
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
day = _day(item.get("actionDate"))
|
||||
if day is None:
|
||||
continue
|
||||
codes: list[str] = []
|
||||
committees = item.get("committees") or []
|
||||
if isinstance(committees, list):
|
||||
for committee in committees:
|
||||
if isinstance(committee, dict) and committee.get("systemCode"):
|
||||
codes.append(str(committee["systemCode"]))
|
||||
rows.append(
|
||||
BillActionRow(
|
||||
congress,
|
||||
bill_type,
|
||||
number,
|
||||
day,
|
||||
str(item.get("text") or ""),
|
||||
str(item.get("actionCode") or ""),
|
||||
"|".join(codes),
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def member_rows(
|
||||
detail: Mapping[str, Any],
|
||||
cosponsors: Mapping[str, Any],
|
||||
congress: int,
|
||||
bill_type: str,
|
||||
number: int,
|
||||
) -> list[BillMemberRow]:
|
||||
"""Sponsors from the detail (``since`` is the introduced date) and
|
||||
cosponsors from the cosponsors payload."""
|
||||
bill = detail.get("bill") if isinstance(detail, Mapping) else None
|
||||
if not isinstance(bill, dict):
|
||||
bill = {}
|
||||
introduced = _day(bill.get("introducedDate"))
|
||||
rows: list[BillMemberRow] = []
|
||||
sponsors = bill.get("sponsors") or []
|
||||
if isinstance(sponsors, list):
|
||||
for sponsor in sponsors:
|
||||
if isinstance(sponsor, dict) and sponsor.get("bioguideId"):
|
||||
rows.append(
|
||||
BillMemberRow(
|
||||
congress,
|
||||
bill_type,
|
||||
number,
|
||||
str(sponsor["bioguideId"]),
|
||||
"sponsor",
|
||||
introduced,
|
||||
)
|
||||
)
|
||||
raw = cosponsors.get("cosponsors") if isinstance(cosponsors, Mapping) else None
|
||||
if not isinstance(raw, list):
|
||||
return rows
|
||||
for person in raw:
|
||||
if not isinstance(person, dict) or not person.get("bioguideId"):
|
||||
continue
|
||||
rows.append(
|
||||
BillMemberRow(
|
||||
congress,
|
||||
bill_type,
|
||||
number,
|
||||
str(person["bioguideId"]),
|
||||
"cosponsor",
|
||||
_day(person.get("sponsorshipDate")),
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def cite_bill(store: Any, row: BillRow) -> str:
|
||||
"""A bib ``source`` item for one kept bill. Returns its item key."""
|
||||
from bib.item import Source
|
||||
from bib.tag import Tag
|
||||
|
||||
item = Source(
|
||||
title=row.title,
|
||||
url=row.url,
|
||||
doc_type="bill",
|
||||
abstract=(
|
||||
f"{row.bill_type} {row.number} in the {ordinal(row.congress)} "
|
||||
f"Congress, policy area {row.policy_area}, matched on "
|
||||
f"{row.matched_term}."
|
||||
),
|
||||
access_date=dt.date.today().isoformat(),
|
||||
)
|
||||
return store.upsert(
|
||||
item,
|
||||
tags=[
|
||||
Tag.module("market"),
|
||||
Tag.table("market.bill"),
|
||||
Tag.source("congress"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def congress_of(year: int) -> int:
|
||||
"""The Congress in session during *year* (the 119th covers 2025–2026)."""
|
||||
return ((year - 1789) // 2) + 1
|
||||
@@ -376,6 +575,20 @@ def _yaml_doc(body: bytes, expect: type, status: int) -> Any:
|
||||
return doc
|
||||
|
||||
|
||||
class _BillBudget:
|
||||
"""At most *limit* uncached requests (0 = unlimited)."""
|
||||
|
||||
def __init__(self, limit: int) -> None:
|
||||
self.limit = limit
|
||||
self.used = 0
|
||||
|
||||
def allow(self) -> bool:
|
||||
if self.limit and self.used >= self.limit:
|
||||
return False
|
||||
self.used += 1
|
||||
return True
|
||||
|
||||
|
||||
def _download(
|
||||
cl: httpx.Client,
|
||||
url: str,
|
||||
@@ -384,10 +597,13 @@ def _download(
|
||||
pacer: Any,
|
||||
path: Path | None,
|
||||
source: str,
|
||||
) -> bytes:
|
||||
budget: _BillBudget | None = None,
|
||||
) -> bytes | None:
|
||||
cached = _cached(path)
|
||||
if cached is not None:
|
||||
return cached
|
||||
if budget is not None and not budget.allow():
|
||||
return None
|
||||
response = http.get(cl, url, params=params, pacer=pacer, source=source)
|
||||
_store(path, response.content, str(response.request.url))
|
||||
return response.content
|
||||
@@ -548,3 +764,216 @@ def fetch_assignments(
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _bill_cache(cache: Path | None, congress: int, *parts: str) -> Path | None:
|
||||
if cache is None:
|
||||
return None
|
||||
return cache.joinpath("bills", str(congress), *parts)
|
||||
|
||||
|
||||
def _offset_path(cache: Path | None, congress: int, *prefix: str) -> Any:
|
||||
def path_for(offset: int) -> Path | None:
|
||||
return _bill_cache(cache, congress, *prefix, f"{offset}.json")
|
||||
|
||||
return path_for
|
||||
|
||||
|
||||
def _first_stamp(kind: str, item: Mapping[str, Any]) -> str:
|
||||
if kind == "bills":
|
||||
return f"{item.get('type')}:{item.get('number')}"
|
||||
if kind == "cosponsors":
|
||||
return str(item.get("bioguideId") or "")
|
||||
return f"{item.get('actionDate')}|{item.get('text')}"
|
||||
|
||||
|
||||
def _page_items(
|
||||
api: httpx.Client,
|
||||
url: str,
|
||||
kind: str,
|
||||
*,
|
||||
pacer: Any,
|
||||
path_for: Any,
|
||||
budget: _BillBudget | None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""Pages of *kind* (``bills``, ``actions``, or ``cosponsors``). ``None``
|
||||
when the request budget stops the pull. A repeated first row ends the
|
||||
page walk; ``pagination.next`` is never followed."""
|
||||
items: list[dict[str, Any]] = []
|
||||
offset = 0
|
||||
seen_first: str | None = None
|
||||
got = 0
|
||||
while True:
|
||||
body = _download(
|
||||
api,
|
||||
url,
|
||||
{"limit": LIMIT, "offset": offset, "format": "json"},
|
||||
pacer=pacer,
|
||||
path=path_for(offset),
|
||||
source="congress",
|
||||
budget=budget,
|
||||
)
|
||||
if body is None:
|
||||
return None
|
||||
payload = _object(body, "congress", 200)
|
||||
raw = payload.get(kind) or []
|
||||
if not isinstance(raw, list):
|
||||
raw = []
|
||||
first = ""
|
||||
if raw and isinstance(raw[0], dict):
|
||||
first = _first_stamp(kind, raw[0])
|
||||
if first and first == seen_first:
|
||||
break
|
||||
if first:
|
||||
seen_first = first
|
||||
items.extend(row for row in raw if isinstance(row, dict))
|
||||
got += len(raw)
|
||||
page = payload.get("pagination")
|
||||
count = page.get("count") if isinstance(page, dict) else None
|
||||
done = not raw or len(raw) < LIMIT
|
||||
if not done and count is not None:
|
||||
done = got >= int(count)
|
||||
if done:
|
||||
break
|
||||
offset += LIMIT
|
||||
return items
|
||||
|
||||
|
||||
def _bill_detail(
|
||||
api: httpx.Client,
|
||||
congress: int,
|
||||
bill_type: str,
|
||||
number: int,
|
||||
*,
|
||||
pacer: Any,
|
||||
cache: Path | None,
|
||||
budget: _BillBudget | None,
|
||||
) -> dict[str, Any] | None:
|
||||
url = f"{CONGRESS_API}bill/{congress}/{bill_type.lower()}/{number}"
|
||||
body = _download(
|
||||
api,
|
||||
url,
|
||||
{"format": "json"},
|
||||
pacer=pacer,
|
||||
path=_bill_cache(cache, congress, bill_type, str(number), "detail.json"),
|
||||
source="congress",
|
||||
budget=budget,
|
||||
)
|
||||
if body is None:
|
||||
return None
|
||||
return _object(body, "congress", 200)
|
||||
|
||||
|
||||
def fetch_bills(
|
||||
api: httpx.Client,
|
||||
congresses: Sequence[int],
|
||||
*,
|
||||
pacer: Any,
|
||||
cache: Path | None = None,
|
||||
limit: int = 0,
|
||||
) -> tuple[list[BillRow], list[BillActionRow], list[BillMemberRow], bool]:
|
||||
"""Title matches in *congresses* whose detail policy area is Health,
|
||||
with their actions, sponsors and cosponsors. The fourth value is true
|
||||
when *limit* uncached requests were spent before the pull finished.
|
||||
A stopped pull returns what it already confirmed and nothing past the
|
||||
request that the budget refused."""
|
||||
budget = _BillBudget(limit)
|
||||
bills: list[BillRow] = []
|
||||
actions: list[BillActionRow] = []
|
||||
members: list[BillMemberRow] = []
|
||||
for congress in congresses:
|
||||
listed = _page_items(
|
||||
api,
|
||||
f"{CONGRESS_API}bill/{congress}",
|
||||
"bills",
|
||||
pacer=pacer,
|
||||
path_for=_offset_path(cache, congress),
|
||||
budget=budget,
|
||||
)
|
||||
if listed is None:
|
||||
return bills, actions, members, True
|
||||
stopped = _keep_matches(
|
||||
api,
|
||||
congress,
|
||||
listed,
|
||||
pacer=pacer,
|
||||
cache=cache,
|
||||
budget=budget,
|
||||
bills=bills,
|
||||
actions=actions,
|
||||
members=members,
|
||||
)
|
||||
if stopped:
|
||||
return bills, actions, members, True
|
||||
return bills, actions, members, False
|
||||
|
||||
|
||||
def _keep_matches(
|
||||
api: httpx.Client,
|
||||
congress: int,
|
||||
listed: Sequence[Mapping[str, Any]],
|
||||
*,
|
||||
pacer: Any,
|
||||
cache: Path | None,
|
||||
budget: _BillBudget,
|
||||
bills: list[BillRow],
|
||||
actions: list[BillActionRow],
|
||||
members: list[BillMemberRow],
|
||||
) -> bool:
|
||||
"""Append Health title-matches from one congress. True when the budget
|
||||
stops the pull before this congress is finished."""
|
||||
for item in listed:
|
||||
if match_title(str(item.get("title") or "")) is None:
|
||||
continue
|
||||
bill_type = str(item.get("type") or "").upper()
|
||||
try:
|
||||
number = int(item["number"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
if bill_type not in _BILL_SLUG:
|
||||
continue
|
||||
detail = _bill_detail(
|
||||
api, congress, bill_type, number, pacer=pacer, cache=cache, budget=budget
|
||||
)
|
||||
if detail is None:
|
||||
return True
|
||||
row = bill_from_detail(detail)
|
||||
if row is None:
|
||||
continue
|
||||
stem = f"{CONGRESS_API}bill/{congress}/{bill_type.lower()}/{number}"
|
||||
act = _page_items(
|
||||
api,
|
||||
f"{stem}/actions",
|
||||
"actions",
|
||||
pacer=pacer,
|
||||
path_for=_offset_path(cache, congress, bill_type, str(number), "actions"),
|
||||
budget=budget,
|
||||
)
|
||||
if act is None:
|
||||
return True
|
||||
cos = _page_items(
|
||||
api,
|
||||
f"{stem}/cosponsors",
|
||||
"cosponsors",
|
||||
pacer=pacer,
|
||||
path_for=_offset_path(
|
||||
cache, congress, bill_type, str(number), "cosponsors"
|
||||
),
|
||||
budget=budget,
|
||||
)
|
||||
if cos is None:
|
||||
return True
|
||||
bills.append(row)
|
||||
actions.extend(
|
||||
action_rows({"actions": act}, row.congress, row.bill_type, row.number)
|
||||
)
|
||||
members.extend(
|
||||
member_rows(
|
||||
detail,
|
||||
{"cosponsors": cos},
|
||||
row.congress,
|
||||
row.bill_type,
|
||||
row.number,
|
||||
)
|
||||
)
|
||||
return False
|
||||
|
||||
@@ -94,6 +94,19 @@ CREATE TABLE IF NOT EXISTS market.congress_member (
|
||||
CREATE TABLE IF NOT EXISTS market.committee_assignment (
|
||||
congress INTEGER, snapshot DATE, committee_id VARCHAR, bioguide_id VARCHAR,
|
||||
rank INTEGER, title VARCHAR, party VARCHAR, source_sha VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.bill (
|
||||
congress INTEGER, bill_type VARCHAR, number INTEGER, title VARCHAR,
|
||||
introduced DATE, policy_area VARCHAR, latest_action_date DATE,
|
||||
latest_action VARCHAR, matched_term VARCHAR, url VARCHAR, item_key VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.bill_action (
|
||||
congress INTEGER, bill_type VARCHAR, number INTEGER, action_date DATE,
|
||||
text VARCHAR, action_code VARCHAR, committee_codes VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.bill_member (
|
||||
congress INTEGER, bill_type VARCHAR, number INTEGER, bioguide_id VARCHAR,
|
||||
role VARCHAR, since DATE
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -380,6 +393,51 @@ class AssignmentRow:
|
||||
source_sha: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BillRow:
|
||||
"""One Congress.gov bill kept because its title matched a physician-payment
|
||||
term and its policy area is Health. ``item_key`` is empty until cited."""
|
||||
|
||||
congress: int
|
||||
bill_type: str
|
||||
number: int
|
||||
title: str
|
||||
introduced: date | None
|
||||
policy_area: str
|
||||
latest_action_date: date | None
|
||||
latest_action: str
|
||||
matched_term: str
|
||||
url: str
|
||||
item_key: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BillActionRow:
|
||||
"""One dated action. ``committee_codes`` is a ``|``-joined list of
|
||||
Congress.gov ``systemCode`` values, in source order."""
|
||||
|
||||
congress: int
|
||||
bill_type: str
|
||||
number: int
|
||||
action_date: date
|
||||
text: str
|
||||
action_code: str
|
||||
committee_codes: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BillMemberRow:
|
||||
"""A sponsor (``since`` is the introduced date) or a cosponsor (``since``
|
||||
is the sponsorship date)."""
|
||||
|
||||
congress: int
|
||||
bill_type: str
|
||||
number: int
|
||||
bioguide_id: str
|
||||
role: str
|
||||
since: date | None
|
||||
|
||||
|
||||
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
|
||||
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
|
||||
a statement per row, far too slow for filing lists)."""
|
||||
@@ -862,6 +920,65 @@ def read_assignments(con: Any) -> list[AssignmentRow]:
|
||||
)
|
||||
|
||||
|
||||
def _replace_congresses(con: Any, table: str, rows: Sequence[Any]) -> None:
|
||||
for chunk in _chunks(sorted({r.congress for r in rows})):
|
||||
con.execute(
|
||||
f"DELETE FROM {table} WHERE congress IN ({','.join('?' * len(chunk))})",
|
||||
list(chunk),
|
||||
)
|
||||
|
||||
|
||||
def write_bills(con: Any, rows: Sequence[BillRow]) -> int:
|
||||
"""Replace the bills of each congress present in *rows*."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
_replace_congresses(con, "market.bill", rows)
|
||||
return _insert(con, "market.bill", rows)
|
||||
|
||||
|
||||
def read_bills(con: Any) -> list[BillRow]:
|
||||
rows = _select(con, BillRow, "market.bill")
|
||||
return sorted(
|
||||
(BillRow(*r) for r in rows),
|
||||
key=lambda r: (r.congress, r.bill_type, r.number),
|
||||
)
|
||||
|
||||
|
||||
def write_bill_actions(con: Any, rows: Sequence[BillActionRow]) -> int:
|
||||
"""Replace the actions of each congress present in *rows*."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
_replace_congresses(con, "market.bill_action", rows)
|
||||
return _insert(con, "market.bill_action", rows)
|
||||
|
||||
|
||||
def read_bill_actions(con: Any) -> list[BillActionRow]:
|
||||
rows = _select(con, BillActionRow, "market.bill_action")
|
||||
return sorted(
|
||||
(BillActionRow(*r) for r in rows),
|
||||
key=lambda r: (r.congress, r.bill_type, r.number, r.action_date, r.text),
|
||||
)
|
||||
|
||||
|
||||
def write_bill_members(con: Any, rows: Sequence[BillMemberRow]) -> int:
|
||||
"""Replace the sponsors and cosponsors of each congress present in *rows*."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
_replace_congresses(con, "market.bill_member", rows)
|
||||
return _insert(con, "market.bill_member", rows)
|
||||
|
||||
|
||||
def read_bill_members(con: Any) -> list[BillMemberRow]:
|
||||
rows = _select(con, BillMemberRow, "market.bill_member")
|
||||
return sorted(
|
||||
(BillMemberRow(*r) for r in rows),
|
||||
key=lambda r: (r.congress, r.bill_type, r.number, r.role, r.bioguide_id),
|
||||
)
|
||||
|
||||
|
||||
# ── districts ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -2210,6 +2210,10 @@ def congress(env, monkeypatch):
|
||||
T.ensure_tables(c)
|
||||
seen: list[httpx.Request] = []
|
||||
members = json.loads((FX / "congress_members_118.json").read_text())
|
||||
bills = json.loads((FX / "congress_bills_page.json").read_text())
|
||||
detail = json.loads((FX / "congress_bill_detail.json").read_text())
|
||||
actions = json.loads((FX / "congress_bill_actions.json").read_text())
|
||||
cosponsors = json.loads((FX / "congress_bill_cosponsors.json").read_text())
|
||||
people = (FX / "legislators_snippet.yaml").read_bytes()
|
||||
membership = (FX / "membership_snippet.yaml").read_bytes()
|
||||
state = {"status": 200}
|
||||
@@ -2222,6 +2226,16 @@ def congress(env, monkeypatch):
|
||||
return httpx.Response(state["status"], text="no")
|
||||
host = request.url.host
|
||||
if host == "api.congress.gov":
|
||||
path = request.url.path
|
||||
if path.endswith("/actions"):
|
||||
return httpx.Response(200, json=actions)
|
||||
if path.endswith("/cosponsors"):
|
||||
return httpx.Response(200, json=cosponsors)
|
||||
parts = [part for part in path.split("/") if part]
|
||||
if "bill" in parts and len(parts) >= 5:
|
||||
return httpx.Response(200, json=detail)
|
||||
if "bill" in parts:
|
||||
return httpx.Response(200, json=bills)
|
||||
return httpx.Response(200, json=members)
|
||||
if host == "api.github.com":
|
||||
return httpx.Response(200, json=[{"sha": "deadbeef"}])
|
||||
@@ -2365,6 +2379,139 @@ def test_committees_cli_write(congress):
|
||||
assert ("market.committee_assignment", 1) in logged
|
||||
|
||||
|
||||
def test_bills_cli_no_write_lists_matches(congress):
|
||||
c, _, published, _, seen, _ = congress
|
||||
res = runner.invoke(app, ["market", "bills", "--from", "118", "--to", "118"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "118 HR 1234 physician fee schedule" in res.output
|
||||
assert "1 bills" in res.output
|
||||
assert "congress 118: 1" in res.output
|
||||
assert "Farm Bill" not in res.output
|
||||
assert "dry run: pass --write" in res.output
|
||||
assert _count(c, "market.bill") == 0
|
||||
assert published == []
|
||||
hits = [r for r in seen if r.url.host == "api.congress.gov"]
|
||||
assert hits
|
||||
assert all(CONGRESS_SECRET not in str(r.url) for r in hits)
|
||||
assert all("api_key" not in str(r.url) for r in hits)
|
||||
assert hits[0].headers["x-api-key"] == CONGRESS_SECRET
|
||||
|
||||
|
||||
def test_bills_cli_offline_write(congress):
|
||||
c, store, published, hoard, seen, _ = congress
|
||||
T.write_gap(c, "congress:bills", "old")
|
||||
res = runner.invoke(
|
||||
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stored: 1 bills" in res.output
|
||||
(bill,) = T.read_bills(c)
|
||||
assert bill.bill_type == "HR"
|
||||
assert bill.number == 1234
|
||||
assert bill.matched_term == "physician fee schedule"
|
||||
assert bill.policy_area == "Health"
|
||||
assert bill.introduced == date(2023, 1, 9)
|
||||
assert bill.item_key
|
||||
assert len(T.read_bill_actions(c)) == 3
|
||||
roles = sorted((m.role, m.bioguide_id) for m in T.read_bill_members(c))
|
||||
assert roles == [
|
||||
("cosponsor", "S000001"),
|
||||
("cosponsor", "Y000033"),
|
||||
("sponsor", "D000624"),
|
||||
]
|
||||
assert T.read_gaps(c) == []
|
||||
assert published == [True]
|
||||
assert any(
|
||||
(i.url or "").endswith("/118th-congress/house-bill/1234")
|
||||
for i in store.list_items()
|
||||
)
|
||||
assert ("market.bill", 1) in c.execute(
|
||||
"SELECT table_name, rows FROM cms.ingest_log"
|
||||
).fetchall()
|
||||
manifest = (hoard / "MANIFEST.jsonl").read_text()
|
||||
assert CONGRESS_SECRET not in manifest
|
||||
assert all(CONGRESS_SECRET not in str(r.url) for r in seen)
|
||||
kept = T.BillRow(
|
||||
117,
|
||||
"S",
|
||||
1,
|
||||
"Telehealth Modernization Act",
|
||||
None,
|
||||
"Health",
|
||||
None,
|
||||
"",
|
||||
"telehealth",
|
||||
"https://www.congress.gov/bill/117th-congress/senate-bill/1",
|
||||
"",
|
||||
)
|
||||
T.write_bills(c, [kept])
|
||||
assert {r.congress for r in T.read_bills(c)} == {117, 118}
|
||||
|
||||
|
||||
def test_bills_cli_refusal_prints_gap(congress):
|
||||
c, _, published, _, seen, state = congress
|
||||
state["status"] = 403
|
||||
res = runner.invoke(
|
||||
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: congress:bills: congress: HTTP 403" in res.output
|
||||
assert CONGRESS_SECRET not in res.output
|
||||
assert _count(c, "market.bill") == 0
|
||||
assert len(seen) == 1
|
||||
assert seen[0].url.host == "api.congress.gov"
|
||||
assert seen[0].headers["x-api-key"] == CONGRESS_SECRET
|
||||
assert CONGRESS_SECRET not in str(seen[0].url)
|
||||
assert published == [True]
|
||||
|
||||
|
||||
def test_bills_cli_from_after_to(congress):
|
||||
res = runner.invoke(app, ["market", "bills", "--from", "119", "--to", "110"])
|
||||
assert res.exit_code == 2
|
||||
assert "--from 119 is after --to 110" in res.output
|
||||
|
||||
|
||||
def test_bills_cli_missing_key_gap_only(congress, monkeypatch):
|
||||
c, _, published, _, seen, _ = congress
|
||||
monkeypatch.setattr(http, "api_key", lambda source, env=None: None)
|
||||
res = runner.invoke(
|
||||
app, ["market", "bills", "--from", "118", "--to", "118", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: congress:bills:" in res.output
|
||||
assert "CONGRESS_API_KEY" in res.output
|
||||
assert seen == []
|
||||
assert [g.source for g in T.read_gaps(c)] == ["congress:bills"]
|
||||
assert _count(c, "market.bill") == 0
|
||||
assert published == [True]
|
||||
|
||||
|
||||
def test_bills_cli_limit_stores_nothing_and_keeps_the_gap(congress):
|
||||
c, _, published, _, seen, _ = congress
|
||||
T.write_gap(c, "congress:bills", "old")
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"market",
|
||||
"bills",
|
||||
"--from",
|
||||
"118",
|
||||
"--to",
|
||||
"119",
|
||||
"--limit",
|
||||
"1",
|
||||
"--write",
|
||||
],
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stopped: request limit" in res.output
|
||||
assert "stored:" not in res.output
|
||||
assert [g.note for g in T.read_gaps(c)] == ["old"]
|
||||
assert _count(c, "market.bill") == 0
|
||||
assert published == []
|
||||
assert [r.url.path for r in seen] == ["/v3/bill/118"]
|
||||
|
||||
|
||||
def test_committees_cli_refusal_prints_gap(congress):
|
||||
c, _, published, _, seen, state = congress
|
||||
state["status"] = 403
|
||||
|
||||
21
tests/market/fixtures/congress_bill_actions.json
Normal file
21
tests/market/fixtures/congress_bill_actions.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"actions": [
|
||||
{
|
||||
"actionDate": "2023-01-09",
|
||||
"text": "Introduced in House",
|
||||
"actionCode": "1000"
|
||||
},
|
||||
{
|
||||
"actionDate": "2023-01-10",
|
||||
"text": "Referred to Ways and Means",
|
||||
"actionCode": "H11100",
|
||||
"committees": [{"systemCode": "hswm"}]
|
||||
},
|
||||
{
|
||||
"actionDate": "2023-02-01",
|
||||
"text": "Referred to Health",
|
||||
"actionCode": "H30000",
|
||||
"committees": [{"systemCode": "hsif"}, {"systemCode": "hsif14"}]
|
||||
}
|
||||
]
|
||||
}
|
||||
6
tests/market/fixtures/congress_bill_cosponsors.json
Normal file
6
tests/market/fixtures/congress_bill_cosponsors.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"cosponsors": [
|
||||
{"bioguideId": "S000001", "sponsorshipDate": "2023-01-11"},
|
||||
{"bioguideId": "Y000033", "sponsorshipDate": "2023-01-12"}
|
||||
]
|
||||
}
|
||||
15
tests/market/fixtures/congress_bill_detail.json
Normal file
15
tests/market/fixtures/congress_bill_detail.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"bill": {
|
||||
"congress": 118,
|
||||
"type": "HR",
|
||||
"number": "1234",
|
||||
"title": "Medicare Physician Fee Schedule Update Act",
|
||||
"introducedDate": "2023-01-09",
|
||||
"policyArea": {"name": "Health"},
|
||||
"latestAction": {
|
||||
"actionDate": "2023-02-01",
|
||||
"text": "Referred to the Committee on Ways and Means."
|
||||
},
|
||||
"sponsors": [{"bioguideId": "D000624"}]
|
||||
}
|
||||
}
|
||||
21
tests/market/fixtures/congress_bills_page.json
Normal file
21
tests/market/fixtures/congress_bills_page.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"bills": [
|
||||
{
|
||||
"congress": 118,
|
||||
"type": "HR",
|
||||
"number": "1234",
|
||||
"title": "Medicare Physician Fee Schedule Update Act",
|
||||
"latestAction": {
|
||||
"actionDate": "2023-02-01",
|
||||
"text": "Referred to the Committee on Ways and Means."
|
||||
}
|
||||
},
|
||||
{
|
||||
"congress": 118,
|
||||
"type": "S",
|
||||
"number": "9",
|
||||
"title": "Farm Bill"
|
||||
}
|
||||
],
|
||||
"pagination": {"count": 2}
|
||||
}
|
||||
@@ -675,3 +675,311 @@ def test_assignments_replace_one_congress_and_keep_the_other():
|
||||
}
|
||||
assert T.write_assignments(c, []) == 0
|
||||
c.close()
|
||||
|
||||
|
||||
def _bill_detail() -> dict:
|
||||
return json.loads((FX / "congress_bill_detail.json").read_text())
|
||||
|
||||
|
||||
def _bill_actions() -> dict:
|
||||
return json.loads((FX / "congress_bill_actions.json").read_text())
|
||||
|
||||
|
||||
def _bill_cosponsors() -> dict:
|
||||
return json.loads((FX / "congress_bill_cosponsors.json").read_text())
|
||||
|
||||
|
||||
def test_bill_title_filter():
|
||||
assert (
|
||||
C.match_title("Medicare Physician Fee Schedule Update Act")
|
||||
== "physician fee schedule"
|
||||
)
|
||||
assert C.match_title("Telehealth Modernization Act") == "telehealth"
|
||||
assert C.match_title("Farm Bill") is None
|
||||
|
||||
|
||||
def test_bill_kept_only_health_policy_area():
|
||||
assert C.bill_from_detail([]) is None
|
||||
assert C.bill_from_detail({}) is None
|
||||
assert C.bill_from_detail({"bill": "nope"}) is None
|
||||
detail = _bill_detail()
|
||||
detail["bill"]["title"] = "Farm Bill"
|
||||
assert C.bill_from_detail(detail) is None
|
||||
detail = _bill_detail()
|
||||
detail["bill"]["policyArea"] = {"name": "Taxation"}
|
||||
assert C.bill_from_detail(detail) is None
|
||||
detail = _bill_detail()
|
||||
detail["bill"]["policyArea"] = "Health"
|
||||
assert C.bill_from_detail(detail) is None
|
||||
detail = _bill_detail()
|
||||
detail["bill"]["latestAction"] = None
|
||||
detail["bill"]["introducedDate"] = ""
|
||||
row = C.bill_from_detail(detail)
|
||||
assert row is not None
|
||||
assert row.introduced is None
|
||||
assert row.latest_action_date is None
|
||||
assert row.latest_action == ""
|
||||
|
||||
|
||||
def test_ordinal():
|
||||
assert C.ordinal(111) == "111th"
|
||||
assert C.ordinal(112) == "112th"
|
||||
assert C.ordinal(121) == "121st"
|
||||
assert C.ordinal(122) == "122nd"
|
||||
assert C.ordinal(123) == "123rd"
|
||||
assert C.ordinal(124) == "124th"
|
||||
|
||||
|
||||
def test_bill_url():
|
||||
assert (
|
||||
C.bill_url(118, "HR", 1234)
|
||||
== "https://www.congress.gov/bill/118th-congress/house-bill/1234"
|
||||
)
|
||||
|
||||
|
||||
def test_actions_and_cosponsors_rows():
|
||||
payload = _bill_actions()
|
||||
payload["actions"].extend(
|
||||
["nope", {"text": "undated"}, {"actionDate": "not-a-date", "text": "bad"}]
|
||||
)
|
||||
actions = C.action_rows(payload, 118, "HR", 1234)
|
||||
assert C.action_rows({"actions": "nope"}, 118, "HR", 1234) == []
|
||||
assert C.action_rows([], 118, "HR", 1234) == []
|
||||
assert [a.action_date for a in actions] == [
|
||||
date(2023, 1, 9),
|
||||
date(2023, 1, 10),
|
||||
date(2023, 2, 1),
|
||||
]
|
||||
assert actions[0].text == "Introduced in House"
|
||||
assert actions[0].action_code == "1000"
|
||||
assert actions[0].committee_codes == ""
|
||||
assert actions[1].committee_codes == "hswm"
|
||||
assert actions[2].committee_codes == "hsif|hsif14"
|
||||
cosponsors = _bill_cosponsors()
|
||||
cosponsors["cosponsors"].extend(["nope", {"name": "nobody"}, {"bioguideId": ""}])
|
||||
members = C.member_rows(_bill_detail(), cosponsors, 118, "HR", 1234)
|
||||
sponsors = [m for m in members if m.role == "sponsor"]
|
||||
cosponsors = [m for m in members if m.role == "cosponsor"]
|
||||
assert len(sponsors) == 1
|
||||
assert sponsors[0].bioguide_id == "D000624"
|
||||
assert sponsors[0].since == date(2023, 1, 9)
|
||||
assert [(m.bioguide_id, m.since) for m in cosponsors] == [
|
||||
("S000001", date(2023, 1, 11)),
|
||||
("Y000033", date(2023, 1, 12)),
|
||||
]
|
||||
bare = _bill_detail()
|
||||
bare["bill"]["sponsors"] = "nope"
|
||||
assert C.member_rows(bare, {"cosponsors": "nope"}, 118, "HR", 1234) == []
|
||||
assert C.member_rows([], {"cosponsors": []}, 118, "HR", 1234) == []
|
||||
row = C.bill_from_detail(_bill_detail())
|
||||
assert row is not None
|
||||
assert row.congress == 118
|
||||
assert row.bill_type == "HR"
|
||||
assert row.number == 1234
|
||||
assert row.matched_term == "physician fee schedule"
|
||||
assert row.policy_area == "Health"
|
||||
assert row.introduced == date(2023, 1, 9)
|
||||
assert row.latest_action_date == date(2023, 2, 1)
|
||||
assert row.url == C.bill_url(118, "HR", 1234)
|
||||
|
||||
|
||||
def _route_bills(request: httpx.Request, page: dict | None = None) -> httpx.Response:
|
||||
path = request.url.path
|
||||
if path.endswith("/actions"):
|
||||
return httpx.Response(200, json=_bill_actions())
|
||||
if path.endswith("/cosponsors"):
|
||||
return httpx.Response(200, json=_bill_cosponsors())
|
||||
parts = [part for part in path.split("/") if part]
|
||||
if "bill" in parts and len(parts) >= 5:
|
||||
detail = _bill_detail()
|
||||
if parts[-2:] == ["s", "9"]:
|
||||
detail["bill"]["type"] = "S"
|
||||
detail["bill"]["number"] = "9"
|
||||
detail["bill"]["title"] = "Telehealth Tax Act"
|
||||
detail["bill"]["policyArea"] = {"name": "Taxation"}
|
||||
return httpx.Response(200, json=detail)
|
||||
body = (
|
||||
page
|
||||
if page is not None
|
||||
else json.loads((FX / "congress_bills_page.json").read_text())
|
||||
)
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
|
||||
def test_fetch_bills_pages_and_drops_a_non_health_match(monkeypatch):
|
||||
monkeypatch.setattr(C, "LIMIT", 1)
|
||||
pages = {
|
||||
0: {
|
||||
"bills": [
|
||||
{
|
||||
"type": "HR",
|
||||
"number": "1234",
|
||||
"title": "Medicare Physician Fee Schedule Update Act",
|
||||
}
|
||||
],
|
||||
"pagination": {"count": 99},
|
||||
},
|
||||
1: {
|
||||
"bills": [
|
||||
{
|
||||
"type": "S",
|
||||
"number": "9",
|
||||
"title": "Telehealth Tax Act",
|
||||
},
|
||||
{"type": "FOO", "number": "1", "title": "Primary Care Act"},
|
||||
{"type": "HR", "number": "nope", "title": "Primary Care Act"},
|
||||
],
|
||||
"pagination": {"count": 99},
|
||||
},
|
||||
2: {
|
||||
"bills": [{"type": "S", "number": "9", "title": "Telehealth Tax Act"}],
|
||||
"pagination": {"count": 99},
|
||||
},
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.rstrip("/").endswith("/bill/118"):
|
||||
offset = int(request.url.params.get("offset", "0"))
|
||||
return httpx.Response(200, json=pages[offset])
|
||||
return _route_bills(request)
|
||||
|
||||
bills, actions, members, stopped = C.fetch_bills(
|
||||
_client(handler), [118], pacer=Pacer()
|
||||
)
|
||||
assert stopped is False
|
||||
assert [b.number for b in bills] == [1234]
|
||||
assert len(actions) == 3
|
||||
assert len(members) == 3
|
||||
|
||||
|
||||
def test_fetch_bills_stops_at_the_request_limit_and_ignores_a_non_list():
|
||||
calls: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
calls.append(request.url.path)
|
||||
if request.url.path.rstrip("/").endswith("/bill/119"):
|
||||
return httpx.Response(200, json={"bills": {"nope": True}})
|
||||
return _route_bills(request)
|
||||
|
||||
bills, _, _, stopped = C.fetch_bills(
|
||||
_client(handler), [118, 119], pacer=Pacer(), limit=1
|
||||
)
|
||||
assert stopped is True
|
||||
assert bills == []
|
||||
assert calls == ["/v3/bill/118"]
|
||||
empty, _, _, stopped = C.fetch_bills(
|
||||
_client(lambda request: httpx.Response(200, json={"bills": {"nope": True}})),
|
||||
[118],
|
||||
pacer=Pacer(),
|
||||
)
|
||||
assert empty == [] and stopped is False
|
||||
|
||||
|
||||
def test_cached_bill_pages_do_not_spend_the_limit(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
|
||||
seen: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request.url.path)
|
||||
return _route_bills(request)
|
||||
|
||||
first, _, _, stopped = C.fetch_bills(
|
||||
_client(handler), [118], pacer=Pacer(), cache=tmp_path
|
||||
)
|
||||
assert len(first) == 1 and stopped is False
|
||||
assert seen
|
||||
seen.clear()
|
||||
second, _, _, stopped = C.fetch_bills(
|
||||
_client(handler), [118], pacer=Pacer(), cache=tmp_path, limit=1
|
||||
)
|
||||
assert seen == []
|
||||
assert stopped is False
|
||||
assert [b.number for b in second] == [1234]
|
||||
|
||||
|
||||
def test_bill_budget_stops_before_the_next_congress_list():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"bills": [{"type": "S", "number": "1", "title": "Farm Bill"}],
|
||||
"pagination": {"count": 1},
|
||||
},
|
||||
)
|
||||
|
||||
bills, _, _, stopped = C.fetch_bills(
|
||||
_client(handler), [118, 119], pacer=Pacer(), limit=1
|
||||
)
|
||||
assert bills == [] and stopped is True
|
||||
|
||||
|
||||
def test_bill_budget_stops_on_actions_and_on_cosponsors(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return _route_bills(request)
|
||||
|
||||
C.fetch_bills(_client(handler), [118], pacer=Pacer(), cache=tmp_path)
|
||||
root = tmp_path / "bills" / "118" / "HR" / "1234"
|
||||
for path in (root / "actions").glob("*.json"):
|
||||
path.unlink()
|
||||
for path in (root / "cosponsors").glob("*.json"):
|
||||
path.unlink()
|
||||
_, _, _, stopped = C.fetch_bills(
|
||||
_client(handler), [118], pacer=Pacer(), cache=tmp_path, limit=1
|
||||
)
|
||||
assert stopped is True
|
||||
for path in (root / "detail.json", *root.joinpath("actions").glob("*.json")):
|
||||
path.unlink()
|
||||
_, _, _, stopped = C.fetch_bills(
|
||||
_client(handler), [118], pacer=Pacer(), cache=tmp_path, limit=1
|
||||
)
|
||||
assert stopped is True
|
||||
|
||||
|
||||
def test_bill_fetch_refuses_a_non_json_page():
|
||||
with pytest.raises(http.Refused, match="not JSON"):
|
||||
C.fetch_bills(
|
||||
_client(lambda request: httpx.Response(200, content=b"nope")),
|
||||
[118],
|
||||
pacer=Pacer(),
|
||||
)
|
||||
|
||||
|
||||
def test_bills_replace_one_congress_and_keep_the_other():
|
||||
import duckdb
|
||||
|
||||
c = duckdb.connect(":memory:")
|
||||
older = T.BillRow(
|
||||
117,
|
||||
"S",
|
||||
1,
|
||||
"Telehealth Modernization Act",
|
||||
None,
|
||||
"Health",
|
||||
None,
|
||||
"",
|
||||
"telehealth",
|
||||
"https://www.congress.gov/bill/117th-congress/senate-bill/1",
|
||||
"",
|
||||
)
|
||||
newer = T.BillRow(
|
||||
118,
|
||||
"HR",
|
||||
1234,
|
||||
"Medicare Physician Fee Schedule Update Act",
|
||||
date(2023, 1, 9),
|
||||
"Health",
|
||||
date(2023, 2, 1),
|
||||
"Referred to the Committee on Ways and Means.",
|
||||
"physician fee schedule",
|
||||
"https://www.congress.gov/bill/118th-congress/house-bill/1234",
|
||||
"ITEM",
|
||||
)
|
||||
T.write_bills(c, [older, newer])
|
||||
T.write_bills(c, [newer])
|
||||
assert {r.congress for r in T.read_bills(c)} == {117, 118}
|
||||
assert T.write_bills(c, []) == 0
|
||||
assert T.write_bill_actions(c, []) == 0
|
||||
assert T.write_bill_members(c, []) == 0
|
||||
c.close()
|
||||
|
||||
@@ -48,6 +48,9 @@ def test_ensure_tables_idempotent():
|
||||
"fec_contribution",
|
||||
"congress_member",
|
||||
"committee_assignment",
|
||||
"bill",
|
||||
"bill_action",
|
||||
"bill_member",
|
||||
}
|
||||
|
||||
|
||||
@@ -200,6 +203,29 @@ MEMBER = T.MemberRow(
|
||||
ASSIGNMENT = T.AssignmentRow(
|
||||
119, date(2025, 4, 1), "HSWM02", "E000001", 1, "Chair", "majority", "abc123"
|
||||
)
|
||||
BILL = T.BillRow(
|
||||
118,
|
||||
"HR",
|
||||
1234,
|
||||
"Medicare Physician Fee Schedule Update Act",
|
||||
date(2023, 1, 9),
|
||||
"Health",
|
||||
date(2023, 2, 1),
|
||||
"Referred to the Committee on Ways and Means.",
|
||||
"physician fee schedule",
|
||||
"https://www.congress.gov/bill/118th-congress/house-bill/1234",
|
||||
"ITEM",
|
||||
)
|
||||
BILL_ACTION = T.BillActionRow(
|
||||
118,
|
||||
"HR",
|
||||
1234,
|
||||
date(2023, 1, 9),
|
||||
"Introduced in House",
|
||||
"1000",
|
||||
"",
|
||||
)
|
||||
BILL_MEMBER = T.BillMemberRow(118, "HR", 1234, "D000624", "sponsor", date(2023, 1, 9))
|
||||
SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY")
|
||||
|
||||
ROUNDTRIP = {
|
||||
@@ -238,6 +264,17 @@ ROUNDTRIP = {
|
||||
lambda c, r: T.write_assignments(c, [r]),
|
||||
lambda c: T.read_assignments(c),
|
||||
),
|
||||
"bill": (BILL, lambda c, r: T.write_bills(c, [r]), lambda c: T.read_bills(c)),
|
||||
"bill_action": (
|
||||
BILL_ACTION,
|
||||
lambda c, r: T.write_bill_actions(c, [r]),
|
||||
lambda c: T.read_bill_actions(c),
|
||||
),
|
||||
"bill_member": (
|
||||
BILL_MEMBER,
|
||||
lambda c, r: T.write_bill_members(c, [r]),
|
||||
lambda c: T.read_bill_members(c),
|
||||
),
|
||||
"entity": (
|
||||
ENTITY,
|
||||
lambda c, r: T.write_entities(c, [r]),
|
||||
@@ -301,6 +338,9 @@ def test_write_empty_is_noop(con):
|
||||
assert T.write_fec_contributions(con, []) == 0
|
||||
assert T.write_members(con, []) == 0
|
||||
assert T.write_assignments(con, []) == 0
|
||||
assert T.write_bills(con, []) == 0
|
||||
assert T.write_bill_actions(con, []) == 0
|
||||
assert T.write_bill_members(con, []) == 0
|
||||
|
||||
|
||||
def test_lda_readers_filter_by_entity(con):
|
||||
|
||||
Reference in New Issue
Block a user