feat(market): district footprint of exposed companies and family billers, joined to members and health committees
All checks were successful
CI / lint (push) Successful in 49s
CI / notebooks-smoke (push) Successful in 1m55s
Deploy / changes (push) Successful in 15s
Infra CI / notebooks (push) Successful in 1m0s
Infra CI / zotero (push) Successful in 16s
CI / test (push) Successful in 4m39s
Infra CI / docs (push) Successful in 23s
Infra CI / api (push) Successful in 1m21s
Infra CI / bcda (push) Successful in 1m51s
Infra CI / mc (push) Successful in 21s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 1m14s
Deploy / api (push) Successful in 1m25s
Deploy / bcda (push) Successful in 2m0s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m20s
Deploy / report (push) Successful in 15s
All checks were successful
CI / lint (push) Successful in 49s
CI / notebooks-smoke (push) Successful in 1m55s
Deploy / changes (push) Successful in 15s
Infra CI / notebooks (push) Successful in 1m0s
Infra CI / zotero (push) Successful in 16s
CI / test (push) Successful in 4m39s
Infra CI / docs (push) Successful in 23s
Infra CI / api (push) Successful in 1m21s
Infra CI / bcda (push) Successful in 1m51s
Infra CI / mc (push) Successful in 21s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 1m14s
Deploy / api (push) Successful in 1m25s
Deploy / bcda (push) Successful in 2m0s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m20s
Deploy / report (push) Successful in 15s
This commit is contained in:
@@ -4,6 +4,9 @@ uv run stack market entities --write # seed + EDGAR entity registry
|
||||
uv run stack market entities --sic-sweep --write
|
||||
uv run stack market districts --write # Census ZCTA and county crosswalks
|
||||
uv run stack market hud --year 2025 --quarter 2 --congress 119 --write
|
||||
uv run stack market footprint --entities --write
|
||||
uv run stack market footprint --family HOME-VISITS --write
|
||||
uv run stack market district-view --family HOME-VISITS --top 10
|
||||
uv run stack market gaps # sources that could not be read
|
||||
|
||||
Every command fetches first, then opens one short DuckDB batch that only
|
||||
@@ -26,6 +29,7 @@ from market import congress as congress_api
|
||||
from market import districts as district_api
|
||||
from market import edgar, http, tiingo
|
||||
from market import fec as fec_api
|
||||
from market import footprint as footprint_api
|
||||
from market import lda as lda_api
|
||||
from market.seed import (
|
||||
SEAT_LABEL_KEYS,
|
||||
@@ -64,10 +68,12 @@ from market.tables import (
|
||||
write_bills,
|
||||
write_county_districts,
|
||||
write_entities,
|
||||
write_entity_sites,
|
||||
write_fec_committees,
|
||||
write_fec_contributions,
|
||||
write_fec_totals,
|
||||
write_filings,
|
||||
write_footprints,
|
||||
write_gap,
|
||||
write_hits,
|
||||
write_links,
|
||||
@@ -1789,6 +1795,188 @@ def hud(
|
||||
typer.echo(f"stored: {n} zip rows")
|
||||
|
||||
|
||||
def _exactly_one(left: bool, right: bool, message: str) -> None:
|
||||
if left == right:
|
||||
typer.echo(message)
|
||||
raise typer.Exit(2)
|
||||
|
||||
|
||||
def _mass(rows: list, field: str) -> str:
|
||||
total = sum(getattr(row, field) for row in rows)
|
||||
unknown = sum(getattr(row, field) for row in rows if row.district_id == "??")
|
||||
if not total:
|
||||
return "unmatched ??: 0"
|
||||
return f"unmatched ??: {unknown / total:.1%} ({unknown:g} / {total:g})"
|
||||
|
||||
|
||||
def _store_footprint(rows: list, sites: list | None) -> None:
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
n_sites = write_entity_sites(con, sites) if sites is not None else 0
|
||||
n = write_footprints(con, rows)
|
||||
if sites is not None:
|
||||
log_ingest(
|
||||
con,
|
||||
module="market.footprint",
|
||||
table_name="market.entity_site",
|
||||
rows=n_sites,
|
||||
source_file=f"cms-owners:{footprint_api.OWNERS_STAMP}",
|
||||
)
|
||||
log_ingest(
|
||||
con,
|
||||
module="market.footprint",
|
||||
table_name="market.footprint",
|
||||
rows=n,
|
||||
source_file="prov-svc:2024" if sites is None else "entity-site",
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: {n} footprint rows")
|
||||
|
||||
|
||||
@app.command()
|
||||
def footprint(
|
||||
entities: bool = typer.Option(
|
||||
False, "--entities", help="Sites of the exposed companies."
|
||||
),
|
||||
family: str = typer.Option(
|
||||
"", "--family", help="A code family whose billers are allocated."
|
||||
),
|
||||
congress: int = typer.Option(footprint_api.CONGRESS, "--congress"),
|
||||
write: bool = typer.Option(False, "--write", help="Store sites and the footprint."),
|
||||
) -> None:
|
||||
"""District footprint for data year 2024. Needs the Task 8 crosswalk
|
||||
(``stack market districts``). An unmatched ZIP is kept as ``??``."""
|
||||
_exactly_one(entities, bool(family), "pass exactly one of --entities or --family")
|
||||
con = _read()
|
||||
try:
|
||||
weights = footprint_api.crosswalk(con, congress)
|
||||
if not weights:
|
||||
typer.echo(
|
||||
f"no district crosswalk for congress {congress}"
|
||||
" — run `stack market districts`"
|
||||
)
|
||||
return
|
||||
if family:
|
||||
codes = footprint_api.family_codes(con, family)
|
||||
if not codes:
|
||||
typer.echo(f"no codes for family {family}")
|
||||
return
|
||||
path = footprint_api.prov_svc_file()
|
||||
typer.echo(f"{len(codes)} codes, scanning {path.name}")
|
||||
billers = footprint_api.family_billers(con, path, codes)
|
||||
loaded = None
|
||||
else:
|
||||
entities_rows = read_entities(con)
|
||||
typer.echo(f"{len(entities_rows)} entities")
|
||||
enrollments = footprint_api.load_enrollments(con)
|
||||
typer.echo(f"{len(enrollments)} enrollments")
|
||||
typer.echo("reading organization NPIs")
|
||||
npi_rows = footprint_api.org_provider_rows(con, entities_rows)
|
||||
typer.echo(f"{len(npi_rows)} organization NPIs")
|
||||
loaded = (entities_rows, enrollments, npi_rows)
|
||||
finally:
|
||||
con.close()
|
||||
if family:
|
||||
items = [
|
||||
footprint_api.AllocItem(zipcode, 1, 0, services, payment)
|
||||
for _npi, zipcode, services, payment in billers
|
||||
]
|
||||
rows = footprint_api.rollup(
|
||||
items,
|
||||
weights,
|
||||
level="family",
|
||||
key=family,
|
||||
congress=congress,
|
||||
year=footprint_api.YEAR,
|
||||
method=footprint_api.METHOD,
|
||||
)
|
||||
typer.echo(f"{len(items)} billers")
|
||||
typer.echo(_mass(rows, "payment"))
|
||||
sites = None
|
||||
else:
|
||||
entities_rows, enrollments, npi_rows = loaded
|
||||
sites = _entity_sites(entities_rows, enrollments, npi_rows)
|
||||
names = {row.key: row.name for row in entities_rows}
|
||||
counts = Counter(row.entity_key for row in sites)
|
||||
typer.echo(f"{len(sites)} sites")
|
||||
for key, n_sites in counts.most_common(10):
|
||||
typer.echo(f" {key} {names.get(key, '')}: {n_sites} sites")
|
||||
rows = []
|
||||
for key, _n in counts.most_common():
|
||||
items = [
|
||||
footprint_api.AllocItem(row.zip, 0, 1, 0, 0)
|
||||
for row in sites
|
||||
if row.entity_key == key
|
||||
]
|
||||
rows.extend(
|
||||
footprint_api.rollup(
|
||||
items,
|
||||
weights,
|
||||
level="entity",
|
||||
key=key,
|
||||
congress=congress,
|
||||
year=footprint_api.YEAR,
|
||||
method=footprint_api.METHOD,
|
||||
)
|
||||
)
|
||||
typer.echo(_mass(rows, "n_sites"))
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write")
|
||||
return
|
||||
_store_footprint(rows, sites)
|
||||
|
||||
|
||||
def _entity_sites(entities_rows, enrollments, npi_rows) -> list:
|
||||
sites = []
|
||||
seen: set[tuple[str, str, str]] = set()
|
||||
for path in footprint_api.owner_files():
|
||||
for row in footprint_api.owner_sites(path, enrollments, entities_rows):
|
||||
key = (row.entity_key, row.site_kind, row.site_id)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
sites.append(row)
|
||||
for row in footprint_api.npi_sites(npi_rows, entities_rows):
|
||||
key = (row.entity_key, row.site_kind, row.site_id)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
sites.append(row)
|
||||
return sites
|
||||
|
||||
|
||||
@app.command("district-view")
|
||||
def district_view(
|
||||
entity: str = typer.Option("", "--entity", help="An exposed-company key."),
|
||||
family: str = typer.Option("", "--family", help="A code-family key."),
|
||||
congress: int = typer.Option(footprint_api.CONGRESS, "--congress"),
|
||||
top: int = typer.Option(20, "--top", help="How many districts to print."),
|
||||
) -> None:
|
||||
"""Stored footprint joined to that congress's House member and health
|
||||
committees. ``agi_k`` is empty until the SOI join."""
|
||||
_exactly_one(bool(entity), bool(family), "pass exactly one of --entity or --family")
|
||||
level = "family" if family else "entity"
|
||||
key = family or entity
|
||||
con = _read()
|
||||
try:
|
||||
views = footprint_api.district_view(
|
||||
con, congress=congress, level=level, key=key, year=footprint_api.YEAR
|
||||
)
|
||||
finally:
|
||||
con.close()
|
||||
shown = views[:top]
|
||||
if not shown:
|
||||
typer.echo("no footprint stored")
|
||||
return
|
||||
for view in shown:
|
||||
seats = ",".join(view.health_committees) or "-"
|
||||
typer.echo(
|
||||
f"{view.district_id}\t{view.member}\t{view.party}\t{seats}\t"
|
||||
f"providers={view.n_providers:g}\tsites={view.n_sites:g}\t"
|
||||
f"services={view.services:g}\tpayment={view.payment:g}"
|
||||
)
|
||||
|
||||
|
||||
@app.command()
|
||||
def gaps() -> None:
|
||||
"""Sources that could not be read (market.source_gap)."""
|
||||
|
||||
431
src/market/footprint.py
Normal file
431
src/market/footprint.py
Normal file
@@ -0,0 +1,431 @@
|
||||
"""Where exposed companies and a code family's billers sit, by district.
|
||||
|
||||
Sites come from the CMS All Owners snapshot of 2026-09-29 joined to
|
||||
enrollment ZIPs, and from NPPES organization rows. Family billers come from
|
||||
the 2024 Medicare physician-and-other-practitioners file. Amounts split
|
||||
across districts by :func:`market.districts.zip_weights`. An unmatched ZIP
|
||||
is kept as ``??``. ``agi_k`` stays empty until the SOI join.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from market.districts import zip_weights
|
||||
from market.seed import norm_org
|
||||
from market.tables import (
|
||||
EntityRow,
|
||||
FootprintRow,
|
||||
SiteRow,
|
||||
read_assignments,
|
||||
read_footprints,
|
||||
read_members,
|
||||
)
|
||||
from pfs.codetables import is_missing_table_error
|
||||
|
||||
YEAR = 2024
|
||||
CONGRESS = 119
|
||||
METHOD = "zip-weight"
|
||||
OWNERS_STAMP = "2026.09.29"
|
||||
ENROLLMENT_TABLES = (
|
||||
("hospital", "hospital_enrollments"),
|
||||
("hha", "home_health_agency_enrollments"),
|
||||
("hospice", "hospice_enrollments"),
|
||||
("snf", "skilled_nursing_facility_enrollments"),
|
||||
("fqhc", "federally_qualified_health_center_enrollments"),
|
||||
("rhc", "rural_health_clinic_enrollments"),
|
||||
)
|
||||
OWNER_DIRS = (
|
||||
"cms_hospital_all_owners",
|
||||
"cms_home_health_agency_all_owners",
|
||||
"cms_hospice_all_owners",
|
||||
"cms_skilled_nursing_facility_all_owners",
|
||||
"cms_federally_qualified_health_center_all_owners",
|
||||
"cms_rural_health_clinic_all_owners",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Enrollment:
|
||||
"""One CMS enrollment the owners file can point at."""
|
||||
|
||||
enrollment_id: str
|
||||
name: str
|
||||
zip: str
|
||||
state: str
|
||||
kind: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AllocItem:
|
||||
"""One ZIP's providers, sites, services and payment before the split."""
|
||||
|
||||
zip: str
|
||||
n_providers: float
|
||||
n_sites: float
|
||||
services: float
|
||||
payment: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DistrictView:
|
||||
"""One district's footprint beside its House member. ``agi_k`` is filled
|
||||
by the SOI join."""
|
||||
|
||||
district_id: str
|
||||
bioguide_id: str
|
||||
member: str
|
||||
party: str
|
||||
health_committees: tuple[str, ...]
|
||||
n_providers: float
|
||||
n_sites: float
|
||||
services: float
|
||||
payment: float
|
||||
agi_k: float | None = None
|
||||
|
||||
|
||||
def enrichment_root() -> Path:
|
||||
return Path("data/public/lead_enrichment")
|
||||
|
||||
|
||||
def owner_files() -> list[Path]:
|
||||
"""All Owners CSVs from the 2026-09-29 snapshot."""
|
||||
root = enrichment_root()
|
||||
found: list[Path] = []
|
||||
for kind in OWNER_DIRS:
|
||||
found.extend(sorted((root / kind).glob(f"*/*{OWNERS_STAMP}*.csv")))
|
||||
return found
|
||||
|
||||
|
||||
def prov_svc_file() -> Path:
|
||||
"""The 2024 physician by-provider-and-service file."""
|
||||
return (
|
||||
enrichment_root()
|
||||
/ "cms_medicare_physician_other_practitioners_by_provider_and_service"
|
||||
/ "2024_12_01"
|
||||
/ "PHY_R26_P05_V10_D24_Prov_Svc.csv"
|
||||
)
|
||||
|
||||
|
||||
def zip5(value: str) -> str:
|
||||
digits = "".join(ch for ch in (value or "") if ch.isdigit())
|
||||
return digits[:5]
|
||||
|
||||
|
||||
def _index(entities: Sequence[EntityRow]) -> dict[str, list[EntityRow]]:
|
||||
"""Normalized alias (and the entity's own name) to the entities that
|
||||
carry it. A pseudo entity is never a match."""
|
||||
exact: dict[str, list[EntityRow]] = {}
|
||||
for entity in entities:
|
||||
if entity.pseudo:
|
||||
continue
|
||||
seen: set[str] = set()
|
||||
for raw in (entity.name, *entity.aliases.split("|")):
|
||||
alias = norm_org(raw)
|
||||
if not alias or alias in seen:
|
||||
continue
|
||||
seen.add(alias)
|
||||
exact.setdefault(alias, []).append(entity)
|
||||
return exact
|
||||
|
||||
|
||||
def _matching(name: str, exact: Mapping[str, Sequence[EntityRow]]) -> list[EntityRow]:
|
||||
"""Entities whose alias equals *name* or is a prefix of at least five
|
||||
characters plus a space. Same rule as ``alias_match(..., prefix=True)``."""
|
||||
folded = norm_org(name)
|
||||
if not folded:
|
||||
return []
|
||||
found: list[EntityRow] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(items: Sequence[EntityRow]) -> None:
|
||||
for entity in items:
|
||||
if entity.key not in seen:
|
||||
seen.add(entity.key)
|
||||
found.append(entity)
|
||||
|
||||
add(exact.get(folded, ()))
|
||||
lead = ""
|
||||
parts = folded.split(" ")
|
||||
for part in parts[:-1]:
|
||||
lead = part if not lead else f"{lead} {part}"
|
||||
if len(lead) >= 5:
|
||||
add(exact.get(lead, ()))
|
||||
return found
|
||||
|
||||
|
||||
def owner_sites(
|
||||
owners_csv: Path,
|
||||
enrollments: Mapping[str, Enrollment],
|
||||
entities: Sequence[EntityRow],
|
||||
) -> list[SiteRow]:
|
||||
"""One site per entity and enrollment whose owner name matches."""
|
||||
exact = _index(entities)
|
||||
chosen: dict[tuple[str, str], SiteRow] = {}
|
||||
with Path(owners_csv).open(newline="", encoding="cp1252") as fh:
|
||||
for rec in csv.DictReader(fh):
|
||||
owner = (rec.get("ORGANIZATION NAME - OWNER") or "").strip()
|
||||
enrollment_id = (rec.get("ENROLLMENT ID") or "").strip()
|
||||
if not owner or not enrollment_id:
|
||||
continue
|
||||
enrollment = enrollments.get(enrollment_id)
|
||||
if enrollment is None:
|
||||
continue
|
||||
role = (rec.get("ROLE TEXT - OWNER") or "").strip()
|
||||
for entity in _matching(owner, exact):
|
||||
key = (entity.key, enrollment_id)
|
||||
if key in chosen:
|
||||
continue
|
||||
chosen[key] = SiteRow(
|
||||
entity.key,
|
||||
enrollment.kind,
|
||||
enrollment_id,
|
||||
enrollment.name,
|
||||
zip5(enrollment.zip),
|
||||
enrollment.state,
|
||||
role,
|
||||
"cms-owners",
|
||||
str(owners_csv),
|
||||
)
|
||||
return sorted(chosen.values(), key=lambda r: (r.entity_key, r.site_id))
|
||||
|
||||
|
||||
def npi_sites(
|
||||
rows: Iterable[Mapping[str, Any]], entities: Sequence[EntityRow]
|
||||
) -> list[SiteRow]:
|
||||
"""Organization NPIs (entity type 2) whose parent or own name matches."""
|
||||
exact = _index(entities)
|
||||
out: list[SiteRow] = []
|
||||
for rec in rows:
|
||||
if str(rec.get("entity_type_code") or "") != "2":
|
||||
continue
|
||||
npi = str(rec.get("npi") or "").strip()
|
||||
if not npi:
|
||||
continue
|
||||
hits: list[EntityRow] = []
|
||||
seen: set[str] = set()
|
||||
for field in ("parent_organization_name", "provider_organization_name"):
|
||||
for entity in _matching(str(rec.get(field) or ""), exact):
|
||||
if entity.key not in seen:
|
||||
seen.add(entity.key)
|
||||
hits.append(entity)
|
||||
if not hits:
|
||||
continue
|
||||
label = str(
|
||||
rec.get("provider_organization_name")
|
||||
or rec.get("parent_organization_name")
|
||||
or ""
|
||||
)
|
||||
z = zip5(str(rec.get("practice_zip_code") or ""))
|
||||
state = str(rec.get("practice_state") or "")
|
||||
for entity in hits:
|
||||
out.append(
|
||||
SiteRow(
|
||||
entity.key,
|
||||
"org_npi",
|
||||
npi,
|
||||
label,
|
||||
z,
|
||||
state,
|
||||
"",
|
||||
"nppes",
|
||||
"terminology.provider",
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def family_billers(
|
||||
con: Any, csv_path: Path, codes: Sequence[str]
|
||||
) -> list[tuple[str, str, float, float]]:
|
||||
"""``(npi, zip5, services, payment)`` for *codes*. Payment is
|
||||
``Tot_Srvcs × Avg_Mdcr_Pymt_Amt``, summed when one NPI bills several."""
|
||||
wanted = [c for c in codes if c]
|
||||
if not wanted:
|
||||
return []
|
||||
marks = ", ".join("?" * len(wanted))
|
||||
fetched = con.execute(
|
||||
f"""
|
||||
SELECT Rndrng_NPI,
|
||||
Rndrng_Prvdr_Zip5,
|
||||
sum(try_cast(Tot_Srvcs AS DOUBLE)),
|
||||
sum(
|
||||
try_cast(Tot_Srvcs AS DOUBLE)
|
||||
* try_cast(Avg_Mdcr_Pymt_Amt AS DOUBLE)
|
||||
)
|
||||
FROM read_csv(?, header = true, all_varchar = true)
|
||||
WHERE HCPCS_Cd IN ({marks}) AND Rndrng_NPI <> ''
|
||||
GROUP BY 1, 2
|
||||
""",
|
||||
[str(csv_path), *wanted],
|
||||
).fetchall()
|
||||
out: list[tuple[str, str, float, float]] = []
|
||||
for npi, zipcode, services, payment in fetched:
|
||||
if services is None or payment is None:
|
||||
continue
|
||||
out.append(
|
||||
(str(npi), zip5(str(zipcode or "")), float(services), float(payment))
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def rollup(
|
||||
items: Sequence[AllocItem],
|
||||
weights: Mapping[str, Sequence[tuple[str, float]]],
|
||||
*,
|
||||
level: str,
|
||||
key: str,
|
||||
congress: int,
|
||||
year: int,
|
||||
method: str,
|
||||
) -> list[FootprintRow]:
|
||||
"""Split each item by its ZIP's district weights. A ZIP with no weight
|
||||
lands whole on ``??``."""
|
||||
source = "prov-svc" if level == "family" else "entity-site"
|
||||
acc: dict[str, list[float]] = {}
|
||||
for item in items:
|
||||
pairs = list(weights.get(item.zip) or ())
|
||||
if not pairs:
|
||||
pairs = [("??", 1.0)]
|
||||
for district, weight in pairs:
|
||||
bucket = acc.setdefault(district, [0.0, 0.0, 0.0, 0.0])
|
||||
bucket[0] += item.n_providers * weight
|
||||
bucket[1] += item.n_sites * weight
|
||||
bucket[2] += item.services * weight
|
||||
bucket[3] += item.payment * weight
|
||||
return [
|
||||
FootprintRow(level, key, congress, district, year, *vals, method, source)
|
||||
for district, vals in sorted(acc.items())
|
||||
]
|
||||
|
||||
|
||||
def district_view(
|
||||
con: Any, *, congress: int, level: str, key: str, year: int
|
||||
) -> list[DistrictView]:
|
||||
"""Footprint rows for one key joined to that congress's House member and
|
||||
health-committee seats."""
|
||||
from market.congress import HEALTH_COMMITTEES
|
||||
|
||||
totals: dict[str, list[float]] = {}
|
||||
for row in read_footprints(con):
|
||||
if (
|
||||
row.level != level
|
||||
or row.key != key
|
||||
or row.congress != congress
|
||||
or row.year != year
|
||||
):
|
||||
continue
|
||||
bucket = totals.setdefault(row.district_id, [0.0, 0.0, 0.0, 0.0])
|
||||
bucket[0] += row.n_providers
|
||||
bucket[1] += row.n_sites
|
||||
bucket[2] += row.services
|
||||
bucket[3] += row.payment
|
||||
members = {
|
||||
row.district_id: row
|
||||
for row in read_members(con)
|
||||
if row.congress == congress and row.chamber == "house" and row.district_id
|
||||
}
|
||||
committees: dict[str, set[str]] = {}
|
||||
for row in read_assignments(con):
|
||||
if row.congress != congress or row.committee_id not in HEALTH_COMMITTEES:
|
||||
continue
|
||||
committees.setdefault(row.bioguide_id, set()).add(row.committee_id)
|
||||
views: list[DistrictView] = []
|
||||
for district, (providers, sites, services, payment) in totals.items():
|
||||
member = members.get(district)
|
||||
codes = tuple(sorted(committees.get(member.bioguide_id, ()))) if member else ()
|
||||
views.append(
|
||||
DistrictView(
|
||||
district,
|
||||
member.bioguide_id if member else "",
|
||||
member.name if member else "",
|
||||
member.party if member else "",
|
||||
codes,
|
||||
providers,
|
||||
sites,
|
||||
services,
|
||||
payment,
|
||||
None,
|
||||
)
|
||||
)
|
||||
views.sort(key=lambda v: (-v.payment, v.district_id))
|
||||
return views
|
||||
|
||||
|
||||
def _missing(exc: Exception) -> bool:
|
||||
return is_missing_table_error(exc)
|
||||
|
||||
|
||||
def family_codes(con: Any, key: str) -> list[str]:
|
||||
try:
|
||||
rows = con.execute(
|
||||
"SELECT code FROM pfs.code_family WHERE key = ? ORDER BY code", [key]
|
||||
).fetchall()
|
||||
except Exception as exc:
|
||||
if _missing(exc):
|
||||
return []
|
||||
raise
|
||||
return [row[0] for row in rows]
|
||||
|
||||
|
||||
def load_enrollments(con: Any) -> dict[str, Enrollment]:
|
||||
"""Enrollment id to facility. Ids do not collide across these tables."""
|
||||
out: dict[str, Enrollment] = {}
|
||||
for kind, table in ENROLLMENT_TABLES:
|
||||
try:
|
||||
rows = con.execute(
|
||||
f"SELECT enrollment_id, organization_name, zip_code, state "
|
||||
f"FROM cms.{table}"
|
||||
).fetchall()
|
||||
except Exception as exc:
|
||||
if _missing(exc):
|
||||
continue
|
||||
raise
|
||||
for enrollment_id, name, zipcode, state in rows:
|
||||
if not enrollment_id:
|
||||
continue
|
||||
out[str(enrollment_id)] = Enrollment(
|
||||
str(enrollment_id),
|
||||
str(name or ""),
|
||||
str(zipcode or ""),
|
||||
str(state or ""),
|
||||
kind,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def org_provider_rows(con: Any, entities: Sequence[EntityRow]) -> list[dict[str, Any]]:
|
||||
"""Type-2 NPPES rows. *entities* selects nothing here; the matcher does."""
|
||||
del entities
|
||||
try:
|
||||
rows = con.execute(
|
||||
"SELECT npi, entity_type_code, provider_organization_name, "
|
||||
"parent_organization_name, practice_zip_code, practice_state "
|
||||
"FROM terminology.provider WHERE entity_type_code = '2'"
|
||||
).fetchall()
|
||||
except Exception as exc:
|
||||
if _missing(exc):
|
||||
return []
|
||||
raise
|
||||
cols = (
|
||||
"npi",
|
||||
"entity_type_code",
|
||||
"provider_organization_name",
|
||||
"parent_organization_name",
|
||||
"practice_zip_code",
|
||||
"practice_state",
|
||||
)
|
||||
return [dict(zip(cols, row, strict=True)) for row in rows]
|
||||
|
||||
|
||||
def crosswalk(con: Any, congress: int) -> dict[str, list[tuple[str, float]]]:
|
||||
try:
|
||||
return zip_weights(con, congress)
|
||||
except Exception as exc:
|
||||
if _missing(exc):
|
||||
return {}
|
||||
raise
|
||||
@@ -115,6 +115,15 @@ CREATE TABLE IF NOT EXISTS market.zip_district (
|
||||
CREATE TABLE IF NOT EXISTS market.county_district (
|
||||
congress INTEGER, county_fips VARCHAR, district_id VARCHAR, weight DOUBLE,
|
||||
source_file VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.entity_site (
|
||||
entity_key VARCHAR, site_kind VARCHAR, site_id VARCHAR, name VARCHAR,
|
||||
zip VARCHAR, state VARCHAR, note VARCHAR, source VARCHAR, source_file VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.footprint (
|
||||
level VARCHAR, key VARCHAR, congress INTEGER, district_id VARCHAR,
|
||||
year INTEGER, n_providers DOUBLE, n_sites DOUBLE, services DOUBLE,
|
||||
payment DOUBLE, method VARCHAR, source VARCHAR
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -472,6 +481,39 @@ class CountyDistrictRow:
|
||||
source_file: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SiteRow:
|
||||
"""One facility or organization NPI tied to a market entity."""
|
||||
|
||||
entity_key: str
|
||||
site_kind: str
|
||||
site_id: str
|
||||
name: str
|
||||
zip: str
|
||||
state: str
|
||||
note: str
|
||||
source: str
|
||||
source_file: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FootprintRow:
|
||||
"""Providers, sites, services and payment allocated to one district.
|
||||
``district_id`` is ``??`` when the ZIP has no crosswalk weight."""
|
||||
|
||||
level: str
|
||||
key: str
|
||||
congress: int
|
||||
district_id: str
|
||||
year: int
|
||||
n_providers: float
|
||||
n_sites: float
|
||||
services: float
|
||||
payment: float
|
||||
method: str
|
||||
source: str
|
||||
|
||||
|
||||
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
|
||||
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
|
||||
a statement per row, far too slow for filing lists)."""
|
||||
@@ -1054,6 +1096,46 @@ def read_county_districts(con: Any) -> list[CountyDistrictRow]:
|
||||
)
|
||||
|
||||
|
||||
def write_entity_sites(con: Any, rows: Sequence[SiteRow]) -> int:
|
||||
"""Replace every stored site. An empty batch deletes nothing."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
con.execute("DELETE FROM market.entity_site")
|
||||
return _insert(con, "market.entity_site", rows)
|
||||
|
||||
|
||||
def read_entity_sites(con: Any) -> list[SiteRow]:
|
||||
rows = _select(con, SiteRow, "market.entity_site")
|
||||
return sorted(
|
||||
(SiteRow(*r) for r in rows),
|
||||
key=lambda r: (r.entity_key, r.site_kind, r.site_id),
|
||||
)
|
||||
|
||||
|
||||
def write_footprints(con: Any, rows: Sequence[FootprintRow]) -> int:
|
||||
"""Replace each ``(level, key, congress)`` present in *rows*."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
keys = sorted({(r.level, r.key, r.congress) for r in rows})
|
||||
for part in _chunks(keys, 400):
|
||||
con.execute(
|
||||
"DELETE FROM market.footprint WHERE (level, key, congress) IN "
|
||||
"(SELECT * FROM (VALUES " + ", ".join(["(?, ?, ?)"] * len(part)) + "))",
|
||||
[v for key in part for v in key],
|
||||
)
|
||||
return _insert(con, "market.footprint", rows)
|
||||
|
||||
|
||||
def read_footprints(con: Any) -> list[FootprintRow]:
|
||||
rows = _select(con, FootprintRow, "market.footprint")
|
||||
return sorted(
|
||||
(FootprintRow(*r) for r in rows),
|
||||
key=lambda r: (r.level, r.key, r.congress, r.district_id),
|
||||
)
|
||||
|
||||
|
||||
# ── districts ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -2771,6 +2771,285 @@ def test_hud_cli_write_from_cache(crosswalk):
|
||||
assert {r.method for r in T.read_zip_districts(c)} == {"hud-res", "hud-bus"}
|
||||
|
||||
|
||||
def test_footprint_cli_missing_crosswalk_says_run_districts(env):
|
||||
(
|
||||
_,
|
||||
_,
|
||||
published,
|
||||
_,
|
||||
) = env
|
||||
res = runner.invoke(
|
||||
app, ["market", "footprint", "--family", "HOME-VISITS", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stack market districts" in res.output
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_footprint_cli_entities_no_write(env, monkeypatch):
|
||||
from market import footprint as footprint_api
|
||||
|
||||
c, _, published, _ = env
|
||||
c.execute("CREATE SCHEMA IF NOT EXISTS cms")
|
||||
c.execute(
|
||||
"CREATE TABLE cms.home_health_agency_enrollments ("
|
||||
"enrollment_id VARCHAR, organization_name VARCHAR, zip_code VARCHAR, "
|
||||
"state VARCHAR)"
|
||||
)
|
||||
c.execute(
|
||||
"INSERT INTO cms.home_health_agency_enrollments VALUES "
|
||||
"('E1', 'Amedisys Home Health', '36009', 'AL'), "
|
||||
"('E2', 'Amedisys Hospice', '48201', 'MI')"
|
||||
)
|
||||
T.write_entities(
|
||||
c,
|
||||
[T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")],
|
||||
)
|
||||
T.write_zip_districts(
|
||||
c,
|
||||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
footprint_api,
|
||||
"owner_files",
|
||||
lambda: [FX / "hha_all_owners_head.csv"],
|
||||
)
|
||||
monkeypatch.setattr(footprint_api, "org_provider_rows", lambda con, entities: [])
|
||||
res = runner.invoke(app, ["market", "footprint", "--entities", "--congress", "119"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "2 sites" in res.output
|
||||
assert "dry run: pass --write" in res.output
|
||||
assert _count(c, "market.entity_site") == 0
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_footprint_cli_family_offline_write(env, monkeypatch):
|
||||
from market import footprint as footprint_api
|
||||
|
||||
c, _, published, _ = env
|
||||
T.write_zip_districts(
|
||||
c,
|
||||
[
|
||||
T.ZipDistrictRow(
|
||||
119, "36009", "AL-01", 0.75, "census-zcta-land", "2020", "c"
|
||||
),
|
||||
T.ZipDistrictRow(
|
||||
119, "36009", "AL-02", 0.25, "census-zcta-land", "2020", "c"
|
||||
),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
footprint_api, "prov_svc_file", lambda: FX / "prov_svc_head.csv"
|
||||
)
|
||||
monkeypatch.setattr(footprint_api, "family_codes", lambda con, key: ["99343"])
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"market",
|
||||
"footprint",
|
||||
"--family",
|
||||
"HOME-VISITS",
|
||||
"--congress",
|
||||
"119",
|
||||
"--write",
|
||||
],
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stored:" in res.output
|
||||
rows = T.read_footprints(c)
|
||||
assert {r.district_id for r in rows} >= {"AL-01", "AL-02", "??"}
|
||||
assert published == [True]
|
||||
|
||||
|
||||
def test_district_view_cli_prints(env):
|
||||
c, _, _, _ = env
|
||||
T.write_members(
|
||||
c,
|
||||
[
|
||||
T.MemberRow(
|
||||
119,
|
||||
"T000481",
|
||||
"Tlaib, Rashida",
|
||||
"Democratic",
|
||||
"MI",
|
||||
"MI-12",
|
||||
"house",
|
||||
"",
|
||||
"congress",
|
||||
)
|
||||
],
|
||||
)
|
||||
T.write_assignments(
|
||||
c,
|
||||
[
|
||||
T.AssignmentRow(
|
||||
119, date(2025, 4, 1), "HSWM02", "T000481", 1, "", "D", "sha"
|
||||
)
|
||||
],
|
||||
)
|
||||
T.write_footprints(
|
||||
c,
|
||||
[
|
||||
T.FootprintRow(
|
||||
"family",
|
||||
"HOME-VISITS",
|
||||
119,
|
||||
"MI-12",
|
||||
2024,
|
||||
1,
|
||||
0,
|
||||
12,
|
||||
75,
|
||||
"census-zcta-land",
|
||||
"prov-svc",
|
||||
)
|
||||
],
|
||||
)
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"market",
|
||||
"district-view",
|
||||
"--family",
|
||||
"HOME-VISITS",
|
||||
"--congress",
|
||||
"119",
|
||||
"--top",
|
||||
"10",
|
||||
],
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "MI-12" in res.output
|
||||
assert "Tlaib, Rashida" in res.output
|
||||
assert "HSWM02" in res.output
|
||||
|
||||
|
||||
def test_footprint_cli_rejects_both_switches(env):
|
||||
res = runner.invoke(
|
||||
app, ["market", "footprint", "--entities", "--family", "HOME-VISITS"]
|
||||
)
|
||||
assert res.exit_code == 2
|
||||
assert "pass exactly one of --entities or --family" in res.output
|
||||
|
||||
|
||||
def test_footprint_cli_family_without_codes_writes_nothing(env, monkeypatch):
|
||||
from market import footprint as footprint_api
|
||||
|
||||
c, _, published, _ = env
|
||||
T.write_zip_districts(
|
||||
c,
|
||||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||||
)
|
||||
monkeypatch.setattr(footprint_api, "family_codes", lambda con, key: [])
|
||||
res = runner.invoke(
|
||||
app, ["market", "footprint", "--family", "HOME-VISITS", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "no codes for family HOME-VISITS" in res.output
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_footprint_cli_entities_write(env, monkeypatch):
|
||||
from market import footprint as footprint_api
|
||||
|
||||
c, _, published, _ = env
|
||||
c.execute("CREATE SCHEMA IF NOT EXISTS cms")
|
||||
c.execute(
|
||||
"CREATE TABLE cms.home_health_agency_enrollments ("
|
||||
"enrollment_id VARCHAR, organization_name VARCHAR, zip_code VARCHAR, "
|
||||
"state VARCHAR)"
|
||||
)
|
||||
c.execute(
|
||||
"INSERT INTO cms.home_health_agency_enrollments VALUES "
|
||||
"('E1', 'Amedisys Home Health', '36009', 'AL'), "
|
||||
"('E2', 'Amedisys Hospice', '48201', 'MI')"
|
||||
)
|
||||
T.write_entities(
|
||||
c,
|
||||
[T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")],
|
||||
)
|
||||
T.write_zip_districts(
|
||||
c,
|
||||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
footprint_api, "owner_files", lambda: [FX / "hha_all_owners_head.csv"]
|
||||
)
|
||||
monkeypatch.setattr(footprint_api, "org_provider_rows", lambda con, entities: [])
|
||||
res = runner.invoke(
|
||||
app, ["market", "footprint", "--entities", "--congress", "119", "--write"]
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stored:" in res.output
|
||||
assert _count(c, "market.entity_site") == 2
|
||||
assert {r.district_id for r in T.read_footprints(c)} >= {"AL-01", "??"}
|
||||
assert published == [True]
|
||||
|
||||
|
||||
def test_footprint_cli_counts_an_npi_site(env, monkeypatch):
|
||||
from market import footprint as footprint_api
|
||||
|
||||
c, _, published, _ = env
|
||||
T.write_entities(
|
||||
c,
|
||||
[T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")],
|
||||
)
|
||||
T.write_zip_districts(
|
||||
c,
|
||||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||||
)
|
||||
monkeypatch.setattr(footprint_api, "owner_files", lambda: [])
|
||||
monkeypatch.setattr(
|
||||
footprint_api,
|
||||
"org_provider_rows",
|
||||
lambda con, entities: [
|
||||
{
|
||||
"npi": "1234567890",
|
||||
"entity_type_code": "2",
|
||||
"provider_organization_name": "Clinic",
|
||||
"parent_organization_name": "AMEDISYS HOLDING LLC",
|
||||
"practice_zip_code": "36009",
|
||||
"practice_state": "AL",
|
||||
}
|
||||
],
|
||||
)
|
||||
res = runner.invoke(app, ["market", "footprint", "--entities", "--congress", "119"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "1 sites" in res.output
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_footprint_cli_empty_family_reports_zero_unmatched(env, monkeypatch):
|
||||
from market import footprint as footprint_api
|
||||
|
||||
c, _, published, _ = env
|
||||
T.write_zip_districts(
|
||||
c,
|
||||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||||
)
|
||||
monkeypatch.setattr(footprint_api, "family_codes", lambda con, key: ["99343"])
|
||||
monkeypatch.setattr(footprint_api, "family_billers", lambda con, path, codes: [])
|
||||
res = runner.invoke(app, ["market", "footprint", "--family", "HOME-VISITS"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "0 billers" in res.output
|
||||
assert "unmatched ??: 0" in res.output
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_district_view_cli_empty(env):
|
||||
c, _, _, _ = env
|
||||
T.write_footprints(c, [])
|
||||
res = runner.invoke(app, ["market", "district-view", "--family", "HOME-VISITS"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "no footprint stored" in res.output
|
||||
|
||||
|
||||
def test_district_view_cli_needs_a_key(env):
|
||||
res = runner.invoke(app, ["market", "district-view"])
|
||||
assert res.exit_code == 2
|
||||
assert "pass exactly one of --entity or --family" in res.output
|
||||
|
||||
|
||||
def test_committees_cli_from_after_to(congress):
|
||||
res = runner.invoke(app, ["market", "committees", "--from", "119", "--to", "113"])
|
||||
assert res.exit_code == 2
|
||||
|
||||
6
tests/market/fixtures/hha_all_owners_head.csv
Normal file
6
tests/market/fixtures/hha_all_owners_head.csv
Normal file
@@ -0,0 +1,6 @@
|
||||
"ENROLLMENT ID","ASSOCIATE ID","ORGANIZATION NAME","ASSOCIATE ID - OWNER","TYPE - OWNER","ROLE CODE - OWNER","ROLE TEXT - OWNER","ASSOCIATION DATE - OWNER","FIRST NAME - OWNER","MIDDLE NAME - OWNER","LAST NAME - OWNER","TITLE - OWNER","ORGANIZATION NAME - OWNER","DOING BUSINESS AS NAME - OWNER","ADDRESS LINE 1 - OWNER","ADDRESS LINE 2 - OWNER","CITY - OWNER","STATE - OWNER","ZIP CODE - OWNER","PERCENTAGE OWNERSHIP","CREATED FOR ACQUISITION - OWNER","CORPORATION - OWNER","LLC - OWNER","MEDICAL PROVIDER SUPPLIER - OWNER","MANAGEMENT SERVICES COMPANY - OWNER","MEDICAL STAFFING COMPANY - OWNER","HOLDING COMPANY - OWNER","INVESTMENT FIRM - OWNER","FINANCIAL INSTITUTION - OWNER","CONSULTING FIRM - OWNER","FOR PROFIT - OWNER","NON PROFIT - OWNER","PRIVATE EQUITY COMPANY - OWNER","REIT - OWNER","CHAIN HOME OFFICE - OWNER","OTHER TYPE - OWNER","OTHER TYPE TEXT - OWNER","OWNED BY ANOTHER ORG OR IND - OWNER"
|
||||
"E1","","","","","","OPERATIONAL/MANAGERIAL CONTROL","","","","","","AMEDISYS INC","","","","","","","100","","","","","","","","","","","","","","","","","",""
|
||||
"E1","","","","","","5% OR MORE OWNERSHIP INTEREST","","","","","","AMEDISYS HOLDING LLC","","","","","","","100","","","","","","","","","","","","","","","","","",""
|
||||
"E2","","","","","","OPERATIONAL/MANAGERIAL CONTROL","","","","","","AMEDISYS INC","","","","","","","50","","","","","","","","","","","","","","","","","",""
|
||||
"E3","","","","","","5% OR MORE OWNERSHIP INTEREST","","","","","","HCA SOMETHING LLC","","","","","","","10","","","","","","","","","","","","","","","","","",""
|
||||
"E4","","","","","","INDIVIDUAL","","","","","","","","","","","","","0","","","","","","","","","","","","","","","","","",""
|
||||
|
6
tests/market/fixtures/prov_svc_head.csv
Normal file
6
tests/market/fixtures/prov_svc_head.csv
Normal file
@@ -0,0 +1,6 @@
|
||||
Rndrng_NPI,Rndrng_Prvdr_Last_Org_Name,Rndrng_Prvdr_First_Name,Rndrng_Prvdr_MI,Rndrng_Prvdr_Crdntls,Rndrng_Prvdr_Ent_Cd,Rndrng_Prvdr_St1,Rndrng_Prvdr_St2,Rndrng_Prvdr_City,Rndrng_Prvdr_State_Abrvtn,Rndrng_Prvdr_State_FIPS,Rndrng_Prvdr_Zip5,Rndrng_Prvdr_RUCA,Rndrng_Prvdr_RUCA_Desc,Rndrng_Prvdr_Cntry,Rndrng_Prvdr_Type,Rndrng_Prvdr_Mdcr_Prtcptg_Ind,HCPCS_Cd,HCPCS_Desc,HCPCS_Drug_Ind,Place_Of_Srvc,Tot_Benes,Tot_Srvcs,Tot_Bene_Day_Srvcs,Avg_Sbmtd_Chrg,Avg_Mdcr_Alowd_Amt,Avg_Mdcr_Pymt_Amt,Avg_Mdcr_Stdzd_Amt
|
||||
1111111111,,,,,,,,,,,36009,,,,,,99343,,,,,12,,,,85.5,
|
||||
1111111111,,,,,,,,,,,36009,,,,,,99213,,,,,99,,,,10,
|
||||
2222222222,,,,,,,,,,,99999,,,,,,99343,,,,,4,,,,10,
|
||||
3333333333,,,,,,,,,,,48201,,,,,,99344,,,,,,,,,1,
|
||||
,,,,,,,,,,,36009,,,,,,99343,,,,,1,,,,1,
|
||||
|
390
tests/market/test_footprint.py
Normal file
390
tests/market/test_footprint.py
Normal file
@@ -0,0 +1,390 @@
|
||||
"""Company and family-biller footprints rolled onto a congressional district."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import pytest
|
||||
|
||||
from market import footprint as F
|
||||
from market import tables as T
|
||||
|
||||
FX = Path(__file__).parent / "fixtures"
|
||||
AMED = T.EntityRow("0000896262", "AMEDISYS INC", "company", aliases="AMEDISYS")
|
||||
HCA = T.EntityRow(
|
||||
"0000860730", "HCA Healthcare, Inc.", "company", aliases="HCA HEALTHCARE|HCA"
|
||||
)
|
||||
PSEUDO = T.EntityRow(
|
||||
"pseudo", "AMEDISYS", "organization", aliases="AMEDISYS", pseudo=True
|
||||
)
|
||||
|
||||
|
||||
def test_owner_sites_from_owners_csv():
|
||||
enrollments = {
|
||||
"E1": F.Enrollment("E1", "Amedisys Home Health", "36009-1234", "AL", "hha"),
|
||||
"E2": F.Enrollment("E2", "Amedisys Hospice", "48201", "MI", "hha"),
|
||||
"E3": F.Enrollment("E3", "Other", "10001", "NY", "hha"),
|
||||
}
|
||||
rows = F.owner_sites(FX / "hha_all_owners_head.csv", enrollments, [AMED, PSEUDO])
|
||||
assert [(r.entity_key, r.site_id, r.zip, r.state) for r in rows] == [
|
||||
("0000896262", "E1", "36009", "AL"),
|
||||
("0000896262", "E2", "48201", "MI"),
|
||||
]
|
||||
assert {r.site_kind for r in rows} == {"hha"}
|
||||
assert {r.source for r in rows} == {"cms-owners"}
|
||||
|
||||
|
||||
def test_owner_sites_reads_windows_1252(tmp_path):
|
||||
"""The 2026-09-29 All Owners files are Windows-1252 (en dash is 0x96)."""
|
||||
src = (FX / "hha_all_owners_head.csv").read_bytes()
|
||||
raw = src.replace(b"OPERATIONAL/MANAGERIAL CONTROL", b"OPERATIONAL\x96CONTROL", 1)
|
||||
assert b"\x96" in raw
|
||||
dest = tmp_path / "owners.csv"
|
||||
dest.write_bytes(raw)
|
||||
enrollments = {
|
||||
"E1": F.Enrollment("E1", "Amedisys Home Health", "36009", "AL", "hha"),
|
||||
"E2": F.Enrollment("E2", "Amedisys Hospice", "48201", "MI", "hha"),
|
||||
}
|
||||
rows = F.owner_sites(dest, enrollments, [AMED])
|
||||
assert [r.site_id for r in rows] == ["E1", "E2"]
|
||||
assert "\u2013" in rows[0].note
|
||||
|
||||
|
||||
def test_owner_sites_short_alias_needs_long_form(tmp_path):
|
||||
"""``HCA`` is under five characters, so it does not prefix-match.
|
||||
``HCA HEALTHCARE`` does."""
|
||||
import csv
|
||||
|
||||
src = FX / "hha_all_owners_head.csv"
|
||||
with src.open(newline="") as fh:
|
||||
header = next(csv.reader(fh))
|
||||
body = list(csv.reader(fh))
|
||||
owner_i = header.index("ORGANIZATION NAME - OWNER")
|
||||
id_i = header.index("ENROLLMENT ID")
|
||||
kept = []
|
||||
for rec in body:
|
||||
if rec[id_i] == "E3":
|
||||
kept.append(rec)
|
||||
elif rec[id_i] == "E1":
|
||||
rec = list(rec)
|
||||
rec[id_i] = "E5"
|
||||
rec[owner_i] = "HCA HEALTHCARE INC"
|
||||
kept.append(rec)
|
||||
dest = tmp_path / "owners.csv"
|
||||
with dest.open("w", newline="") as fh:
|
||||
csv.writer(fh, quoting=csv.QUOTE_ALL).writerows([header, *kept])
|
||||
enrollments = {
|
||||
"E3": F.Enrollment("E3", "HCA Something", "10001", "NY", "hospital"),
|
||||
"E5": F.Enrollment("E5", "HCA Healthcare", "30301", "GA", "hospital"),
|
||||
}
|
||||
rows = F.owner_sites(dest, enrollments, [HCA])
|
||||
assert [(r.site_id, r.entity_key) for r in rows] == [("E5", "0000860730")]
|
||||
|
||||
|
||||
def test_npi_sites_from_nppes():
|
||||
rows = F.npi_sites(
|
||||
[
|
||||
{
|
||||
"npi": "1234567890",
|
||||
"entity_type_code": "2",
|
||||
"provider_organization_name": "OTHER CLINIC",
|
||||
"parent_organization_name": "AMEDISYS HOLDING LLC",
|
||||
"practice_zip_code": "360091234",
|
||||
"practice_state": "AL",
|
||||
},
|
||||
{
|
||||
"npi": "9999999999",
|
||||
"entity_type_code": "2",
|
||||
"provider_organization_name": "WALGREENS",
|
||||
"parent_organization_name": "",
|
||||
"practice_zip_code": "10001",
|
||||
"practice_state": "NY",
|
||||
},
|
||||
{
|
||||
"npi": "1111111111",
|
||||
"entity_type_code": "1",
|
||||
"provider_organization_name": "",
|
||||
"parent_organization_name": "AMEDISYS HOLDING LLC",
|
||||
"practice_zip_code": "36009",
|
||||
"practice_state": "AL",
|
||||
},
|
||||
],
|
||||
[AMED],
|
||||
)
|
||||
assert len(rows) == 1
|
||||
assert rows[0].site_kind == "org_npi"
|
||||
assert rows[0].site_id == "1234567890"
|
||||
assert rows[0].zip == "36009"
|
||||
assert rows[0].source == "nppes"
|
||||
|
||||
|
||||
def test_family_billers_from_prov_svc():
|
||||
con = duckdb.connect(":memory:")
|
||||
rows = F.family_billers(con, FX / "prov_svc_head.csv", ["99343"])
|
||||
con.close()
|
||||
assert ("1111111111", "36009", 12.0, pytest.approx(12 * 85.5)) in rows
|
||||
assert all(npi for npi, *_ in rows)
|
||||
|
||||
|
||||
def test_rollup_fractional_and_unmatched_kept():
|
||||
items = [
|
||||
F.AllocItem("36009", 0, 1, 100, 100),
|
||||
F.AllocItem("00000", 0, 1, 5, 5),
|
||||
]
|
||||
weights = {"36009": [("AL-01", 0.75), ("AL-02", 0.25)]}
|
||||
rows = F.rollup(
|
||||
items,
|
||||
weights,
|
||||
level="entity",
|
||||
key="0000896262",
|
||||
congress=119,
|
||||
year=2024,
|
||||
method="census-zcta-land",
|
||||
)
|
||||
by = {r.district_id: r for r in rows}
|
||||
assert by["AL-01"].payment == 75
|
||||
assert by["AL-02"].payment == 25
|
||||
assert by["AL-01"].n_sites == pytest.approx(0.75)
|
||||
assert by["??"].payment == 5
|
||||
assert by["??"].n_sites == 1
|
||||
assert by["??"].district_id == "??"
|
||||
|
||||
|
||||
def test_district_view_joins_member_and_committees():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
T.write_members(
|
||||
con,
|
||||
[
|
||||
T.MemberRow(
|
||||
119,
|
||||
"T000481",
|
||||
"Tlaib, Rashida",
|
||||
"Democratic",
|
||||
"MI",
|
||||
"MI-12",
|
||||
"house",
|
||||
"",
|
||||
"congress",
|
||||
)
|
||||
],
|
||||
)
|
||||
T.write_assignments(
|
||||
con,
|
||||
[
|
||||
T.AssignmentRow(
|
||||
119, date(2025, 4, 1), "HSWM02", "T000481", 1, "", "D", "sha"
|
||||
),
|
||||
T.AssignmentRow(
|
||||
119, date(2025, 4, 1), "HSHM", "T000481", 2, "", "D", "sha"
|
||||
),
|
||||
],
|
||||
)
|
||||
T.write_footprints(
|
||||
con,
|
||||
[
|
||||
T.FootprintRow(
|
||||
"family",
|
||||
"HOME-VISITS",
|
||||
119,
|
||||
"MI-12",
|
||||
2024,
|
||||
1,
|
||||
0,
|
||||
12,
|
||||
75,
|
||||
"census-zcta-land",
|
||||
"prov-svc",
|
||||
)
|
||||
],
|
||||
)
|
||||
views = F.district_view(
|
||||
con, congress=119, level="family", key="HOME-VISITS", year=2024
|
||||
)
|
||||
con.close()
|
||||
assert len(views) == 1
|
||||
assert views[0].district_id == "MI-12"
|
||||
assert views[0].member == "Tlaib, Rashida"
|
||||
assert views[0].health_committees == ("HSWM02",)
|
||||
assert views[0].agi_k is None
|
||||
assert views[0].payment == 75
|
||||
|
||||
|
||||
def test_enrichment_root_blank_npi_and_null_payment(tmp_path):
|
||||
assert F.enrichment_root().name == "lead_enrichment"
|
||||
assert (
|
||||
F.npi_sites(
|
||||
[
|
||||
{
|
||||
"npi": " ",
|
||||
"entity_type_code": "2",
|
||||
"provider_organization_name": "AMEDISYS INC",
|
||||
"parent_organization_name": "",
|
||||
"practice_zip_code": "36009",
|
||||
"practice_state": "AL",
|
||||
}
|
||||
],
|
||||
[AMED],
|
||||
)
|
||||
== []
|
||||
)
|
||||
csv_path = tmp_path / "svc.csv"
|
||||
csv_path.write_text(
|
||||
"Rndrng_NPI,Rndrng_Prvdr_Zip5,HCPCS_Cd,Tot_Srvcs,Avg_Mdcr_Pymt_Amt\n"
|
||||
"1111111111,36009,99343,,85.5\n"
|
||||
)
|
||||
con = duckdb.connect(":memory:")
|
||||
assert F.family_billers(con, csv_path, ["99343"]) == []
|
||||
con.close()
|
||||
|
||||
|
||||
def test_district_view_skips_other_keys():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
T.write_footprints(
|
||||
con,
|
||||
[
|
||||
T.FootprintRow(
|
||||
"family",
|
||||
"OTHER",
|
||||
119,
|
||||
"MI-12",
|
||||
2024,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
"zip-weight",
|
||||
"prov-svc",
|
||||
)
|
||||
],
|
||||
)
|
||||
views = F.district_view(
|
||||
con, congress=119, level="family", key="HOME-VISITS", year=2024
|
||||
)
|
||||
con.close()
|
||||
assert views == []
|
||||
|
||||
|
||||
def test_owner_files_keep_the_owners_stamp(tmp_path, monkeypatch):
|
||||
root = tmp_path / "lead"
|
||||
kind = root / "cms_home_health_agency_all_owners" / "2026_10_01"
|
||||
kind.mkdir(parents=True)
|
||||
hit = kind / "x_2026.09.29_All_Owners.csv"
|
||||
hit.write_text("a\n")
|
||||
(kind / "other.csv").write_text("b\n")
|
||||
monkeypatch.setattr(F, "enrichment_root", lambda: root)
|
||||
assert F.owner_files() == [hit]
|
||||
assert F.prov_svc_file().name == "PHY_R26_P05_V10_D24_Prov_Svc.csv"
|
||||
|
||||
|
||||
def test_loaders_missing_then_present_and_errors_propagate():
|
||||
con = duckdb.connect(":memory:")
|
||||
assert F.family_codes(con, "HOME-VISITS") == []
|
||||
assert F.load_enrollments(con) == {}
|
||||
assert F.org_provider_rows(con, []) == []
|
||||
assert F.crosswalk(con, 119) == {}
|
||||
assert F.family_billers(con, FX / "prov_svc_head.csv", []) == []
|
||||
|
||||
con.execute("CREATE SCHEMA pfs")
|
||||
con.execute("CREATE TABLE pfs.code_family (key VARCHAR, code VARCHAR)")
|
||||
con.execute("INSERT INTO pfs.code_family VALUES ('HOME-VISITS', '99343')")
|
||||
assert F.family_codes(con, "HOME-VISITS") == ["99343"]
|
||||
|
||||
con.execute("CREATE SCHEMA cms")
|
||||
con.execute(
|
||||
"CREATE TABLE cms.home_health_agency_enrollments ("
|
||||
"enrollment_id VARCHAR, organization_name VARCHAR, zip_code VARCHAR, "
|
||||
"state VARCHAR)"
|
||||
)
|
||||
con.execute(
|
||||
"INSERT INTO cms.home_health_agency_enrollments VALUES "
|
||||
"('E1', 'Amedisys', '36009', 'AL'), ('', 'blank', '1', 'AL')"
|
||||
)
|
||||
got = F.load_enrollments(con)
|
||||
assert list(got) == ["E1"]
|
||||
assert got["E1"].kind == "hha" and got["E1"].zip == "36009"
|
||||
|
||||
con.execute("CREATE SCHEMA terminology")
|
||||
con.execute(
|
||||
"CREATE TABLE terminology.provider ("
|
||||
"npi VARCHAR, entity_type_code VARCHAR, provider_organization_name VARCHAR, "
|
||||
"parent_organization_name VARCHAR, practice_zip_code VARCHAR, "
|
||||
"practice_state VARCHAR)"
|
||||
)
|
||||
con.execute(
|
||||
"INSERT INTO terminology.provider VALUES "
|
||||
"('1234567890', '2', 'Clinic', 'AMEDISYS', '36009', 'AL'), "
|
||||
"('1', '1', 'Person', '', '36009', 'AL')"
|
||||
)
|
||||
rows = F.org_provider_rows(con, [])
|
||||
assert [row["npi"] for row in rows] == ["1234567890"]
|
||||
|
||||
T.write_zip_districts(
|
||||
con,
|
||||
[T.ZipDistrictRow(119, "36009", "AL-01", 1.0, "census-zcta-land", "2020", "c")],
|
||||
)
|
||||
assert F.crosswalk(con, 119) == {"36009": [("AL-01", 1.0)]}
|
||||
|
||||
con.execute("CREATE OR REPLACE TABLE pfs.code_family (x INTEGER)")
|
||||
with pytest.raises(duckdb.Error):
|
||||
F.family_codes(con, "HOME-VISITS")
|
||||
con.execute("CREATE OR REPLACE TABLE cms.hospital_enrollments (x INTEGER)")
|
||||
with pytest.raises(duckdb.Error):
|
||||
F.load_enrollments(con)
|
||||
con.execute("CREATE OR REPLACE TABLE terminology.provider (x INTEGER)")
|
||||
with pytest.raises(duckdb.Error):
|
||||
F.org_provider_rows(con, [])
|
||||
con.execute("CREATE OR REPLACE TABLE market.zip_district (x INTEGER)")
|
||||
with pytest.raises(duckdb.Error):
|
||||
F.crosswalk(con, 119)
|
||||
con.close()
|
||||
|
||||
|
||||
def test_district_view_unmatched_has_no_member():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
T.write_footprints(
|
||||
con,
|
||||
[
|
||||
T.FootprintRow(
|
||||
"family",
|
||||
"HOME-VISITS",
|
||||
119,
|
||||
"??",
|
||||
2024,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
"zip-weight",
|
||||
"prov-svc",
|
||||
)
|
||||
],
|
||||
)
|
||||
views = F.district_view(
|
||||
con, congress=119, level="family", key="HOME-VISITS", year=2024
|
||||
)
|
||||
con.close()
|
||||
assert len(views) == 1
|
||||
assert views[0].district_id == "??"
|
||||
assert views[0].member == ""
|
||||
assert views[0].bioguide_id == ""
|
||||
assert views[0].health_committees == ()
|
||||
|
||||
|
||||
def test_rollup_family_source_is_prov_svc():
|
||||
rows = F.rollup(
|
||||
[F.AllocItem("00000", 1, 0, 4, 10)],
|
||||
{},
|
||||
level="family",
|
||||
key="HOME-VISITS",
|
||||
congress=119,
|
||||
year=2024,
|
||||
method="zip-weight",
|
||||
)
|
||||
assert rows[0].source == "prov-svc"
|
||||
assert rows[0].district_id == "??"
|
||||
assert rows[0].payment == 10
|
||||
@@ -16,6 +16,7 @@ MODULES = [
|
||||
"market.fec",
|
||||
"market.congress",
|
||||
"market.districts",
|
||||
"market.footprint",
|
||||
]
|
||||
|
||||
_PROBE = """
|
||||
|
||||
@@ -53,6 +53,8 @@ def test_ensure_tables_idempotent():
|
||||
"bill_member",
|
||||
"zip_district",
|
||||
"county_district",
|
||||
"entity_site",
|
||||
"footprint",
|
||||
}
|
||||
|
||||
|
||||
@@ -232,6 +234,30 @@ ZIP_DISTRICT = T.ZipDistrictRow(
|
||||
119, "36009", "AL-02", 0.25, "census-zcta-land", "2020", "census/zcta.txt"
|
||||
)
|
||||
COUNTY_DISTRICT = T.CountyDistrictRow(119, "01003", "AL-01", 0.5, "census/county.txt")
|
||||
ENTITY_SITE = T.SiteRow(
|
||||
"0000896262",
|
||||
"hha",
|
||||
"E1",
|
||||
"Amedisys Home Health",
|
||||
"36009",
|
||||
"AL",
|
||||
"OPERATIONAL/MANAGERIAL CONTROL",
|
||||
"cms-owners",
|
||||
"owners.csv",
|
||||
)
|
||||
FOOTPRINT = T.FootprintRow(
|
||||
"family",
|
||||
"HOME-VISITS",
|
||||
119,
|
||||
"MI-12",
|
||||
2024,
|
||||
1,
|
||||
0.25,
|
||||
12,
|
||||
75,
|
||||
"census-zcta-land",
|
||||
"prov-svc",
|
||||
)
|
||||
SERIES = TI.MonthlyRow("tiingo:OSH:tri", "OSH", "tiingo", 2021, 1, 101.5, "KEY")
|
||||
|
||||
ROUNDTRIP = {
|
||||
@@ -291,6 +317,16 @@ ROUNDTRIP = {
|
||||
lambda c, r: T.write_county_districts(c, [r]),
|
||||
lambda c: T.read_county_districts(c),
|
||||
),
|
||||
"entity_site": (
|
||||
ENTITY_SITE,
|
||||
lambda c, r: T.write_entity_sites(c, [r]),
|
||||
lambda c: T.read_entity_sites(c),
|
||||
),
|
||||
"footprint": (
|
||||
FOOTPRINT,
|
||||
lambda c, r: T.write_footprints(c, [r]),
|
||||
lambda c: T.read_footprints(c),
|
||||
),
|
||||
"entity": (
|
||||
ENTITY,
|
||||
lambda c, r: T.write_entities(c, [r]),
|
||||
@@ -359,6 +395,8 @@ def test_write_empty_is_noop(con):
|
||||
assert T.write_bill_members(con, []) == 0
|
||||
assert T.write_zip_districts(con, []) == 0
|
||||
assert T.write_county_districts(con, []) == 0
|
||||
assert T.write_entity_sites(con, []) == 0
|
||||
assert T.write_footprints(con, []) == 0
|
||||
|
||||
|
||||
def test_lda_readers_filter_by_entity(con):
|
||||
|
||||
Reference in New Issue
Block a user