f66b35f5fe
Split dev/ flat directory into dev/scripts/ (29 .py files) and dev/seeds/ (PDFs, Excel, ZIPs, NDJSON, grafana, PY2023). Update all Path(__file__) references to use parents[2] for project root and dev/seeds/ for seed data. Fix path references in src/ too. Add PY2024 CMS quality measure value sets (HWR, UAMCC, ACR) to the REGISTRY and load into DuckDB — enables 2024->2025->2026 diffs. Add HWR tables to notebook DIFF_SPECS now that two years exist.
591 lines
18 KiB
Python
591 lines
18 KiB
Python
"""Scrape all public datasets from data.cms.gov and generate SQLTable models.
|
|
|
|
Fetches the DCAT catalog at ``https://data.cms.gov/data.json``, discovers
|
|
column schemas via the ``data-viewer`` endpoint, generates Pydantic
|
|
SQLTable models under ``src/cms/table/``, and optionally downloads the
|
|
raw CSV data to ``data/cms/raw/``.
|
|
|
|
Usage::
|
|
|
|
uv run python dev/scripts/scrape_cms_data.py # generate models + download CSVs
|
|
uv run python dev/scripts/scrape_cms_data.py --no-download # models only
|
|
uv run python dev/scripts/scrape_cms_data.py --force # re-download existing CSVs
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import zipfile
|
|
from pathlib import Path
|
|
from urllib.request import Request, urlopen
|
|
|
|
CATALOG_URL = "https://data.cms.gov/data.json"
|
|
DATA_VIEWER_URL = "https://data.cms.gov/data-api/v1/dataset/{uuid}/data-viewer"
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
|
TABLE_DIR = PROJECT_ROOT / "src" / "cms" / "table"
|
|
RAW_DIR = PROJECT_ROOT / "data" / "cms" / "raw"
|
|
|
|
_STOP_WORDS = {
|
|
"a",
|
|
"an",
|
|
"and",
|
|
"for",
|
|
"in",
|
|
"of",
|
|
"on",
|
|
"the",
|
|
"to",
|
|
"with",
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Catalog + schema fetching
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _get_json(url: str) -> dict | list:
|
|
"""GET a URL and parse JSON response."""
|
|
req = Request(url, headers={"User-Agent": "cms-scraper/1.0"})
|
|
with urlopen(req, timeout=60) as resp:
|
|
return json.loads(resp.read())
|
|
|
|
|
|
def fetch_catalog() -> list[dict]:
|
|
"""Fetch the DCAT catalog and return the dataset list."""
|
|
catalog = _get_json(CATALOG_URL)
|
|
return catalog.get("dataset", [])
|
|
|
|
|
|
def extract_uuid(dataset: dict) -> str | None:
|
|
"""Extract the dataset UUID from the identifier URL."""
|
|
ident = dataset.get("identifier", "")
|
|
m = re.search(r"/dataset/([0-9a-f-]{36})/", ident)
|
|
return m.group(1) if m else None
|
|
|
|
|
|
def fetch_schema(uuid: str) -> dict[str, str] | None:
|
|
"""Fetch column types from the data-viewer endpoint.
|
|
|
|
Returns dict like ``{"ACO_ID": "TEXT", "Start_Date": "DATE"}``,
|
|
or None if the endpoint is unavailable.
|
|
"""
|
|
url = DATA_VIEWER_URL.format(uuid=uuid)
|
|
try:
|
|
data = _get_json(url)
|
|
except Exception:
|
|
return None
|
|
# csvColumnTypes lives at meta.data_file_meta_data.csvColumnTypes
|
|
meta = data.get("meta", {})
|
|
file_meta = meta.get("data_file_meta_data", {})
|
|
raw = file_meta.get("csvColumnTypes")
|
|
if not raw:
|
|
return None
|
|
if isinstance(raw, str):
|
|
return json.loads(raw)
|
|
return raw
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Name sanitization
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def sanitize_table_name(title: str) -> str:
|
|
"""Convert dataset title to a snake_case table name.
|
|
|
|
>>> sanitize_table_name("Accountable Care Organization Participants")
|
|
'aco_participants'
|
|
>>> sanitize_table_name("CMS Program Statistics - Medicare Advantage - Inpatient Hospital")
|
|
'program_statistics_medicare_advantage_inpatient_hospital'
|
|
"""
|
|
name = title.strip()
|
|
# Strip leading "CMS " prefix
|
|
name = re.sub(r"^CMS\s+", "", name)
|
|
# Remove parenthetical content like "(AHRQ)" and "(PSI-11)"
|
|
name = re.sub(r"\([^)]*\)", " ", name)
|
|
# Replace non-alphanumeric chars with spaces
|
|
name = re.sub(r"[^a-zA-Z0-9]+", " ", name)
|
|
# Split into words, drop stop words, rejoin
|
|
words = name.split()
|
|
words = [w for w in words if w.lower() not in _STOP_WORDS]
|
|
name = "_".join(words)
|
|
# Collapse multiple underscores, lowercase
|
|
name = re.sub(r"_+", "_", name).strip("_").lower()
|
|
return name
|
|
|
|
|
|
def sanitize_class_name(title: str) -> str:
|
|
"""Convert dataset title to a PascalCase class name.
|
|
|
|
>>> sanitize_class_name("Accountable Care Organization Participants")
|
|
'AcoParticipants'
|
|
"""
|
|
name = title.strip()
|
|
name = re.sub(r"^CMS\s+", "", name)
|
|
# Remove parenthetical content
|
|
name = re.sub(r"\([^)]*\)", " ", name)
|
|
# Replace non-alphanumeric chars with spaces
|
|
name = re.sub(r"[^a-zA-Z0-9]+", " ", name)
|
|
words = name.split()
|
|
words = [w for w in words if w.lower() not in _STOP_WORDS]
|
|
|
|
parts = []
|
|
for w in words:
|
|
parts.append(w.capitalize())
|
|
return "".join(parts)
|
|
|
|
|
|
def sanitize_column_name(name: str) -> str:
|
|
"""Convert API column name to a valid Python field name.
|
|
|
|
>>> sanitize_column_name("ACO_ID")
|
|
'aco_id'
|
|
>>> sanitize_column_name("SNF_3-Day_Rule_Waiver")
|
|
'snf_3_day_rule_waiver'
|
|
"""
|
|
name = name.strip()
|
|
# Replace hyphens and spaces with underscores
|
|
name = re.sub(r"[-\s]+", "_", name)
|
|
# Remove any non-ASCII and non-alphanumeric chars (except underscore)
|
|
name = re.sub(r"[^a-zA-Z0-9_]", "", name)
|
|
name = name.lower()
|
|
# Prefix with underscore if starts with digit
|
|
if name and name[0].isdigit():
|
|
name = "_" + name
|
|
# Avoid shadowing Python builtins and imported types
|
|
if name in {
|
|
"date",
|
|
"type",
|
|
"id",
|
|
"list",
|
|
"set",
|
|
"dict",
|
|
"str",
|
|
"int",
|
|
"float",
|
|
"bool",
|
|
"bytes",
|
|
"hash",
|
|
"format",
|
|
}:
|
|
name = name + "_"
|
|
return name
|
|
|
|
|
|
def map_column_type(cms_type: str) -> str:
|
|
"""Map CMS csvColumnTypes value to Python type string.
|
|
|
|
TEXT → str, NUMERIC → str (codes, phones, zips), DATE → date.
|
|
"""
|
|
cms_type = cms_type.strip().upper()
|
|
if cms_type == "DATE":
|
|
return "date"
|
|
return "str"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Code generation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def generate_table_module(
|
|
class_name: str,
|
|
table_name: str,
|
|
columns: dict[str, str],
|
|
title: str,
|
|
uuid: str,
|
|
) -> str:
|
|
"""Generate a Python module with a single SQLTable class."""
|
|
has_date = any(map_column_type(t) == "date" for t in columns.values())
|
|
|
|
lines = [
|
|
f'"""{class_name} — {title}.',
|
|
"",
|
|
"Auto-generated from CMS Data API.",
|
|
f"UUID: {uuid}",
|
|
f"Source: https://data.cms.gov/data-api/v1/dataset/{uuid}/data",
|
|
'"""',
|
|
"",
|
|
"from __future__ import annotations",
|
|
"",
|
|
]
|
|
|
|
if has_date:
|
|
lines.append("from datetime import date")
|
|
lines.append("")
|
|
|
|
lines.extend(
|
|
[
|
|
"from aco.table.base import SQLTable",
|
|
"",
|
|
"",
|
|
f"class {class_name}(SQLTable):",
|
|
f' """{title}.',
|
|
"",
|
|
f" {len(columns)} fields from data.cms.gov.",
|
|
' """',
|
|
"",
|
|
' __schema__ = "cms"',
|
|
f' __tablename__ = "{table_name}"',
|
|
]
|
|
)
|
|
|
|
for col_name, col_type in columns.items():
|
|
py_name = sanitize_column_name(col_name)
|
|
py_type = map_column_type(col_type)
|
|
lines.append("")
|
|
lines.append(f" {py_name}: {py_type} | None = None")
|
|
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def generate_init_module(
|
|
tables: list[tuple[str, str, str]],
|
|
) -> str:
|
|
"""Generate src/cms/table/__init__.py.
|
|
|
|
tables: list of (module_name, class_name, table_name) tuples.
|
|
"""
|
|
sorted_tables = sorted(tables)
|
|
lines = [
|
|
'"""CMS table models — auto-generated by dev/scripts/scrape_cms_data.py."""',
|
|
"",
|
|
]
|
|
|
|
for mod, cls, _tbl in sorted_tables:
|
|
lines.append(f"from cms.table.{mod} import {cls}")
|
|
|
|
lines.append("")
|
|
lines.append("__all__ = [")
|
|
for _mod, cls, _tbl in sorted_tables:
|
|
lines.append(f' "{cls}",')
|
|
lines.append("]")
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def generate_namespace_init(tables: list[tuple[str, str, str]]) -> str:
|
|
"""Generate src/cms/__init__.py."""
|
|
table_list = "\n".join(
|
|
f" - {cls} (cms.{tbl})" for _mod, cls, tbl in sorted(tables)
|
|
)
|
|
return f'''"""CMS — Public datasets from data.cms.gov.
|
|
|
|
Auto-generated from the CMS Data API catalog.
|
|
Contains SQLTable models for all publicly available datasets.
|
|
|
|
Datasets cover Medicare Shared Savings Program (MSSP),
|
|
ACO REACH, CMS Program Statistics, and other CMS programs.
|
|
|
|
Tables::
|
|
|
|
{table_list}
|
|
|
|
Usage::
|
|
|
|
from cms.table import AcoParticipants
|
|
"""
|
|
'''
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CSV download
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def get_download_url(dataset: dict) -> str | None:
|
|
"""Extract the best download URL from dataset distributions.
|
|
|
|
Prefers the most recent CSV. Falls back to ZIP.
|
|
"""
|
|
csv_urls = []
|
|
zip_urls = []
|
|
for dist in dataset.get("distribution", []):
|
|
url = dist.get("downloadURL")
|
|
if not url:
|
|
continue
|
|
media = (dist.get("mediaType") or "").lower()
|
|
if media == "text/csv" or url.endswith(".csv"):
|
|
csv_urls.append(url)
|
|
elif media == "application/zip" or url.endswith(".zip"):
|
|
zip_urls.append(url)
|
|
|
|
# Return last CSV (most recent) or last ZIP
|
|
if csv_urls:
|
|
return csv_urls[-1]
|
|
if zip_urls:
|
|
return zip_urls[-1]
|
|
return None
|
|
|
|
|
|
def download_file(url: str, dest: Path, *, force: bool = False) -> int:
|
|
"""Download a file from URL to dest. Returns file size in bytes."""
|
|
if dest.exists() and not force:
|
|
print(f" Skipping (exists): {dest.name}")
|
|
return dest.stat().st_size
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
req = Request(url, headers={"User-Agent": "cms-scraper/1.0"})
|
|
with urlopen(req, timeout=600) as resp:
|
|
data = resp.read()
|
|
dest.write_bytes(data)
|
|
return len(data)
|
|
|
|
|
|
def download_and_extract_zip(
|
|
url: str,
|
|
dest_dir: Path,
|
|
*,
|
|
force: bool = False,
|
|
) -> list[Path]:
|
|
"""Download ZIP and extract CSVs/Excel files to dest_dir."""
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
zip_path = dest_dir / "archive.zip"
|
|
if not zip_path.exists() or force:
|
|
req = Request(url, headers={"User-Agent": "cms-scraper/1.0"})
|
|
with urlopen(req, timeout=300) as resp:
|
|
zip_path.write_bytes(resp.read())
|
|
extracted = []
|
|
with zipfile.ZipFile(zip_path) as zf:
|
|
for name in zf.namelist():
|
|
if name.lower().endswith((".csv", ".xlsx", ".xls")):
|
|
zf.extract(name, dest_dir)
|
|
extracted.append(dest_dir / name)
|
|
return extracted
|
|
|
|
|
|
def infer_schema_from_csv(csv_path: Path) -> dict[str, str] | None:
|
|
"""Infer column types from a CSV file header + first rows.
|
|
|
|
Returns dict like {"COL_NAME": "TEXT", ...}.
|
|
Used as fallback when data-viewer endpoint is unavailable.
|
|
"""
|
|
import csv
|
|
|
|
try:
|
|
with open(csv_path, newline="", encoding="utf-8-sig") as f:
|
|
reader = csv.DictReader(f)
|
|
if not reader.fieldnames:
|
|
return None
|
|
# Read up to 100 rows for type inference
|
|
rows = []
|
|
for i, row in enumerate(reader):
|
|
rows.append(row)
|
|
if i >= 99:
|
|
break
|
|
|
|
result = {}
|
|
for col in reader.fieldnames:
|
|
values = [r[col] for r in rows if r.get(col)]
|
|
col_type = _infer_type(values)
|
|
result[col] = col_type
|
|
return result
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def infer_schema_from_excel(xlsx_path: Path) -> dict[str, str] | None:
|
|
"""Infer column types from an Excel file header + first rows.
|
|
|
|
Handles CMS xlsx files that have multi-row title headers by
|
|
scanning for the first row with 3+ non-null cells as the header.
|
|
Tries all sheets to handle workbooks where the first sheet is a
|
|
table of contents.
|
|
"""
|
|
try:
|
|
import openpyxl
|
|
|
|
wb = openpyxl.load_workbook(xlsx_path, read_only=True)
|
|
|
|
for ws in wb.worksheets:
|
|
rows_iter = ws.iter_rows(values_only=True)
|
|
|
|
header = None
|
|
for row in rows_iter:
|
|
non_null = [c for c in row if c is not None]
|
|
if len(non_null) >= 3:
|
|
header = row
|
|
break
|
|
if not header:
|
|
continue
|
|
|
|
cols = [str(h).strip() for h in header if h is not None]
|
|
sample = []
|
|
for i, row in enumerate(rows_iter):
|
|
sample.append(row)
|
|
if i >= 99:
|
|
break
|
|
result = {}
|
|
for j, col in enumerate(cols):
|
|
values = [str(r[j]) for r in sample if j < len(r) and r[j] is not None]
|
|
result[col] = _infer_type(values)
|
|
wb.close()
|
|
return result
|
|
|
|
wb.close()
|
|
return None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _infer_type(values: list[str]) -> str:
|
|
"""Infer CMS-style type from a list of string values."""
|
|
from datetime import datetime
|
|
|
|
if not values:
|
|
return "TEXT"
|
|
date_count = 0
|
|
for v in values:
|
|
try:
|
|
datetime.strptime(v.strip(), "%Y-%m-%d")
|
|
date_count += 1
|
|
except (ValueError, AttributeError):
|
|
pass
|
|
if date_count > len(values) * 0.8:
|
|
return "DATE"
|
|
return "TEXT"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Scrape CMS data.gov datasets and generate SQLTable models"
|
|
)
|
|
parser.add_argument(
|
|
"--no-download",
|
|
action="store_true",
|
|
help="Skip CSV downloads, only generate table models",
|
|
)
|
|
parser.add_argument(
|
|
"--force",
|
|
action="store_true",
|
|
help="Re-download CSVs even if they already exist",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
print("Fetching CMS catalog...")
|
|
datasets = fetch_catalog()
|
|
print(f"Found {len(datasets)} datasets\n")
|
|
|
|
TABLE_DIR.mkdir(parents=True, exist_ok=True)
|
|
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
tables: list[tuple[str, str, str]] = [] # (module_name, class_name, table_name)
|
|
skipped: list[str] = []
|
|
|
|
for ds in datasets:
|
|
title = ds.get("title", "Unknown")
|
|
uuid = extract_uuid(ds)
|
|
print(f"--- {title}")
|
|
|
|
if not uuid:
|
|
print(" SKIP: no UUID found")
|
|
skipped.append(title)
|
|
continue
|
|
|
|
# Fetch schema from data-viewer endpoint
|
|
schema = fetch_schema(uuid)
|
|
|
|
# Fallback: download CSV and infer schema
|
|
if not schema and not args.no_download:
|
|
dl_url = get_download_url(ds)
|
|
if dl_url and dl_url.endswith(".csv"):
|
|
csv_dest = RAW_DIR / f"{sanitize_table_name(title)}.csv"
|
|
print(" No data-viewer; downloading CSV for schema...")
|
|
download_file(dl_url, csv_dest, force=args.force)
|
|
schema = infer_schema_from_csv(csv_dest)
|
|
elif dl_url and dl_url.endswith(".zip"):
|
|
zip_dir = RAW_DIR / sanitize_table_name(title)
|
|
print(" No data-viewer; downloading ZIP for schema...")
|
|
csvs = download_and_extract_zip(
|
|
dl_url,
|
|
zip_dir,
|
|
force=args.force,
|
|
)
|
|
if csvs:
|
|
# Try all files, pick schema with most columns
|
|
best = None
|
|
for f in csvs:
|
|
if f.suffix.lower() in (".xlsx", ".xls"):
|
|
s = infer_schema_from_excel(f)
|
|
else:
|
|
s = infer_schema_from_csv(f)
|
|
if s and (not best or len(s) > len(best)):
|
|
best = s
|
|
schema = best
|
|
|
|
if not schema:
|
|
print(" SKIP: could not determine schema")
|
|
skipped.append(title)
|
|
continue
|
|
|
|
table_name = sanitize_table_name(title)
|
|
class_name = sanitize_class_name(title)
|
|
|
|
# Generate table module
|
|
code = generate_table_module(
|
|
class_name,
|
|
table_name,
|
|
schema,
|
|
title,
|
|
uuid,
|
|
)
|
|
module_path = TABLE_DIR / f"{table_name}.py"
|
|
module_path.write_text(code)
|
|
tables.append((table_name, class_name, table_name))
|
|
print(f" Generated: {module_path.name} ({len(schema)} columns)")
|
|
|
|
# Download CSV data
|
|
if not args.no_download:
|
|
dl_url = get_download_url(ds)
|
|
if dl_url:
|
|
if dl_url.endswith(".zip"):
|
|
zip_dir = RAW_DIR / table_name
|
|
download_and_extract_zip(
|
|
dl_url,
|
|
zip_dir,
|
|
force=args.force,
|
|
)
|
|
print(f" Downloaded: {table_name}/ (ZIP)")
|
|
else:
|
|
csv_dest = RAW_DIR / f"{table_name}.csv"
|
|
size = download_file(
|
|
dl_url,
|
|
csv_dest,
|
|
force=args.force,
|
|
)
|
|
mb = size / 1024 / 1024
|
|
print(f" Downloaded: {csv_dest.name} ({mb:.1f} MB)")
|
|
else:
|
|
print(" No download URL available")
|
|
|
|
# Generate __init__.py files
|
|
init_code = generate_init_module(tables)
|
|
(TABLE_DIR / "__init__.py").write_text(init_code)
|
|
|
|
ns_init = generate_namespace_init(tables)
|
|
(TABLE_DIR.parent / "__init__.py").write_text(ns_init)
|
|
|
|
# Summary
|
|
print(f"\n{'=' * 60}")
|
|
print(f"Generated {len(tables)} table models in src/cms/table/")
|
|
for mod, cls, tbl in sorted(tables):
|
|
print(f" cms.{tbl} → {cls}")
|
|
if skipped:
|
|
print(f"\nSkipped {len(skipped)} datasets:")
|
|
for s in skipped:
|
|
print(f" - {s}")
|
|
print(f"{'=' * 60}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|