feat(opps): recover CY2014-2020 Addendum B + fix ingest finder
CMS purged the pre-2021 OPPS addenda in its 2023 site restructure, so
opps.addendum_b only held CY2021-2026. Recover CY2014-2020 from the
Internet Archive Wayback Machine (the `web/{ts}id_/{url}` raw-binary form)
and wire the URLs into download_opps_files.py, with retry/backoff since
Wayback rate-limits.
Key subtlety: the payment data (Relative Weight + Payment Rate per HCPCS)
lives in the real "Addendum B - Final OPPS Payment by HCPCS Code", which
ships inside the COMBINED addenda ZIP — NOT the "Data Addendum B /
2-Times-Rule" file (SI/APC/comment-indicator only, no payment). Point the
older years at the combined ZIPs (CY2016's only surviving snapshot is
2015-vintage but serves the real Addendum B).
Fix the ingest file-finder accordingly:
- match "Addendum B" separator-insensitively (CMS mixes "Addendum B.xlsx"
and "CMS-1613-FC-Addendum-B.xlsx" across years);
- exclude the "Data Addendum B" file (normalised "dataadd") so it never
shadows the payment file.
After this, all 13 years ingest and reconcile 100% exact against Addendum B.
Data files remain gitignored; download_opps_files.py + ingest_opps.py
reproduce them.
This commit is contained in:
kert
@@ -27,11 +27,26 @@ USER_AGENT = "stack-opps-ingest/1.0"
|
||||
# ---------------------------------------------------------------------------
|
||||
# CMS OPPS download URLs by year
|
||||
# ---------------------------------------------------------------------------
|
||||
# CMS publishes combined addenda ZIPs starting ~CY2021 at:
|
||||
# CY2021+ : CMS publishes combined addenda ZIPs at
|
||||
# https://www.cms.gov/files/zip/{year}-nfrm-opps-addenda.zip
|
||||
#
|
||||
# Pre-2021 files were purged from cms.gov during the 2023 site restructure.
|
||||
# They may be recoverable from the Wayback Machine. Filed as a TODO.
|
||||
# CY2014-2020 : purged from cms.gov in the 2023 site restructure, recovered
|
||||
# from the Internet Archive Wayback Machine. The `web/{ts}id_/{url}` form
|
||||
# returns the raw archived ZIP — the `id_` suffix is essential; without it
|
||||
# Wayback serves an HTML viewer page instead of the binary.
|
||||
#
|
||||
# IMPORTANT — must be the COMBINED "Addenda" ZIP, which contains the real
|
||||
# "Addendum B - Final OPPS Payment by HCPCS Code" (Relative Weight + Payment
|
||||
# Rate columns). Do NOT point at the "Data Addendum B / 2-Times-Rule" file:
|
||||
# that one carries only SI/APC/Comment-Indicator (used for geometric-mean-cost
|
||||
# development) and has NO payment rate, so nothing reconciles. 2014/2015 ship
|
||||
# one combined "Addenda.zip"; 2016-2020 an "...OPPS-FR-Addenda.zip". CY2016's
|
||||
# only surviving snapshot is 2015-vintage (2025 snapshots 302), but it serves
|
||||
# the real Addendum B. 2020 is the NFRM addenda, computed with CY2020's
|
||||
# as-published CF ($80.784); CMS later cited a corrected $80.793 — see the note
|
||||
# in opps.rules. (The 2019 URL is lowercased — the snapshot's exact casing.
|
||||
# If a timestamp goes stale, find alternates via
|
||||
# web.archive.org/cdx/search/cdx?url=<original>.)
|
||||
|
||||
OPPS_ADDENDA_URLS: dict[str, str] = {
|
||||
"2026": "https://www.cms.gov/files/zip/2026-nfrm-opps-addenda.zip",
|
||||
@@ -40,7 +55,16 @@ OPPS_ADDENDA_URLS: dict[str, str] = {
|
||||
"2023": "https://www.cms.gov/files/zip/2023-nfrm-opps-addenda.zip",
|
||||
"2022": "https://www.cms.gov/files/zip/2022-nfrm-opps-addenda.zip",
|
||||
"2021": "https://www.cms.gov/files/zip/2021-nfrm-opps-addenda.zip",
|
||||
# Pre-2021: purged from CMS. Need Wayback Machine recovery.
|
||||
# CY2014-2020 recovered from the Wayback Machine (combined addenda; note above).
|
||||
"2020": "https://web.archive.org/web/20250630084418id_/https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Payment/HospitalOutpatientPPS/Downloads/CMS-1717-FC-2020-OPPS-Addenda.zip",
|
||||
"2019": "https://web.archive.org/web/20250703013519id_/https://www.cms.gov/medicare/medicare-fee-for-service-payment/hospitaloutpatientpps/downloads/cms-1695-fc-2019-opps-fr-addenda.zip",
|
||||
"2018": "https://web.archive.org/web/20250630084406id_/https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Payment/HospitalOutpatientPPS/Downloads/CMS-1678-FC-2018-OPPS-FR-Addenda.zip",
|
||||
"2017": "https://web.archive.org/web/20250630084507id_/https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Payment/HospitalOutpatientPPS/Downloads/CMS-1656-FC-2017-OPPS-FR-Addenda.zip",
|
||||
# CY2016 combined addenda: only a 2015-vintage snapshot survives (the 2025
|
||||
# snapshots 302); this timestamp returns the real Addendum B (payment).
|
||||
"2016": "https://web.archive.org/web/20151105123722id_/https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Payment/HospitalOutpatientPPS/Downloads/CMS-1633-FC-2016-OPPS-FR-Addenda.zip",
|
||||
"2015": "https://web.archive.org/web/20250630084359id_/https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Payment/HospitalOutpatientPPS/Downloads/CMS-1613-FC-Addenda.zip",
|
||||
"2014": "https://web.archive.org/web/20250630084534id_/https://www.cms.gov/Medicare/Medicare-Fee-for-Service-Payment/HospitalOutpatientPPS/Downloads/CMS-1601-FC-Addenda.zip",
|
||||
}
|
||||
|
||||
# Supporting files (cost stats, impacts, etc.)
|
||||
@@ -53,27 +77,37 @@ OPPS_SUPPORTING_URLS: dict[str, dict[str, str]] = {
|
||||
}
|
||||
|
||||
|
||||
def download_file(url: str, dest: Path) -> bool:
|
||||
"""Download a file if it doesn't already exist."""
|
||||
def download_file(url: str, dest: Path, *, retries: int = 3) -> bool:
|
||||
"""Download a file if it doesn't already exist.
|
||||
|
||||
Retries on transport errors and 5xx — the Wayback Machine (the source
|
||||
for CY2014-2020) rate-limits and 5xx's under load.
|
||||
"""
|
||||
if dest.exists():
|
||||
print(f" EXISTS {dest.name}")
|
||||
return True
|
||||
try:
|
||||
print(f" GET {dest.name} ...", end="", flush=True)
|
||||
resp = httpx.get(
|
||||
url, headers={"User-Agent": USER_AGENT}, timeout=60, follow_redirects=True
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
dest.write_bytes(resp.content)
|
||||
size_mb = len(resp.content) / 1024 / 1024
|
||||
print(f" {size_mb:.1f} MB")
|
||||
return True
|
||||
else:
|
||||
print(f" HTTP {resp.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}")
|
||||
return False
|
||||
print(f" GET {dest.name} ...", end="", flush=True)
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
resp = httpx.get(
|
||||
url,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
timeout=120,
|
||||
follow_redirects=True,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
dest.write_bytes(resp.content)
|
||||
print(f" {len(resp.content) / 1024 / 1024:.1f} MB")
|
||||
return True
|
||||
if resp.status_code < 500 or attempt == retries:
|
||||
print(f" HTTP {resp.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
if attempt == retries:
|
||||
print(f" ERROR: {e}")
|
||||
return False
|
||||
time.sleep(2 * attempt) # linear backoff before the next attempt
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -19,6 +19,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
@@ -107,17 +108,31 @@ def find_data_file(zf: zipfile.ZipFile, pattern: str = "") -> str | None:
|
||||
if low.endswith((".csv", ".xlsx", ".xls")):
|
||||
candidates.append(name)
|
||||
if pattern:
|
||||
# Match "Addendum A" or "Addendum B" etc. in filename
|
||||
filtered = [c for c in candidates if pattern.lower() in c.lower()]
|
||||
# Prefer xlsx over csv, prefer root over 508 subfolder
|
||||
xlsx = [f for f in filtered if f.lower().endswith(".xlsx") and "/" not in f]
|
||||
if xlsx:
|
||||
return xlsx[0]
|
||||
csv = [f for f in filtered if f.lower().endswith(".csv")]
|
||||
if csv:
|
||||
return csv[0]
|
||||
if filtered:
|
||||
return filtered[0]
|
||||
# Match "Addendum A" / "Addendum B" etc., separator-insensitively:
|
||||
# CMS names use spaces ("Addendum B.xlsx") and hyphens
|
||||
# ("CMS-1613-FC-Addendum-B.xlsx") interchangeably across years.
|
||||
pn = re.sub(r"[^a-z0-9]", "", pattern.lower())
|
||||
filtered = [c for c in candidates if pn in re.sub(r"[^a-z0-9]", "", c.lower())]
|
||||
# Exclude the "Data Addendum B" geometric-mean-cost file — it carries
|
||||
# SI/APC/Comment-Indicator only, no payment rate, so it never
|
||||
# reconciles. Its name varies ("Data-Addendum-B", "DataAddB",
|
||||
# "Data Add B"); normalise and drop anything containing "dataadd".
|
||||
real = [
|
||||
c for c in filtered if "dataadd" not in re.sub(r"[^a-z0-9]", "", c.lower())
|
||||
]
|
||||
pool = real or filtered
|
||||
# Prefer xlsx, then xls, then csv; prefer root over a 508 subfolder.
|
||||
for in_root in (True, False):
|
||||
for ext in (".xlsx", ".xls", ".csv"):
|
||||
hits = [
|
||||
f
|
||||
for f in pool
|
||||
if f.lower().endswith(ext) and (("/" not in f) == in_root)
|
||||
]
|
||||
if hits:
|
||||
return sorted(hits)[0]
|
||||
if pool:
|
||||
return sorted(pool)[0]
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user