f66b35f5fe
Split dev/ flat directory into dev/scripts/ (29 .py files) and dev/seeds/ (PDFs, Excel, ZIPs, NDJSON, grafana, PY2023). Update all Path(__file__) references to use parents[2] for project root and dev/seeds/ for seed data. Fix path references in src/ too. Add PY2024 CMS quality measure value sets (HWR, UAMCC, ACR) to the REGISTRY and load into DuckDB — enables 2024->2025->2026 diffs. Add HWR tables to notebook DIFF_SPECS now that two years exist.
403 lines
12 KiB
Python
403 lines
12 KiB
Python
"""Extract CCLF file naming conventions from the CCLF Information Packet.
|
|
|
|
Parses the PDF to verify patterns, then generates
|
|
``src/aco/table/cclf_filenames.py`` — a module containing regex
|
|
patterns and a classifier function that rex can use to identify
|
|
which CCLF file a given filename represents.
|
|
|
|
Usage::
|
|
|
|
python dev/scripts/generate_cclf_filenames.py
|
|
|
|
Source: https://www.cms.gov/files/document/cclf-information-packet.pdf
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pdfplumber
|
|
|
|
# CCLF file IDs → table names
|
|
CCLF_FILES = {
|
|
"0": ("CCLF0", "Summary Statistics Header Record"),
|
|
"1": ("CCLF1", "Part A Claims Header File"),
|
|
"2": ("CCLF2", "Part A Claims Revenue Center Detail File"),
|
|
"3": ("CCLF3", "Part A Procedure Code File"),
|
|
"4": ("CCLF4", "Part A Diagnosis Code File"),
|
|
"5": ("CCLF5", "Part B Physicians File"),
|
|
"6": ("CCLF6", "Part B DME File"),
|
|
"7": ("CCLF7", "Part D File"),
|
|
"8": ("CCLF8", "Beneficiary Demographics File"),
|
|
"9": ("CCLF9", "Beneficiary XREF File"),
|
|
"A": ("CCLFA", "Part A Benefit Enhancement and Demo Codes"),
|
|
"B": ("CCLFB", "Part B Benefit Enhancement and Demo Codes"),
|
|
}
|
|
|
|
PROGRAMS = {
|
|
"A": ("sssp", "Medicare Shared Savings Program"),
|
|
"D": ("reach", "ACO REACH Model"),
|
|
"K": ("kcf", "Kidney Care First"),
|
|
"C": ("ckcc", "Comprehensive Kidney Care Contracting"),
|
|
"P": ("pcf", "Primary Care First"),
|
|
"IOTA": ("iota", "Increasing Organ Transplant Access"),
|
|
}
|
|
|
|
|
|
def extract_filename_patterns(pdf_path: str) -> list[dict]:
|
|
"""Extract all filename convention patterns from the PDF."""
|
|
pdf = pdfplumber.open(pdf_path)
|
|
patterns: list[dict] = []
|
|
seen: set[str] = set()
|
|
|
|
pattern_re = re.compile(
|
|
r"P\."
|
|
r"(IOTA\*{3}|[A-Z]\*{4,6})"
|
|
r"\."
|
|
r"(ACO|PRT)"
|
|
r"\."
|
|
r"ZC"
|
|
r"([0-9A-B])?"
|
|
r"([YR])"
|
|
r"(\*{2}|\d{2})"
|
|
r"\."
|
|
r"D"
|
|
)
|
|
|
|
for page in pdf.pages:
|
|
text = page.extract_text() or ""
|
|
for m in pattern_re.finditer(text):
|
|
program_id = m.group(1).rstrip("*")
|
|
entity = m.group(2)
|
|
file_id = m.group(3) or ""
|
|
run_type = m.group(4)
|
|
year = m.group(5)
|
|
|
|
key = f"{program_id}|{entity}|{file_id}|{run_type}"
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
|
|
patterns.append(
|
|
{
|
|
"program_prefix": program_id,
|
|
"entity": entity,
|
|
"file_id": file_id,
|
|
"run_type": run_type,
|
|
"year_example": year,
|
|
}
|
|
)
|
|
|
|
return patterns
|
|
|
|
|
|
MODULE_SOURCE = r'''"""CCLF filename conventions and regex classifier for rex.
|
|
|
|
Auto-generated from the CCLF Information Packet (IP).
|
|
Source: https://www.cms.gov/files/document/cclf-information-packet.pdf
|
|
|
|
CMS delivers CCLF data as a single ZIP per ACO per month.
|
|
Inside the ZIP, each of the 12 CCLF files has a filename
|
|
encoding the program, ACO ID, file type, run type, and
|
|
performance year.
|
|
|
|
Naming convention::
|
|
|
|
ZIP: P.{prog}{id}.{ent}.ZC{run}{yy}.D{yymmdd}.T{hhmmsst}
|
|
File: P.{prog}{id}.{ent}.ZC{file}{run}{yy}.D{yymmdd}.T{hhmmsst}
|
|
|
|
Programs::
|
|
|
|
A = Medicare Shared Savings Program (sssp)
|
|
D = ACO REACH Model (reach)
|
|
K = Kidney Care First (kcf)
|
|
C = Comprehensive Kidney Care Contracting (ckcc)
|
|
P = Primary Care First (pcf)
|
|
IOTA = Increasing Organ Transplant Access (iota)
|
|
|
|
File IDs::
|
|
|
|
0 = CCLF0: Summary Statistics Header Record
|
|
1 = CCLF1: Part A Claims Header File
|
|
2 = CCLF2: Part A Claims Revenue Center Detail File
|
|
3 = CCLF3: Part A Procedure Code File
|
|
4 = CCLF4: Part A Diagnosis Code File
|
|
5 = CCLF5: Part B Physicians File
|
|
6 = CCLF6: Part B DME File
|
|
7 = CCLF7: Part D File
|
|
8 = CCLF8: Beneficiary Demographics File
|
|
9 = CCLF9: Beneficiary XREF File
|
|
A = CCLFA: Part A Benefit Enhancement and Demo Codes
|
|
B = CCLFB: Part B Benefit Enhancement and Demo Codes
|
|
|
|
Run types::
|
|
|
|
Y = Regular monthly CCLF
|
|
R = Run-out (3-month claims run-out for prior PY)
|
|
|
|
Usage::
|
|
|
|
from aco.table.cclf_filenames import classify, is_cclf
|
|
|
|
result = classify("P.A1234.ACO.ZC1Y25.D250716.T1234567")
|
|
# CclfFilename(program='sssp', aco_id='1234', file_id='1',
|
|
# cclf_table='cclf1', run_type='Y', ...)
|
|
|
|
identify_cclf_table("P.D5678.ACO.ZC5R24.D250101.T0000001")
|
|
# 'cclf5'
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import NamedTuple
|
|
|
|
|
|
# ── File ID -> CCLF table mapping ────────────────────────────────
|
|
|
|
CCLF_FILE_IDS: dict[str, str] = {
|
|
"0": "cclf0", # Summary Statistics Header Record
|
|
"1": "cclf1", # Part A Claims Header File
|
|
"2": "cclf2", # Part A Claims Revenue Center Detail File
|
|
"3": "cclf3", # Part A Procedure Code File
|
|
"4": "cclf4", # Part A Diagnosis Code File
|
|
"5": "cclf5", # Part B Physicians File
|
|
"6": "cclf6", # Part B DME File
|
|
"7": "cclf7", # Part D File
|
|
"8": "cclf8", # Beneficiary Demographics File
|
|
"9": "cclf9", # Beneficiary XREF File
|
|
"A": "cclfa", # Part A Benefit Enhancement and Demo Codes
|
|
"B": "cclfb", # Part B Benefit Enhancement and Demo Codes
|
|
}
|
|
|
|
|
|
# ── Program prefix -> program name mapping ───────────────────────
|
|
|
|
PROGRAMS: dict[str, str] = {
|
|
"A": "sssp", # Medicare Shared Savings Program
|
|
"D": "reach", # ACO REACH Model
|
|
"K": "kcf", # Kidney Care First
|
|
"C": "ckcc", # Comprehensive Kidney Care Contracting
|
|
"P": "pcf", # Primary Care First
|
|
"IOTA": "iota", # Increasing Organ Transplant Access
|
|
}
|
|
|
|
|
|
# ── Parsed filename result ───────────────────────────────────────
|
|
|
|
|
|
class CclfFilename(NamedTuple):
|
|
"""Parsed components of a CCLF filename."""
|
|
|
|
program: str
|
|
"""Program short name (sssp, reach, kcf, ckcc, pcf, iota)."""
|
|
|
|
aco_id: str
|
|
"""ACO/entity identifier."""
|
|
|
|
entity: str
|
|
"""Entity type (ACO or PRT)."""
|
|
|
|
file_id: str
|
|
"""CCLF file ID (0-9, A, B). Empty for ZIP files."""
|
|
|
|
cclf_table: str
|
|
"""CCLF table name (cclf0-cclf9, cclfa, cclfb). Empty for ZIPs."""
|
|
|
|
run_type: str
|
|
"""Run type: Y=regular, R=run-out."""
|
|
|
|
performance_year: int
|
|
"""Performance year (4-digit, e.g. 2025)."""
|
|
|
|
delivery_date: str
|
|
"""Delivery date string (yymmdd)."""
|
|
|
|
delivery_time: str
|
|
"""Delivery timestamp string (hhmmsst)."""
|
|
|
|
is_zip: bool
|
|
"""True if this is the outer ZIP filename."""
|
|
|
|
|
|
# ── Regex patterns ───────────────────────────────────────────────
|
|
|
|
# Individual CCLF file inside a ZIP:
|
|
# P.A1234.ACO.ZC1Y25.D250716.T1234567
|
|
CCLF_FILE_RE = re.compile(
|
|
r"P\."
|
|
r"(IOTA|[A-Z])" # program prefix
|
|
r"([A-Z0-9*]{3,6})" # ACO ID
|
|
r"\."
|
|
r"(ACO|PRT)" # entity type
|
|
r"\."
|
|
r"ZC"
|
|
r"([0-9A-B])" # file ID
|
|
r"([YR])" # run type
|
|
r"(\d{2})" # performance year
|
|
r"\."
|
|
r"D(\d{6})" # delivery date
|
|
r"\."
|
|
r"T(\d{7})" # delivery time
|
|
)
|
|
|
|
# Outer ZIP file (no file ID):
|
|
# P.A1234.ACO.ZCY25.D250716.T1234567
|
|
CCLF_ZIP_RE = re.compile(
|
|
r"P\."
|
|
r"(IOTA|[A-Z])" # program prefix
|
|
r"([A-Z0-9*]{3,6})" # ACO ID
|
|
r"\."
|
|
r"(ACO|PRT)" # entity type
|
|
r"\."
|
|
r"ZC"
|
|
r"([YR])" # run type
|
|
r"(\d{2})" # performance year
|
|
r"\."
|
|
r"D(\d{6})" # delivery date
|
|
r"\."
|
|
r"T(\d{7})" # delivery time
|
|
)
|
|
|
|
|
|
def classify(filename: str) -> CclfFilename | None:
|
|
"""Parse a CCLF filename and return its components.
|
|
|
|
Handles both individual CCLF files (inside ZIPs) and the
|
|
outer ZIP filenames. Returns None if the filename does
|
|
not match any known CCLF pattern.
|
|
|
|
Parameters
|
|
----------
|
|
filename : str
|
|
CCLF filename (with or without directory path).
|
|
|
|
Returns
|
|
-------
|
|
CclfFilename or None
|
|
Parsed components, or None if not a CCLF file.
|
|
|
|
Examples
|
|
--------
|
|
>>> r = classify("P.A1234.ACO.ZC1Y25.D250716.T1234567")
|
|
>>> r.program, r.cclf_table, r.run_type
|
|
('sssp', 'cclf1', 'Y')
|
|
|
|
>>> r = classify("P.D5678.ACO.ZCR24.D250101.T0000001")
|
|
>>> r.program, r.is_zip, r.run_type
|
|
('reach', True, 'R')
|
|
"""
|
|
# Strip directory path if present
|
|
name = filename.rsplit("/", 1)[-1]
|
|
name = name.rsplit("\\", 1)[-1]
|
|
|
|
# Try individual file pattern first (more specific)
|
|
m = CCLF_FILE_RE.match(name)
|
|
if m:
|
|
prog_prefix = m.group(1)
|
|
program = PROGRAMS.get(prog_prefix, prog_prefix.lower())
|
|
file_id = m.group(4)
|
|
cclf_table = CCLF_FILE_IDS.get(file_id, "")
|
|
py = int(m.group(6))
|
|
return CclfFilename(
|
|
program=program,
|
|
aco_id=m.group(2),
|
|
entity=m.group(3),
|
|
file_id=file_id,
|
|
cclf_table=cclf_table,
|
|
run_type=m.group(5),
|
|
performance_year=2000 + py,
|
|
delivery_date=m.group(7),
|
|
delivery_time=m.group(8),
|
|
is_zip=False,
|
|
)
|
|
|
|
# Try ZIP pattern
|
|
m = CCLF_ZIP_RE.match(name)
|
|
if m:
|
|
prog_prefix = m.group(1)
|
|
program = PROGRAMS.get(prog_prefix, prog_prefix.lower())
|
|
py = int(m.group(5))
|
|
return CclfFilename(
|
|
program=program,
|
|
aco_id=m.group(2),
|
|
entity=m.group(3),
|
|
file_id="",
|
|
cclf_table="",
|
|
run_type=m.group(4),
|
|
performance_year=2000 + py,
|
|
delivery_date=m.group(6),
|
|
delivery_time=m.group(7),
|
|
is_zip=True,
|
|
)
|
|
|
|
return None
|
|
|
|
|
|
def identify_cclf_table(filename: str) -> str:
|
|
"""Return the CCLF table name for a given filename.
|
|
|
|
Convenience function for rex integration.
|
|
|
|
Parameters
|
|
----------
|
|
filename : str
|
|
CCLF filename.
|
|
|
|
Returns
|
|
-------
|
|
str
|
|
Table name (e.g. ``cclf1``), or empty string.
|
|
"""
|
|
result = classify(filename)
|
|
if result is None:
|
|
return ""
|
|
return result.cclf_table
|
|
|
|
|
|
def is_cclf(filename: str) -> bool:
|
|
"""Check if a filename matches any CCLF naming convention.
|
|
|
|
Parameters
|
|
----------
|
|
filename : str
|
|
Filename to check.
|
|
|
|
Returns
|
|
-------
|
|
bool
|
|
True if the filename is a CCLF file or ZIP.
|
|
"""
|
|
return classify(filename) is not None
|
|
'''
|
|
|
|
|
|
def main() -> None:
|
|
project_root = Path(__file__).resolve().parents[2]
|
|
pdf_path = project_root / "dev" / "seeds" / "cclf-information-packet.pdf"
|
|
output_path = project_root / "src" / "aco" / "table" / "cclf_filenames.py"
|
|
|
|
print(f"Parsing {pdf_path.name} for filename conventions...")
|
|
patterns = extract_filename_patterns(str(pdf_path))
|
|
|
|
print(f"Found {len(patterns)} unique filename patterns:")
|
|
by_program: dict[str, list[dict]] = {}
|
|
for p in patterns:
|
|
prog = p["program_prefix"]
|
|
by_program.setdefault(prog, []).append(p)
|
|
|
|
for prog, pats in sorted(by_program.items()):
|
|
prog_name = PROGRAMS.get(prog, (prog, prog))[1]
|
|
file_ids = sorted(set(p["file_id"] for p in pats if p["file_id"]))
|
|
run_types = sorted(set(p["run_type"] for p in pats))
|
|
print(f" {prog_name}: files={file_ids or ['ZIP']}, run_types={run_types}")
|
|
|
|
output_path.write_text(MODULE_SOURCE.lstrip())
|
|
print(f"\nWritten to {output_path}")
|
|
print(f" {len(MODULE_SOURCE.splitlines())} lines")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|