300 lines
8.3 KiB
Python
300 lines
8.3 KiB
Python
"""Format registry — bundles sieves with readers and target schemas.
|
|
|
|
A ``Format`` ties together:
|
|
|
|
1. A ``Sieve`` (how to classify lines)
|
|
2. A reader callable (how to open the file)
|
|
3. A target table ref (where the extracted data maps to in ``aco``)
|
|
4. A column name mapping (raw field names → ``input_layer`` column names)
|
|
|
|
Reader functions are format-specific: ``read_text`` streams lines from
|
|
a plain text file, ``read_excel`` iterates rows from worksheets
|
|
(handling merged cells and multi-row headers), ``read_pdf`` extracts
|
|
text from PDF pages, etc.
|
|
|
|
The ``registry`` is a plain dict mapping format names (e.g.
|
|
``"cms_mmr"``, ``"sas_listing"``, ``"x12_835"``) to ``Format``
|
|
instances. This is the lookup used by ``rex.pipe.extract()``
|
|
when called with ``format="cms_mmr"``.
|
|
|
|
Usage::
|
|
|
|
from rex.format import Format, registry, read_text, read_excel
|
|
|
|
# Look up a registered format
|
|
fmt = registry["sas_listing"]
|
|
lines = fmt.reader(Path("report.lst"))
|
|
|
|
# Or build a custom format
|
|
fmt = Format(
|
|
name="custom_claims",
|
|
sieve=my_sieve,
|
|
reader=read_text,
|
|
target="input_layer.medical_claim",
|
|
column_map={"CLAIM-ID": "claim_id", "PAID-AMT": "paid_amount"},
|
|
)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
from pydantic import BaseModel, ConfigDict
|
|
|
|
from rex.sieve import Sieve
|
|
|
|
|
|
class Format(BaseModel):
|
|
"""A complete file format definition.
|
|
|
|
Bundles a ``Sieve`` (line classifier) with the IO strategy
|
|
(reader) and the output mapping (target table + column names).
|
|
"""
|
|
|
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
|
|
name: str
|
|
"""Format identifier (e.g. ``"sas_listing"``, ``"cms_mmr"``)."""
|
|
|
|
sieve: Sieve
|
|
"""The four-regex line classifier for this format."""
|
|
|
|
reader: Any = None # Callable[[Path], Iterable[str | dict]]
|
|
"""Callable that opens a file and yields lines or row dicts.
|
|
|
|
Signature: ``(Path) -> Iterable[str | dict]``
|
|
|
|
For text-oriented formats, yields ``str`` lines.
|
|
For structured formats (Excel, JSON), yields ``dict`` rows.
|
|
"""
|
|
|
|
target: str = ""
|
|
"""Target ``aco.table.input_layer`` table ref.
|
|
|
|
E.g. ``"input_layer.medical_claim"`` or ``"input_layer.eligibility"``.
|
|
If empty, the caller must specify the target explicitly.
|
|
"""
|
|
|
|
column_map: dict[str, str] = {}
|
|
"""Raw field name → ``input_layer`` column name.
|
|
|
|
E.g. ``{"CLAIM-ID": "claim_id", "PAID-AMT": "paid_amount"}``.
|
|
Applied after field extraction, before the express cleaning step.
|
|
"""
|
|
|
|
|
|
# ── Registry ─────────────────────────────────────────────────────
|
|
|
|
|
|
registry: dict[str, Format] = {}
|
|
"""Global format registry.
|
|
|
|
Populated at module load time with built-in formats.
|
|
Users can register custom formats via ``registry["my_format"] = Format(...)``.
|
|
"""
|
|
|
|
|
|
# ── Reader functions ─────────────────────────────────────────────
|
|
#
|
|
# Each reader opens a file and yields its content in a form that
|
|
# the sieve can process. Text readers yield str lines; structured
|
|
# readers (Excel, JSON) yield dict rows that bypass the sieve
|
|
# entirely and go straight to DataFrame construction.
|
|
|
|
|
|
def read_text(path: Path) -> Iterable[str]:
|
|
"""Read a plain text file line by line.
|
|
|
|
Handles: SAS .lst output, JCL spool dumps, fixed-width flat
|
|
files, any ASCII/UTF-8 text.
|
|
|
|
Strips trailing newlines. Respects the sieve's ``encoding``
|
|
attribute for non-UTF-8 files (e.g. ``cp1252``, ``cp037``).
|
|
|
|
Parameters
|
|
----------
|
|
path : Path
|
|
Path to the text file.
|
|
|
|
Yields
|
|
------
|
|
str
|
|
One line per iteration, newline stripped.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
def read_fixed(path: Path) -> Iterable[str]:
|
|
"""Read a fixed-width binary file (COBOL/EBCDIC layouts).
|
|
|
|
Unlike ``read_text``, this reads in binary mode and slices
|
|
records by a fixed record length (from the ``FieldMap``).
|
|
Handles EBCDIC encoding (``cp037``), packed decimal (COMP-3)
|
|
fields, and binary (COMP) fields that would corrupt in text
|
|
mode.
|
|
|
|
Parameters
|
|
----------
|
|
path : Path
|
|
Path to the fixed-width file.
|
|
|
|
Yields
|
|
------
|
|
str
|
|
One record per iteration (decoded from bytes).
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
def read_excel(path: Path) -> Iterable[dict[str, Any]]:
|
|
"""Read rows from an Excel workbook (.xls, .xlsx).
|
|
|
|
Handles the common mainframe-to-Excel artifacts:
|
|
|
|
- **Merged cells**: forward-fills merged header values
|
|
- **Multi-row headers**: detects header depth and flattens
|
|
nested column names (e.g. ``"Inpatient | Claims"`` →
|
|
``"inpatient_claims"``)
|
|
- **Summary rows**: marks rows containing subtotals/grand
|
|
totals so the sieve can classify them as ``Zone.SKIP``
|
|
- **Multiple sheets**: iterates all sheets by default,
|
|
tagging each row with ``_sheet_name``
|
|
- **Leading zero preservation**: reads numeric-looking
|
|
columns as strings to prevent Excel's auto-conversion
|
|
|
|
Parameters
|
|
----------
|
|
path : Path
|
|
Path to the Excel file.
|
|
|
|
Yields
|
|
------
|
|
dict[str, Any]
|
|
One row dict per data row, with flattened column names.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
def read_csv(path: Path) -> Iterable[str]:
|
|
"""Read a CSV-like delimited file line by line.
|
|
|
|
Handles variants common in healthcare mainframe exports:
|
|
|
|
- Mixed delimiters (comma, pipe, tab, tilde)
|
|
- Inconsistent quoting
|
|
- BOM markers (UTF-8-BOM, UTF-16 LE/BE)
|
|
- Encoding artifacts from EBCDIC→ASCII conversion
|
|
- Embedded newlines in quoted fields
|
|
|
|
Lines are yielded raw for sieve classification. For clean
|
|
CSVs where the sieve is trivial (every line is data), the
|
|
``FieldMap`` handles delimiter-based splitting.
|
|
|
|
Parameters
|
|
----------
|
|
path : Path
|
|
Path to the delimited file.
|
|
|
|
Yields
|
|
------
|
|
str
|
|
One line per iteration.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
def read_pdf(path: Path) -> Iterable[str]:
|
|
"""Extract text lines from a PDF document.
|
|
|
|
Targets PDFs generated from mainframe report printing — these
|
|
are typically text-based (not scanned images) with fixed-width
|
|
layout preserved from the original SAS/COBOL listing.
|
|
|
|
Each page is extracted as a block of text, then split into
|
|
lines. Form-feed boundaries between pages are preserved as
|
|
empty lines for sieve classification.
|
|
|
|
Parameters
|
|
----------
|
|
path : Path
|
|
Path to the PDF file.
|
|
|
|
Yields
|
|
------
|
|
str
|
|
One text line per iteration.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
def read_json(path: Path) -> Iterable[dict[str, Any]]:
|
|
"""Read records from a JSON file.
|
|
|
|
Supports:
|
|
|
|
- JSON array of objects (``[{...}, {...}]``)
|
|
- Newline-delimited JSON / JSON Lines (``.jsonl``)
|
|
- Nested JSON (flattened with dot-separated keys)
|
|
|
|
Bypasses the sieve entirely — JSON is already structured.
|
|
Each yielded dict goes straight to DataFrame construction.
|
|
|
|
Parameters
|
|
----------
|
|
path : Path
|
|
Path to the JSON file.
|
|
|
|
Yields
|
|
------
|
|
dict[str, Any]
|
|
One record per iteration.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
def read_xml(path: Path) -> Iterable[dict[str, Any]]:
|
|
"""Read records from an XML file.
|
|
|
|
Targets healthcare XML formats: CDA documents, state
|
|
Medicaid XML extracts, custom payer XML feeds.
|
|
|
|
Detects repeating elements and extracts them as flat dicts.
|
|
Nested elements are flattened with dot-separated keys.
|
|
|
|
Parameters
|
|
----------
|
|
path : Path
|
|
Path to the XML file.
|
|
|
|
Yields
|
|
------
|
|
dict[str, Any]
|
|
One record per iteration.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
def read_docx(path: Path) -> Iterable[str]:
|
|
"""Extract text and table content from a Word document.
|
|
|
|
Targets .docx files generated from mainframe report exports
|
|
(common when mainframe output is pasted into Word templates).
|
|
|
|
Extracts:
|
|
|
|
- Paragraph text (as lines for sieve classification)
|
|
- Table rows (as tab-delimited lines, one per table row)
|
|
|
|
Parameters
|
|
----------
|
|
path : Path
|
|
Path to the .docx file.
|
|
|
|
Yields
|
|
------
|
|
str
|
|
One line per iteration (paragraphs or table rows).
|
|
"""
|
|
raise NotImplementedError
|