8f99eeb154
- 5519 unit tests covering all modules (aco, bcda, bls, cms, pfs, rex, bib) - ruff lint + format enforcement across entire codebase (377 files reformatted) - pre-commit hook: ruff check, ruff format, pytest - Woodpecker CI split into ci.yml (quality gate) and deploy.yml (package + images) - ci.yml: lint → test → validate-compose, runs on every push/PR - deploy.yml: build + publish Python package to Gitea PyPI registry, then container image builds, Trivy scans, and registry push (main branch only) - Gitea branch protection on main: requires CI status checks to pass - .gitignore updated for .coverage, dist/, *.egg-info/ - grafana config moved to dev/grafana/ Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
414 lines
13 KiB
Python
414 lines
13 KiB
Python
"""Tests for aco.pipe modules — structural validation.
|
|
|
|
Validates that all 12 pipe modules export well-formed Pipeline
|
|
instances whose Expr objects satisfy naming conventions,
|
|
dependency integrity, uniqueness, and callable fn attributes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import re
|
|
|
|
import pytest
|
|
|
|
from aco.express.base import Expr
|
|
from aco.pipe.base import Pipeline
|
|
|
|
# All 12 pipe modules under test
|
|
PIPE_MODULES = [
|
|
"aco.pipe.core",
|
|
"aco.pipe.readmissions",
|
|
"aco.pipe.ahrq_measures",
|
|
"aco.pipe.pharmacy",
|
|
"aco.pipe.hcc_suspecting",
|
|
"aco.pipe.input_layer",
|
|
"aco.pipe.claims_preprocessing",
|
|
"aco.pipe.cclf",
|
|
"aco.pipe.data_quality",
|
|
"aco.pipe.main",
|
|
"aco.pipe.quality_measures",
|
|
"aco.pipe.provider_attribution",
|
|
]
|
|
|
|
# Short labels for parametrize IDs
|
|
_IDS = [m.split(".")[-1] for m in PIPE_MODULES]
|
|
|
|
|
|
def _load_pipeline(module_path: str) -> Pipeline:
|
|
"""Import a pipe module and return its pipeline."""
|
|
mod = importlib.import_module(module_path)
|
|
return mod.pipeline
|
|
|
|
|
|
# ── Fixtures ──────────────────────────────────────────────
|
|
|
|
|
|
@pytest.fixture(params=PIPE_MODULES, ids=_IDS)
|
|
def pipe_module(request):
|
|
"""Yield each pipe module as an imported module object."""
|
|
return importlib.import_module(request.param)
|
|
|
|
|
|
@pytest.fixture(params=PIPE_MODULES, ids=_IDS)
|
|
def pipeline(request):
|
|
"""Yield each pipeline instance."""
|
|
return _load_pipeline(request.param)
|
|
|
|
|
|
# ── 1. Module exports a pipeline attribute ────────────────
|
|
|
|
|
|
class TestModuleExportsPipeline:
|
|
"""Each pipe module must export a `pipeline` attribute."""
|
|
|
|
def test_has_pipeline_attr(self, pipe_module):
|
|
assert hasattr(pipe_module, "pipeline"), (
|
|
f"{pipe_module.__name__} missing 'pipeline'"
|
|
)
|
|
|
|
def test_pipeline_is_pipeline_instance(self, pipe_module):
|
|
assert isinstance(pipe_module.pipeline, Pipeline)
|
|
|
|
def test_pipeline_is_nonempty(self, pipe_module):
|
|
assert len(pipe_module.pipeline) > 0, (
|
|
f"{pipe_module.__name__} pipeline is empty"
|
|
)
|
|
|
|
def test_has_run_attr(self, pipe_module):
|
|
"""Each module also exports a `run` convenience."""
|
|
assert hasattr(pipe_module, "run")
|
|
assert callable(pipe_module.run)
|
|
|
|
|
|
# ── 2. Expr names contain "." (qualified) ────────────────
|
|
|
|
|
|
class TestExprNamesQualified:
|
|
"""Every Expr name must be a qualified schema.table ref."""
|
|
|
|
def test_all_names_contain_dot(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
assert "." in expr.name, f"Expr name {expr.name!r} not qualified"
|
|
|
|
def test_all_names_have_nonempty_schema(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
schema = expr.name.split(".")[0]
|
|
assert len(schema) > 0, f"Empty schema in {expr.name!r}"
|
|
|
|
def test_all_names_have_nonempty_table(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
table = expr.name.split(".", 1)[1]
|
|
assert len(table) > 0, f"Empty table in {expr.name!r}"
|
|
|
|
|
|
# ── 3. Expr names follow underscore convention ───────────
|
|
|
|
|
|
class TestExprNamingConvention:
|
|
"""Expr names must use dot notation, not double/triple
|
|
underscores (those are for parameter names only)."""
|
|
|
|
def test_no_double_underscore_in_name(self, pipeline):
|
|
"""Expr.name uses dots, never raw __ encoding."""
|
|
for expr in pipeline.exprs:
|
|
# The name itself should not contain __ as a
|
|
# schema-table separator. Double underscores
|
|
# within a table part (e.g. input_layer__xxx)
|
|
# are allowed as sub-naming within tables.
|
|
schema = expr.name.split(".")[0]
|
|
assert "__" not in schema, f"Schema part has __: {expr.name!r}"
|
|
|
|
def test_names_match_dot_pattern(self, pipeline):
|
|
"""Every name matches schema.table_name pattern."""
|
|
pattern = re.compile(r"^[a-z][a-z0-9_]*\.[a-z_][a-z0-9_]*$")
|
|
for expr in pipeline.exprs:
|
|
assert pattern.match(expr.name), f"Name {expr.name!r} doesn't match pattern"
|
|
|
|
|
|
# ── 4. Each Expr has a callable fn ───────────────────────
|
|
|
|
|
|
class TestExprFnCallable:
|
|
"""Every Expr must have a callable fn attribute."""
|
|
|
|
def test_fn_is_callable(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
assert callable(expr.fn), f"{expr.name}: fn is not callable"
|
|
|
|
def test_fn_is_not_none(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
assert expr.fn is not None, f"{expr.name}: fn is None"
|
|
|
|
|
|
# ── 5. after lists reference valid names ─────────────────
|
|
|
|
|
|
class TestAfterDependencies:
|
|
"""Expr.after lists should reference names that are
|
|
either within the same pipeline or are external tables
|
|
(loaded from DB). We validate internal references."""
|
|
|
|
def test_after_entries_are_strings(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
for dep in expr.after:
|
|
assert isinstance(dep, str), (
|
|
f"{expr.name}: after entry {dep!r} is not a string"
|
|
)
|
|
|
|
def test_after_entries_are_qualified(self, pipeline):
|
|
"""Each after entry must be a qualified ref."""
|
|
for expr in pipeline.exprs:
|
|
for dep in expr.after:
|
|
assert "." in dep, f"{expr.name}: after dep {dep!r} is not qualified"
|
|
|
|
def test_after_refs_in_pipeline_are_valid_names(self, pipeline):
|
|
"""After deps that reference a pipeline output
|
|
must be valid names within the pipeline.
|
|
|
|
Note: after is a documentation/graph field --
|
|
the runner resolves inputs from parameter names
|
|
regardless of after ordering. So we only check
|
|
that referenced pipeline names actually exist,
|
|
not that they appear before the dependent expr.
|
|
"""
|
|
name_set = set(pipeline.names())
|
|
for expr in pipeline.exprs:
|
|
for dep in expr.after:
|
|
if dep in name_set:
|
|
# Valid: this dep references a
|
|
# pipeline-internal output
|
|
pass
|
|
else:
|
|
# External dep (DB table) -- must
|
|
# still be a qualified reference
|
|
assert "." in dep, (
|
|
f"{expr.name}: after dep {dep!r} is not qualified"
|
|
)
|
|
|
|
def test_after_is_list(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
assert isinstance(expr.after, list), f"{expr.name}: after is not a list"
|
|
|
|
def test_no_self_reference_in_after(self, pipeline):
|
|
"""An expression must not depend on itself."""
|
|
for expr in pipeline.exprs:
|
|
assert expr.name not in expr.after, f"{expr.name} references itself"
|
|
|
|
|
|
# ── 6. Unique names within each pipeline ─────────────────
|
|
|
|
|
|
class TestUniqueNames:
|
|
"""All Expr names within a pipeline must be unique."""
|
|
|
|
def test_no_duplicate_names(self, pipeline):
|
|
names = pipeline.names()
|
|
seen = set()
|
|
dupes = []
|
|
for n in names:
|
|
if n in seen:
|
|
dupes.append(n)
|
|
seen.add(n)
|
|
assert not dupes, f"Duplicate names: {dupes}"
|
|
|
|
def test_names_count_matches_exprs_count(self, pipeline):
|
|
assert len(pipeline.names()) == len(pipeline.exprs)
|
|
|
|
|
|
# ── 7. Pipeline.run / runner basics ──────────────────────
|
|
|
|
|
|
class TestPipelineRunBasics:
|
|
"""Basic behavioral tests for Pipeline.run using the
|
|
real Pipeline class from aco.pipe.base."""
|
|
|
|
def test_pipeline_names_method(self, pipeline):
|
|
names = pipeline.names()
|
|
assert isinstance(names, list)
|
|
assert all(isinstance(n, str) for n in names)
|
|
|
|
def test_pipeline_len(self, pipeline):
|
|
assert len(pipeline) == len(pipeline.exprs)
|
|
|
|
def test_pipeline_exprs_are_expr_instances(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
assert isinstance(expr, Expr), f"Expected Expr, got {type(expr)}"
|
|
|
|
|
|
# ── Cross-pipeline: all exprs across all 12 modules ──────
|
|
|
|
|
|
def _all_exprs():
|
|
"""Collect (module_name, expr) pairs from all modules."""
|
|
pairs = []
|
|
for mod_path in PIPE_MODULES:
|
|
p = _load_pipeline(mod_path)
|
|
short = mod_path.split(".")[-1]
|
|
for expr in p.exprs:
|
|
pairs.append((short, expr))
|
|
return pairs
|
|
|
|
|
|
_ALL_EXPRS = _all_exprs()
|
|
_EXPR_IDS = [f"{mod}::{e.name}" for mod, e in _ALL_EXPRS]
|
|
|
|
|
|
class TestAllExprsAcrossModules:
|
|
"""Validate properties that hold for every single Expr
|
|
across all 12 pipe modules."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"mod_name,expr",
|
|
_ALL_EXPRS,
|
|
ids=_EXPR_IDS,
|
|
)
|
|
def test_expr_name_is_qualified(self, mod_name, expr):
|
|
assert "." in expr.name
|
|
|
|
@pytest.mark.parametrize(
|
|
"mod_name,expr",
|
|
_ALL_EXPRS,
|
|
ids=_EXPR_IDS,
|
|
)
|
|
def test_expr_fn_callable(self, mod_name, expr):
|
|
assert callable(expr.fn)
|
|
|
|
@pytest.mark.parametrize(
|
|
"mod_name,expr",
|
|
_ALL_EXPRS,
|
|
ids=_EXPR_IDS,
|
|
)
|
|
def test_expr_has_inputs(self, mod_name, expr):
|
|
"""Expr.inputs property must not raise."""
|
|
inputs = expr.inputs
|
|
assert isinstance(inputs, list)
|
|
|
|
@pytest.mark.parametrize(
|
|
"mod_name,expr",
|
|
_ALL_EXPRS,
|
|
ids=_EXPR_IDS,
|
|
)
|
|
def test_expr_has_description_or_docstring(self, mod_name, expr):
|
|
"""Every expr should have some documentation."""
|
|
doc = expr.doc
|
|
assert isinstance(doc, str)
|
|
assert len(doc) > 0, f"{expr.name}: no description or docstring"
|
|
|
|
|
|
# ── Expected step counts per module ──────────────────────
|
|
|
|
_EXPECTED_COUNTS = {
|
|
"aco.pipe.core": 12,
|
|
"aco.pipe.readmissions": 12,
|
|
"aco.pipe.ahrq_measures": 39,
|
|
"aco.pipe.pharmacy": 4,
|
|
"aco.pipe.hcc_suspecting": 6,
|
|
"aco.pipe.claims_preprocessing": 63,
|
|
"aco.pipe.data_quality": 1,
|
|
"aco.pipe.cclf": 19,
|
|
"aco.pipe.input_layer": 13,
|
|
"aco.pipe.main": 14,
|
|
"aco.pipe.quality_measures": 15,
|
|
"aco.pipe.provider_attribution": 10,
|
|
}
|
|
|
|
_COUNT_IDS = [m.split(".")[-1] for m in _EXPECTED_COUNTS]
|
|
|
|
|
|
class TestExpectedStepCounts:
|
|
"""Modules with known step counts should match."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"mod_path,expected",
|
|
list(_EXPECTED_COUNTS.items()),
|
|
ids=_COUNT_IDS,
|
|
)
|
|
def test_step_count(self, mod_path, expected):
|
|
p = _load_pipeline(mod_path)
|
|
assert len(p) == expected, (
|
|
f"{mod_path}: expected {expected} steps, got {len(p)}"
|
|
)
|
|
|
|
|
|
# ── Expr.inputs align with _param_to_table ──────────────
|
|
|
|
|
|
class TestInputsParamToTable:
|
|
"""Verify Expr.inputs uses _param_to_table correctly
|
|
for a sample of known expressions."""
|
|
|
|
def test_core_first_expr_inputs(self):
|
|
"""core._stg_claims_member_months has two params."""
|
|
p = _load_pipeline("aco.pipe.core")
|
|
expr = p.exprs[0]
|
|
assert expr.name == "core._stg_claims_member_months"
|
|
inputs = expr.inputs
|
|
assert len(inputs) >= 2
|
|
|
|
def test_readmissions_first_expr_inputs(self):
|
|
p = _load_pipeline("aco.pipe.readmissions")
|
|
expr = p.exprs[0]
|
|
assert expr.name == "readmissions._int_encounter"
|
|
inputs = expr.inputs
|
|
assert "core.encounter" in inputs
|
|
|
|
def test_pharmacy_first_expr_inputs(self):
|
|
p = _load_pipeline("aco.pipe.pharmacy")
|
|
expr = p.exprs[0]
|
|
assert "pharmacy" in expr.name
|
|
inputs = expr.inputs
|
|
assert all("." in i or "_" not in i for i in inputs)
|
|
|
|
|
|
# ── Schema consistency ───────────────────────────────────
|
|
|
|
|
|
class TestSchemaConsistency:
|
|
"""Expr names within a module should use a consistent
|
|
schema prefix (the module's domain)."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"mod_path",
|
|
PIPE_MODULES,
|
|
ids=_IDS,
|
|
)
|
|
def test_single_schema_per_module(self, mod_path):
|
|
"""All exprs in a module share the same schema."""
|
|
p = _load_pipeline(mod_path)
|
|
schemas = {e.name.split(".")[0] for e in p.exprs}
|
|
assert len(schemas) == 1, f"{mod_path}: multiple schemas: {schemas}"
|
|
|
|
@pytest.mark.parametrize(
|
|
"mod_path",
|
|
PIPE_MODULES,
|
|
ids=_IDS,
|
|
)
|
|
def test_schema_matches_module_name(self, mod_path):
|
|
"""The schema used in exprs should match the
|
|
module name (or a known alias)."""
|
|
mod_name = mod_path.split(".")[-1]
|
|
p = _load_pipeline(mod_path)
|
|
schema = p.exprs[0].name.split(".")[0]
|
|
assert schema == mod_name, (
|
|
f"{mod_path}: schema {schema!r} != module {mod_name!r}"
|
|
)
|
|
|
|
|
|
# ── Expr output attribute ────────────────────────────────
|
|
|
|
|
|
class TestExprOutput:
|
|
"""Each Expr should declare an output SQLTable class."""
|
|
|
|
def test_all_exprs_have_output(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
assert expr.output is not None, f"{expr.name}: output is None"
|
|
|
|
def test_output_is_a_class(self, pipeline):
|
|
for expr in pipeline.exprs:
|
|
if expr.output is not None:
|
|
assert isinstance(expr.output, type), (
|
|
f"{expr.name}: output is not a class"
|
|
)
|