8047640c3e
- stack run <name> --target local|lake|trino|databricks --save: execute any
of 13 registered pipelines with context-aware backend selection
- stack validate [pipeline]: run pipelines and verify output contracts
against SQLTable models, reports mismatches with lineage context
- Pipeline registry: 13 pipelines auto-registered in aco.pipe.__init__
- Idempotent reruns: fingerprint-based change detection skips unchanged steps
- SchemaError now includes function name, input tables, contract class,
and column count for debugging
- POST /run/{pipeline} with async thread execution and job polling
- GET /pipelines/{name} returns inputs/outputs from Expr dependency graph
- GET /run/{job_id}/status with running/completed/failed states
fixes #9 fixes #10 fixes #11 fixes #12 fixes #13
62 lines
1.8 KiB
Python
62 lines
1.8 KiB
Python
"""stack validate — schema validation across all pipelines."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import typer
|
|
|
|
|
|
def validate(
|
|
pipeline: str = typer.Argument(
|
|
"", help="Pipeline name to validate (all if omitted)."
|
|
),
|
|
target: str = typer.Option(
|
|
"local", help="Target context: local, lake, trino, databricks."
|
|
),
|
|
) -> None:
|
|
"""Verify pipeline output contracts against SQLTable models.
|
|
|
|
Runs each pipeline and checks that every output DataFrame matches
|
|
the expected columns from its SQLTable contract. Reports mismatches
|
|
without saving anything.
|
|
"""
|
|
from aco.pipe import registry
|
|
from aco.pipe.runner import SchemaError
|
|
|
|
if pipeline and pipeline not in registry:
|
|
names = ", ".join(sorted(registry.keys()))
|
|
typer.echo(f"Unknown pipeline '{pipeline}'. Available: {names}", err=True)
|
|
raise typer.Exit(1)
|
|
|
|
pipelines = {pipeline: registry[pipeline]} if pipeline else registry
|
|
|
|
from cli.run import _make_context
|
|
|
|
ctx = _make_context(target, read_only=True)
|
|
|
|
passed = 0
|
|
failed = 0
|
|
errors: list[str] = []
|
|
|
|
for name, pipe in sorted(pipelines.items()):
|
|
typer.echo(f"Validating {name} ({len(pipe)} steps) ...", nl=False)
|
|
try:
|
|
pipe.run(ctx.load)
|
|
typer.echo(" OK")
|
|
passed += 1
|
|
except SchemaError as e:
|
|
typer.echo(" FAIL")
|
|
errors.append(f"{name}: {e}")
|
|
failed += 1
|
|
except Exception as e:
|
|
typer.echo(" ERROR")
|
|
errors.append(f"{name}: {type(e).__name__}: {e}")
|
|
failed += 1
|
|
|
|
typer.echo(f"\n{passed} passed, {failed} failed")
|
|
|
|
if errors:
|
|
typer.echo("\nFailures:")
|
|
for err in errors:
|
|
typer.echo(f" {err}")
|
|
raise typer.Exit(1)
|