e07c8e76e9
- 76 new tests covering CLI commands, API routes, auth, runner, load modules — coverage 99% (169 remaining lines are infrastructure) - Enhanced /health endpoint with DuckDB, bib, pipeline service checks - Added `stack health` CLI command with graceful degradation - SnapshotManager for DuckDB data versioning and rollback - .woodpecker.yml CI pipeline: lint, test (99% gate), marimo check - Error handling audit: no bare except clauses, all exceptions logged Closes #31, #32, #33, #34, #35.
524 lines
19 KiB
Python
524 lines
19 KiB
Python
"""Tests for CLI run, load, lake, validate, bib, generate, api, docs commands."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from typer.testing import CliRunner
|
|
|
|
from cli import app
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
class TestRunCommand:
|
|
def test_unknown_pipeline(self) -> None:
|
|
result = runner.invoke(app, ["run", "nonexistent"])
|
|
assert result.exit_code != 0
|
|
assert "Unknown pipeline" in result.output
|
|
|
|
def test_run_local_default(self) -> None:
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.__len__ = lambda s: 2
|
|
mock_pipeline.run.return_value = {
|
|
"readmissions._int_encounter": MagicMock(__len__=lambda s: 100)
|
|
}
|
|
|
|
mock_ctx = MagicMock()
|
|
with (
|
|
patch.dict(
|
|
"aco.pipe.registry", {"readmissions": mock_pipeline}, clear=True
|
|
),
|
|
patch("cli.run._make_context", return_value=mock_ctx),
|
|
):
|
|
result = runner.invoke(app, ["run", "readmissions"])
|
|
assert result.exit_code == 0
|
|
assert "Done" in result.output
|
|
|
|
def test_run_with_save(self) -> None:
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.__len__ = lambda s: 1
|
|
mock_pipeline.run.return_value = {
|
|
"core.patient": MagicMock(__len__=lambda s: 50)
|
|
}
|
|
|
|
mock_ctx = MagicMock()
|
|
with (
|
|
patch.dict("aco.pipe.registry", {"core": mock_pipeline}, clear=True),
|
|
patch("cli.run._make_context", return_value=mock_ctx),
|
|
patch("cli.run._save_outputs") as mock_save,
|
|
):
|
|
result = runner.invoke(app, ["run", "core", "--save"])
|
|
assert result.exit_code == 0
|
|
mock_save.assert_called_once()
|
|
|
|
|
|
class TestMakeContext:
|
|
def test_local_context(self) -> None:
|
|
from cli.run import _make_context
|
|
|
|
ctx = _make_context("local", read_only=True)
|
|
assert ctx is not None
|
|
|
|
def test_unknown_target(self) -> None:
|
|
result = runner.invoke(app, ["run", "readmissions", "--target", "unknown"])
|
|
assert result.exit_code != 0
|
|
|
|
def test_lake_context(self) -> None:
|
|
from cli.run import _make_context
|
|
|
|
mock_cfg = MagicMock()
|
|
mock_cfg.lake.nessie.catalog_uri = "http://nessie:19120/api/v1"
|
|
mock_cfg.lake.warehouse = "s3://warehouse"
|
|
with (
|
|
patch("conf.cfg", mock_cfg),
|
|
patch("aco.lake.context.IcebergContext") as mock_ice,
|
|
):
|
|
mock_ice.return_value = MagicMock()
|
|
_make_context("lake", read_only=True)
|
|
|
|
def test_trino_context(self) -> None:
|
|
from cli.run import _make_context
|
|
|
|
mock_cfg = MagicMock()
|
|
mock_cfg.lake.trino.host = "localhost"
|
|
mock_cfg.lake.trino.port = 8080
|
|
mock_cfg.lake.trino.catalog = "iceberg"
|
|
with (
|
|
patch("conf.cfg", mock_cfg),
|
|
patch("aco.lake.context.TrinoContext") as mock_trino,
|
|
):
|
|
mock_trino.return_value = MagicMock()
|
|
_make_context("trino", read_only=True)
|
|
|
|
def test_databricks_context(self) -> None:
|
|
from cli.run import _make_context
|
|
|
|
mock_cfg = MagicMock()
|
|
mock_cfg.lake.databricks.catalog_uri = "https://db.example.com"
|
|
mock_cfg.lake.databricks.warehouse = "main"
|
|
with (
|
|
patch("conf.cfg", mock_cfg),
|
|
patch("aco.lake.context.EnterpriseContext") as mock_ent,
|
|
):
|
|
mock_ent.return_value = MagicMock()
|
|
_make_context("databricks", read_only=True)
|
|
|
|
|
|
class TestSaveOutputs:
|
|
def test_save_qualified_names(self) -> None:
|
|
from cli.run import _save_outputs
|
|
|
|
ctx = MagicMock()
|
|
cache = {
|
|
"core.patient": MagicMock(),
|
|
"_internal": MagicMock(),
|
|
"no_schema": MagicMock(),
|
|
}
|
|
_save_outputs(ctx, cache, "core")
|
|
ctx.save.assert_called_once_with(
|
|
"core.patient", cache["core.patient"], mode="replace"
|
|
)
|
|
|
|
|
|
class TestLoadCommands:
|
|
def test_cclf_success(self) -> None:
|
|
with patch(
|
|
"aco.load.cclf.load_cclf_directory", return_value={"cclf.cclf1": 100}
|
|
):
|
|
result = runner.invoke(app, ["load", "cclf", "--path", "/tmp/fake"])
|
|
assert result.exit_code == 0
|
|
assert "100" in result.output
|
|
|
|
def test_cclf_not_found(self) -> None:
|
|
with patch(
|
|
"aco.load.cclf.load_cclf_directory",
|
|
side_effect=FileNotFoundError("no files"),
|
|
):
|
|
result = runner.invoke(app, ["load", "cclf", "--path", "/tmp/fake"])
|
|
assert result.exit_code != 0
|
|
|
|
def test_bcda_success(self) -> None:
|
|
with patch("aco.load.bcda.load_bcda", return_value={"bcda.patient": 50}):
|
|
result = runner.invoke(app, ["load", "bcda", "--ndjson-dir", "/tmp/fake"])
|
|
assert result.exit_code == 0
|
|
|
|
def test_bcda_not_found(self) -> None:
|
|
with patch("aco.load.bcda.load_bcda", side_effect=FileNotFoundError("missing")):
|
|
result = runner.invoke(app, ["load", "bcda", "--ndjson-dir", "/tmp/fake"])
|
|
assert result.exit_code != 0
|
|
|
|
def test_seed_success(self) -> None:
|
|
with patch("aco.load.seed.load_seeds", return_value={"ref.zip": 42000}):
|
|
result = runner.invoke(app, ["load", "seed", "--seed-dir", "/tmp/fake"])
|
|
assert result.exit_code == 0
|
|
|
|
def test_seed_not_found(self) -> None:
|
|
with patch(
|
|
"aco.load.seed.load_seeds", side_effect=FileNotFoundError("no seeds")
|
|
):
|
|
result = runner.invoke(app, ["load", "seed", "--seed-dir", "/tmp/fake"])
|
|
assert result.exit_code != 0
|
|
|
|
|
|
class TestLakeCommands:
|
|
def test_deploy_iceberg(self) -> None:
|
|
with patch(
|
|
"aco.lake.deploy.deploy_schemas",
|
|
return_value={"core": ["core.patient", "core.encounter"]},
|
|
):
|
|
result = runner.invoke(app, ["lake", "deploy"])
|
|
assert result.exit_code == 0
|
|
assert "2 tables" in result.output
|
|
|
|
def test_deploy_iceberg_dry_run(self) -> None:
|
|
with patch(
|
|
"aco.lake.deploy.deploy_schemas",
|
|
return_value={"core": ["core.patient"]},
|
|
):
|
|
result = runner.invoke(app, ["lake", "deploy", "--dry-run"])
|
|
assert result.exit_code == 0
|
|
assert "Would create" in result.output
|
|
|
|
def test_deploy_databricks(self) -> None:
|
|
mock_client = MagicMock()
|
|
with (
|
|
patch("aco.lake.unity.UnityClient.from_env", return_value=mock_client),
|
|
patch(
|
|
"aco.lake.unity.setup_catalog_from_schemas",
|
|
return_value={"tables_created": ["t1"], "errors": []},
|
|
),
|
|
patch("conf.cfg") as mock_cfg,
|
|
):
|
|
mock_cfg.lake.databricks.catalog = "dev"
|
|
mock_cfg.lake.databricks.get.return_value = ""
|
|
result = runner.invoke(app, ["lake", "deploy", "--target", "databricks"])
|
|
assert result.exit_code == 0
|
|
|
|
def test_deploy_databricks_errors(self) -> None:
|
|
mock_client = MagicMock()
|
|
with (
|
|
patch("aco.lake.unity.UnityClient.from_env", return_value=mock_client),
|
|
patch(
|
|
"aco.lake.unity.setup_catalog_from_schemas",
|
|
return_value={"tables_created": [], "errors": ["e1"]},
|
|
),
|
|
patch("conf.cfg") as mock_cfg,
|
|
):
|
|
mock_cfg.lake.databricks.catalog = "dev"
|
|
mock_cfg.lake.databricks.get.return_value = ""
|
|
result = runner.invoke(app, ["lake", "deploy", "--target", "databricks"])
|
|
assert result.exit_code != 0
|
|
|
|
def test_load(self) -> None:
|
|
with patch(
|
|
"aco.lake.load.load_to_iceberg",
|
|
return_value={"core": ["core.patient"]},
|
|
):
|
|
result = runner.invoke(app, ["lake", "load"])
|
|
assert result.exit_code == 0
|
|
assert "Loaded" in result.output
|
|
|
|
def test_validate_ok(self) -> None:
|
|
mock_cat = MagicMock()
|
|
mock_cat.schemas.return_value = ["core"]
|
|
mock_cat.validate.return_value = {
|
|
"missing_in_iceberg": [],
|
|
"missing_in_schema": [],
|
|
"column_mismatches": {},
|
|
}
|
|
with (
|
|
patch("aco.lake.catalog.Catalog", return_value=mock_cat),
|
|
patch("conf.cfg") as mock_cfg,
|
|
):
|
|
mock_cfg.lake.nessie.catalog_uri = "http://nessie:19120"
|
|
mock_cfg.lake.warehouse = "s3://wh"
|
|
result = runner.invoke(app, ["lake", "validate"])
|
|
assert result.exit_code == 0
|
|
assert "valid" in result.output
|
|
|
|
def test_validate_issues(self) -> None:
|
|
mock_cat = MagicMock()
|
|
mock_cat.schemas.return_value = ["core"]
|
|
mock_cat.validate.return_value = {
|
|
"missing_in_iceberg": ["core.patient"],
|
|
"missing_in_schema": [],
|
|
"column_mismatches": {"core.encounter": "missing col x"},
|
|
}
|
|
with (
|
|
patch("aco.lake.catalog.Catalog", return_value=mock_cat),
|
|
patch("conf.cfg") as mock_cfg,
|
|
):
|
|
mock_cfg.lake.nessie.catalog_uri = "http://nessie:19120"
|
|
mock_cfg.lake.warehouse = "s3://wh"
|
|
result = runner.invoke(app, ["lake", "validate"])
|
|
assert result.exit_code != 0
|
|
|
|
def test_parity_ok(self) -> None:
|
|
mock_model = MagicMock()
|
|
mock_model.model_fields = {"col_a": None, "col_b": None}
|
|
|
|
mock_cat = MagicMock()
|
|
mock_cat.schemas.return_value = ["core"]
|
|
mock_cat.tables.return_value = ["core.t1"]
|
|
mock_cat.model.return_value = mock_model
|
|
|
|
mock_df = MagicMock()
|
|
mock_df.columns = ["col_a", "col_b"]
|
|
mock_df.__len__ = lambda s: 5
|
|
|
|
mock_ctx = MagicMock()
|
|
mock_ctx.load.return_value = mock_df
|
|
|
|
with (
|
|
patch("aco.lake.catalog.Catalog", return_value=mock_cat),
|
|
patch("aco.lake.context.DuckDBContext", return_value=mock_ctx),
|
|
patch("conf.path") as mock_path,
|
|
):
|
|
mock_path.return_value = "/tmp/fake.duckdb"
|
|
result = runner.invoke(app, ["lake", "parity"])
|
|
assert result.exit_code == 0
|
|
assert "matching" in result.output
|
|
|
|
def test_parity_mismatch(self) -> None:
|
|
mock_model = MagicMock()
|
|
mock_model.model_fields = {"col_a": None, "col_b": None}
|
|
|
|
mock_cat = MagicMock()
|
|
mock_cat.schemas.return_value = ["core"]
|
|
mock_cat.tables.return_value = ["core.t1"]
|
|
mock_cat.model.return_value = mock_model
|
|
|
|
mock_df = MagicMock()
|
|
mock_df.columns = ["col_a", "col_c"]
|
|
mock_df.__len__ = lambda s: 5
|
|
|
|
mock_ctx = MagicMock()
|
|
mock_ctx.load.return_value = mock_df
|
|
|
|
with (
|
|
patch("aco.lake.catalog.Catalog", return_value=mock_cat),
|
|
patch("aco.lake.context.DuckDBContext", return_value=mock_ctx),
|
|
patch("conf.path") as mock_path,
|
|
):
|
|
mock_path.return_value = "/tmp/fake.duckdb"
|
|
result = runner.invoke(app, ["lake", "parity"])
|
|
assert result.exit_code != 0
|
|
assert "mismatch" in result.output
|
|
|
|
|
|
class TestLakeValidateEdgeCases:
|
|
def test_validate_polaris(self) -> None:
|
|
mock_cat = MagicMock()
|
|
mock_cat.schemas.return_value = ["core"]
|
|
mock_cat.validate.return_value = {
|
|
"missing_in_iceberg": [],
|
|
"missing_in_schema": ["core.extra"],
|
|
"column_mismatches": {},
|
|
}
|
|
with (
|
|
patch("aco.lake.catalog.Catalog", return_value=mock_cat),
|
|
patch("conf.cfg") as mock_cfg,
|
|
):
|
|
mock_cfg.lake.polaris.catalog_uri = "http://polaris:8181"
|
|
mock_cfg.lake.warehouse = "s3://wh"
|
|
result = runner.invoke(
|
|
app, ["lake", "validate", "--catalog-type", "polaris"]
|
|
)
|
|
assert result.exit_code != 0
|
|
assert "extra in Iceberg" in result.output
|
|
|
|
def test_parity_error(self) -> None:
|
|
mock_cat = MagicMock()
|
|
mock_cat.schemas.return_value = ["core"]
|
|
mock_cat.tables.return_value = ["core.t1"]
|
|
mock_cat.model.return_value = MagicMock(model_fields={"a": None})
|
|
|
|
mock_ctx = MagicMock()
|
|
mock_ctx.load.side_effect = Exception("table missing")
|
|
|
|
with (
|
|
patch("aco.lake.catalog.Catalog", return_value=mock_cat),
|
|
patch("aco.lake.context.DuckDBContext", return_value=mock_ctx),
|
|
patch("conf.path") as mock_path,
|
|
):
|
|
mock_path.return_value = "/tmp/fake.duckdb"
|
|
result = runner.invoke(app, ["lake", "parity"])
|
|
assert result.exit_code != 0
|
|
assert "ERROR" in result.output
|
|
|
|
|
|
class TestValidateCommand:
|
|
def test_validate_unknown(self) -> None:
|
|
result = runner.invoke(app, ["validate", "nonexistent"])
|
|
assert result.exit_code != 0
|
|
|
|
def test_validate_pipeline(self) -> None:
|
|
|
|
mock_pipe = MagicMock()
|
|
mock_pipe.__len__ = lambda s: 3
|
|
mock_pipe.run.return_value = {}
|
|
|
|
with (
|
|
patch.dict("aco.pipe.registry", {"readmissions": mock_pipe}, clear=True),
|
|
patch("cli.run._make_context") as mock_ctx,
|
|
):
|
|
mock_ctx.return_value = MagicMock()
|
|
result = runner.invoke(app, ["validate", "readmissions"])
|
|
assert "1 passed" in result.output
|
|
|
|
def test_validate_schema_error(self) -> None:
|
|
from aco.pipe.runner import SchemaError
|
|
|
|
mock_pipe = MagicMock()
|
|
mock_pipe.__len__ = lambda s: 3
|
|
mock_pipe.run.side_effect = SchemaError("bad columns")
|
|
|
|
with (
|
|
patch.dict("aco.pipe.registry", {"core": mock_pipe}, clear=True),
|
|
patch("cli.run._make_context") as mock_ctx,
|
|
):
|
|
mock_ctx.return_value = MagicMock()
|
|
result = runner.invoke(app, ["validate", "core"])
|
|
assert "1 failed" in result.output
|
|
|
|
|
|
class TestBibCommands:
|
|
def test_sync_no_tags(self) -> None:
|
|
mock_store = MagicMock()
|
|
with (
|
|
patch("bib.connect", return_value=mock_store),
|
|
patch("bib.meta.collect_column_comments", return_value={}),
|
|
):
|
|
result = runner.invoke(app, ["bib", "sync"])
|
|
assert result.exit_code == 0
|
|
assert "No col: tags" in result.output
|
|
|
|
def test_sync_dry_run(self) -> None:
|
|
mock_store = MagicMock()
|
|
comments = {"core.patient.patient_id": "Unique patient ID"}
|
|
with (
|
|
patch("bib.connect", return_value=mock_store),
|
|
patch("bib.meta.collect_column_comments", return_value=comments),
|
|
):
|
|
result = runner.invoke(app, ["bib", "sync", "--dry-run"])
|
|
assert result.exit_code == 0
|
|
assert "Would apply" in result.output
|
|
|
|
def test_sync_apply(self) -> None:
|
|
mock_store = MagicMock()
|
|
comments = {"core.patient.patient_id": "Unique patient ID"}
|
|
with (
|
|
patch("bib.connect", return_value=mock_store),
|
|
patch("bib.meta.collect_column_comments", return_value=comments),
|
|
patch("duckdb.connect"),
|
|
patch("conf.path", return_value="/tmp/fake.duckdb"),
|
|
patch("bib.meta.apply_column_comments", return_value=["stmt1"]),
|
|
):
|
|
result = runner.invoke(app, ["bib", "sync"])
|
|
assert result.exit_code == 0
|
|
assert "Applied 1" in result.output
|
|
|
|
|
|
class TestGenerateCommand:
|
|
def test_generate_no_sync(self, tmp_path) -> None:
|
|
|
|
def mock_path(key):
|
|
if key == "db.aco":
|
|
return tmp_path / "aco.duckdb"
|
|
if key == "generate.table_out":
|
|
return tmp_path / "table"
|
|
return tmp_path / key
|
|
|
|
(tmp_path / "table").mkdir()
|
|
|
|
with (
|
|
patch("duckdb.connect"),
|
|
patch("conf.path", side_effect=mock_path),
|
|
patch("conf.cfg") as mock_cfg,
|
|
patch("dev.scripts.generate_models.get_schemas", return_value=["core"]),
|
|
patch(
|
|
"dev.scripts.generate_models.generate_schema_module",
|
|
return_value="# generated\n",
|
|
),
|
|
patch("dev.scripts.generate_models.get_tables", return_value=["t1", "t2"]),
|
|
):
|
|
mock_cfg.generate.base_import = "aco.table.base"
|
|
result = runner.invoke(app, ["generate", "models", "--no-sync"])
|
|
assert result.exit_code == 0
|
|
assert "Generated" in result.output
|
|
|
|
|
|
class TestDocsCommands:
|
|
def test_generate_only(self) -> None:
|
|
with patch("subprocess.run") as mock_run:
|
|
mock_run.return_value = MagicMock(returncode=0)
|
|
result = runner.invoke(app, ["docs", "generate"])
|
|
assert result.exit_code == 0
|
|
assert "Generated" in result.output
|
|
|
|
def test_build_skip_generate(self) -> None:
|
|
with patch("subprocess.run") as mock_run:
|
|
mock_run.return_value = MagicMock(returncode=0)
|
|
result = runner.invoke(app, ["docs", "build", "--skip-generate"])
|
|
assert "Building" in result.output
|
|
|
|
def test_build_npm_fail(self) -> None:
|
|
with patch("subprocess.run") as mock_run:
|
|
mock_run.return_value = MagicMock(returncode=1)
|
|
result = runner.invoke(app, ["docs", "build", "--skip-generate"])
|
|
assert result.exit_code != 0
|
|
|
|
def test_serve_skip_generate(self) -> None:
|
|
with patch("subprocess.run") as mock_run:
|
|
mock_run.return_value = MagicMock(returncode=0)
|
|
result = runner.invoke(
|
|
app, ["docs", "serve", "--skip-generate", "--port", "9000"]
|
|
)
|
|
assert "port=9000" in result.output
|
|
|
|
|
|
class TestGenerateWithSync:
|
|
def test_generate_with_sync(self, tmp_path) -> None:
|
|
def mock_path(key):
|
|
if key == "db.aco":
|
|
return tmp_path / "aco.duckdb"
|
|
if key == "generate.table_out":
|
|
return tmp_path / "table"
|
|
return tmp_path / key
|
|
|
|
(tmp_path / "table").mkdir()
|
|
|
|
with (
|
|
patch("duckdb.connect"),
|
|
patch("conf.path", side_effect=mock_path),
|
|
patch("conf.cfg") as mock_cfg,
|
|
patch("bib.connect"),
|
|
patch("bib.meta.collect_column_comments", return_value={"c": "d"}),
|
|
patch("bib.meta.apply_column_comments", return_value=["s1"]),
|
|
patch("dev.scripts.generate_models.get_schemas", return_value=["core"]),
|
|
patch(
|
|
"dev.scripts.generate_models.generate_schema_module",
|
|
return_value="# gen\n",
|
|
),
|
|
patch("dev.scripts.generate_models.get_tables", return_value=["t1"]),
|
|
):
|
|
mock_cfg.generate.base_import = "aco.table.base"
|
|
result = runner.invoke(app, ["generate", "models", "--sync"])
|
|
assert result.exit_code == 0
|
|
assert "Applied 1" in result.output
|
|
|
|
|
|
class TestApiCommand:
|
|
def test_api_serve(self) -> None:
|
|
with (
|
|
patch("uvicorn.run") as mock_run,
|
|
patch("conf.cfg") as mock_cfg,
|
|
):
|
|
mock_cfg.api.host = "0.0.0.0"
|
|
mock_cfg.api.port = 8080
|
|
mock_cfg.api.workers = 1
|
|
result = runner.invoke(app, ["api", "serve"])
|
|
assert result.exit_code == 0
|
|
mock_run.assert_called_once()
|