4f67b4f9fd
- src/aco/lake/deploy.py: deploy_schemas() creates Iceberg namespaces and tables from SQLTable models via PyIceberg REST Catalog - src/aco/lake/load.py: load_to_iceberg() copies DuckDB tables to Iceberg with append/replace modes - src/cli/lake.py: wire deploy, load, validate commands with --catalog-type (nessie/polaris), --schema, --dry-run, --mode options - Catalog.validate() compares SQLTable definitions vs Iceberg metadata - 20 new tests (type resolution, dry run, mock deploy/load, CLI help) - All 11339 tests pass
235 lines
7.1 KiB
Python
235 lines
7.1 KiB
Python
"""Tests for aco.lake.deploy and aco.lake.load — Iceberg schema/data ops.
|
|
|
|
Tests focus on pure logic (type resolution, schema building) and mock
|
|
external Iceberg/DuckDB connections since those aren't available in CI.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import date, datetime
|
|
from decimal import Decimal
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from aco.lake.deploy import _resolve_iceberg_type
|
|
|
|
|
|
class TestResolveIcebergType:
|
|
def test_str(self):
|
|
assert _resolve_iceberg_type(str) == "string"
|
|
|
|
def test_int(self):
|
|
assert _resolve_iceberg_type(int) == "long"
|
|
|
|
def test_float(self):
|
|
assert _resolve_iceberg_type(float) == "double"
|
|
|
|
def test_bool(self):
|
|
assert _resolve_iceberg_type(bool) == "boolean"
|
|
|
|
def test_date(self):
|
|
assert _resolve_iceberg_type(date) == "date"
|
|
|
|
def test_datetime(self):
|
|
assert _resolve_iceberg_type(datetime) == "timestamp"
|
|
|
|
def test_decimal(self):
|
|
assert _resolve_iceberg_type(Decimal) == "decimal(18,2)"
|
|
|
|
def test_unknown_type_defaults_to_string(self):
|
|
assert _resolve_iceberg_type(bytes) == "string"
|
|
|
|
def test_none_defaults_to_string(self):
|
|
assert _resolve_iceberg_type(None) == "string"
|
|
|
|
def test_union_type_unwraps(self):
|
|
union = str | None
|
|
assert _resolve_iceberg_type(union) == "string"
|
|
|
|
def test_union_int_none(self):
|
|
union = int | None
|
|
assert _resolve_iceberg_type(union) == "long"
|
|
|
|
def test_string_annotation_defaults(self):
|
|
assert _resolve_iceberg_type("SomeForwardRef") == "string"
|
|
|
|
|
|
class TestDeploySchemasDryRun:
|
|
"""Test deploy_schemas with dry_run=True — no Iceberg connection needed."""
|
|
|
|
def test_dry_run_discovers_tables(self):
|
|
from aco.lake.deploy import deploy_schemas
|
|
|
|
# When catalog_uri and warehouse are passed explicitly,
|
|
# cfg is never accessed, so no mock needed
|
|
results = deploy_schemas(
|
|
catalog_uri="http://fake:19120/iceberg/",
|
|
warehouse="s3://fake/",
|
|
dry_run=True,
|
|
schemas=["core"],
|
|
)
|
|
|
|
assert "core" in results
|
|
assert len(results["core"]) > 0
|
|
|
|
def test_dry_run_all_schemas(self):
|
|
from aco.lake.deploy import deploy_schemas
|
|
|
|
results = deploy_schemas(
|
|
catalog_uri="http://fake:19120/iceberg/",
|
|
warehouse="s3://fake/",
|
|
dry_run=True,
|
|
)
|
|
|
|
assert len(results) > 0
|
|
total = sum(len(t) for t in results.values())
|
|
assert total > 0
|
|
|
|
|
|
class TestDeployLive:
|
|
"""Test deploy_schemas with mocked _create_iceberg_table (pyiceberg not installed)."""
|
|
|
|
def test_creates_tables(self):
|
|
from aco.lake.deploy import deploy_schemas
|
|
|
|
created_tables = []
|
|
|
|
def fake_create(cat, table_ref, model):
|
|
created_tables.append(table_ref)
|
|
|
|
with (
|
|
patch("aco.lake.deploy._create_iceberg_table", side_effect=fake_create),
|
|
patch("aco.lake.deploy._ensure_namespace"),
|
|
):
|
|
results = deploy_schemas(
|
|
catalog_uri="http://fake:19120/iceberg/",
|
|
warehouse="s3://fake/",
|
|
schemas=["core"],
|
|
)
|
|
|
|
assert "core" in results
|
|
assert len(results["core"]) > 0
|
|
assert len(created_tables) > 0
|
|
|
|
def test_handles_already_exists(self):
|
|
from aco.lake.deploy import deploy_schemas
|
|
|
|
with (
|
|
patch(
|
|
"aco.lake.deploy._create_iceberg_table",
|
|
side_effect=Exception("Table already exists"),
|
|
),
|
|
patch("aco.lake.deploy._ensure_namespace"),
|
|
):
|
|
results = deploy_schemas(
|
|
catalog_uri="http://fake:19120/iceberg/",
|
|
warehouse="s3://fake/",
|
|
schemas=["core"],
|
|
)
|
|
|
|
assert "core" in results
|
|
assert len(results["core"]) > 0
|
|
|
|
def test_handles_creation_failure(self):
|
|
from aco.lake.deploy import deploy_schemas
|
|
|
|
with (
|
|
patch(
|
|
"aco.lake.deploy._create_iceberg_table",
|
|
side_effect=Exception("Connection refused"),
|
|
),
|
|
patch("aco.lake.deploy._ensure_namespace"),
|
|
):
|
|
results = deploy_schemas(
|
|
catalog_uri="http://fake:19120/iceberg/",
|
|
warehouse="s3://fake/",
|
|
schemas=["core"],
|
|
)
|
|
|
|
assert "core" in results
|
|
assert len(results["core"]) == 0
|
|
|
|
|
|
class TestLoadToIceberg:
|
|
"""Test load_to_iceberg with mocked DuckDB and Iceberg contexts."""
|
|
|
|
def test_loads_tables(self):
|
|
import narwhals as nw
|
|
import polars as pl
|
|
|
|
from aco.lake.load import load_to_iceberg
|
|
|
|
mock_df = nw.from_native(pl.DataFrame({"a": [1, 2, 3]}))
|
|
|
|
mock_duckdb = MagicMock()
|
|
mock_duckdb.load.return_value = mock_df
|
|
mock_iceberg = MagicMock()
|
|
|
|
with (
|
|
patch("conf.cfg") as mock_cfg,
|
|
patch("aco.lake.context.DuckDBContext", return_value=mock_duckdb),
|
|
patch("aco.lake.context.IcebergContext", return_value=mock_iceberg),
|
|
):
|
|
mock_cfg.path.return_value = "/tmp/fake.duckdb"
|
|
|
|
results = load_to_iceberg(
|
|
catalog_uri="http://fake:19120/iceberg/",
|
|
warehouse="s3://fake/",
|
|
schemas=["core"],
|
|
)
|
|
|
|
assert "core" in results
|
|
assert len(results["core"]) > 0
|
|
assert mock_iceberg.save.call_count > 0
|
|
|
|
def test_skips_empty_tables(self):
|
|
import narwhals as nw
|
|
import polars as pl
|
|
|
|
from aco.lake.load import load_to_iceberg
|
|
|
|
empty_df = nw.from_native(pl.DataFrame({"a": pl.Series([], dtype=pl.Int64)}))
|
|
|
|
mock_duckdb = MagicMock()
|
|
mock_duckdb.load.return_value = empty_df
|
|
mock_iceberg = MagicMock()
|
|
|
|
with (
|
|
patch("conf.cfg") as mock_cfg,
|
|
patch("aco.lake.context.DuckDBContext", return_value=mock_duckdb),
|
|
patch("aco.lake.context.IcebergContext", return_value=mock_iceberg),
|
|
):
|
|
mock_cfg.path.return_value = "/tmp/fake.duckdb"
|
|
|
|
results = load_to_iceberg(
|
|
catalog_uri="http://fake:19120/iceberg/",
|
|
warehouse="s3://fake/",
|
|
schemas=["core"],
|
|
)
|
|
|
|
mock_iceberg.save.assert_not_called()
|
|
assert "core" in results
|
|
assert len(results["core"]) == 0
|
|
|
|
def test_handles_load_failure(self):
|
|
from aco.lake.load import load_to_iceberg
|
|
|
|
mock_duckdb = MagicMock()
|
|
mock_duckdb.load.side_effect = RuntimeError("Table not found")
|
|
mock_iceberg = MagicMock()
|
|
|
|
with (
|
|
patch("conf.cfg") as mock_cfg,
|
|
patch("aco.lake.context.DuckDBContext", return_value=mock_duckdb),
|
|
patch("aco.lake.context.IcebergContext", return_value=mock_iceberg),
|
|
):
|
|
mock_cfg.path.return_value = "/tmp/fake.duckdb"
|
|
|
|
results = load_to_iceberg(
|
|
catalog_uri="http://fake:19120/iceberg/",
|
|
warehouse="s3://fake/",
|
|
schemas=["core"],
|
|
)
|
|
|
|
assert "core" in results
|
|
assert len(results["core"]) == 0
|