remove DDLPlugin — CTAS statements in SQL files already create tables
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -8,9 +8,8 @@ zotero/profiles/
|
|||||||
tuva/
|
tuva/
|
||||||
logs/**
|
logs/**
|
||||||
|
|
||||||
# Generated DAB SQL/DDL (regenerate via gen_config.py)
|
# Generated DAB SQL (regenerate via gen_config.py --dab-sql)
|
||||||
bundle/sql/
|
bundle/sql/
|
||||||
bundle/ddl/
|
|
||||||
|
|
||||||
# Marimo cache
|
# Marimo cache
|
||||||
notebooks/__marimo__/
|
notebooks/__marimo__/
|
||||||
|
|||||||
@@ -88,37 +88,6 @@ class SQLPlugin(DabPlugin):
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
class DDLPlugin(DabPlugin):
|
|
||||||
"""Generate CREATE TABLE DDL from SQLTable models for each output."""
|
|
||||||
|
|
||||||
def files(self, cfg: dict, registry: dict) -> dict[str, str]:
|
|
||||||
from aco.lake.catalog import Catalog
|
|
||||||
|
|
||||||
cat = Catalog()
|
|
||||||
result: dict[str, str] = {}
|
|
||||||
|
|
||||||
for schema_name in cat.schemas():
|
|
||||||
for table_ref in cat.tables(schema_name):
|
|
||||||
model = cat.model(table_ref)
|
|
||||||
schema, table = table_ref.split(".", 1)
|
|
||||||
|
|
||||||
# Generate DDL with catalog variable
|
|
||||||
ddl = model.to_ddl()
|
|
||||||
# Rewrite the table ref to include catalog variable
|
|
||||||
ddl = ddl.replace(
|
|
||||||
f"CREATE TABLE {table_ref}",
|
|
||||||
f"CREATE TABLE IF NOT EXISTS ${{var.catalog}}.{table_ref}",
|
|
||||||
)
|
|
||||||
path = f"bundle/ddl/{schema}/{table}.sql"
|
|
||||||
result[path] = (
|
|
||||||
f"-- {table_ref}\n"
|
|
||||||
f"-- Generated by gen_config.py — DO NOT EDIT\n\n"
|
|
||||||
f"{ddl}\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
class JobsPlugin(DabPlugin):
|
class JobsPlugin(DabPlugin):
|
||||||
"""Generate jobs with sql_task references to transpiled SQL files."""
|
"""Generate jobs with sql_task references to transpiled SQL files."""
|
||||||
|
|
||||||
@@ -236,7 +205,6 @@ _HEADER = (
|
|||||||
|
|
||||||
_DEFAULT_PLUGINS: list[DabPlugin] = [
|
_DEFAULT_PLUGINS: list[DabPlugin] = [
|
||||||
SQLPlugin(),
|
SQLPlugin(),
|
||||||
DDLPlugin(),
|
|
||||||
JobsPlugin(),
|
JobsPlugin(),
|
||||||
SchemasPlugin(),
|
SchemasPlugin(),
|
||||||
VolumesPlugin(),
|
VolumesPlugin(),
|
||||||
|
|||||||
@@ -166,28 +166,6 @@ class TestSQLFiles:
|
|||||||
assert has_catalog_ref, "No SQL files reference ${var.catalog}"
|
assert has_catalog_ref, "No SQL files reference ${var.catalog}"
|
||||||
|
|
||||||
|
|
||||||
class TestDDLFiles:
|
|
||||||
"""DDL generation from SQLTable models."""
|
|
||||||
|
|
||||||
def _emit(self) -> dict[str, str]:
|
|
||||||
from backends.databricks import emit
|
|
||||||
|
|
||||||
from conf import cfg
|
|
||||||
|
|
||||||
return emit(cfg._data)
|
|
||||||
|
|
||||||
def test_ddl_files_generated(self):
|
|
||||||
files = self._emit()
|
|
||||||
ddl_files = [k for k in files if k.startswith("bundle/ddl/")]
|
|
||||||
assert len(ddl_files) > 100, f"Only {len(ddl_files)} DDL files generated"
|
|
||||||
|
|
||||||
def test_ddl_contains_create_table(self):
|
|
||||||
files = self._emit()
|
|
||||||
ddl_files = [k for k in files if k.startswith("bundle/ddl/")]
|
|
||||||
for path in ddl_files[:10]:
|
|
||||||
assert "CREATE TABLE" in files[path], f"{path} missing CREATE TABLE"
|
|
||||||
|
|
||||||
|
|
||||||
class TestTargets:
|
class TestTargets:
|
||||||
"""Target isolation."""
|
"""Target isolation."""
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user