remove DDLPlugin — CTAS statements in SQL files already create tables
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -8,9 +8,8 @@ zotero/profiles/
|
||||
tuva/
|
||||
logs/**
|
||||
|
||||
# Generated DAB SQL/DDL (regenerate via gen_config.py)
|
||||
# Generated DAB SQL (regenerate via gen_config.py --dab-sql)
|
||||
bundle/sql/
|
||||
bundle/ddl/
|
||||
|
||||
# Marimo cache
|
||||
notebooks/__marimo__/
|
||||
|
||||
@@ -88,37 +88,6 @@ class SQLPlugin(DabPlugin):
|
||||
return result
|
||||
|
||||
|
||||
class DDLPlugin(DabPlugin):
|
||||
"""Generate CREATE TABLE DDL from SQLTable models for each output."""
|
||||
|
||||
def files(self, cfg: dict, registry: dict) -> dict[str, str]:
|
||||
from aco.lake.catalog import Catalog
|
||||
|
||||
cat = Catalog()
|
||||
result: dict[str, str] = {}
|
||||
|
||||
for schema_name in cat.schemas():
|
||||
for table_ref in cat.tables(schema_name):
|
||||
model = cat.model(table_ref)
|
||||
schema, table = table_ref.split(".", 1)
|
||||
|
||||
# Generate DDL with catalog variable
|
||||
ddl = model.to_ddl()
|
||||
# Rewrite the table ref to include catalog variable
|
||||
ddl = ddl.replace(
|
||||
f"CREATE TABLE {table_ref}",
|
||||
f"CREATE TABLE IF NOT EXISTS ${{var.catalog}}.{table_ref}",
|
||||
)
|
||||
path = f"bundle/ddl/{schema}/{table}.sql"
|
||||
result[path] = (
|
||||
f"-- {table_ref}\n"
|
||||
f"-- Generated by gen_config.py — DO NOT EDIT\n\n"
|
||||
f"{ddl}\n"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class JobsPlugin(DabPlugin):
|
||||
"""Generate jobs with sql_task references to transpiled SQL files."""
|
||||
|
||||
@@ -236,7 +205,6 @@ _HEADER = (
|
||||
|
||||
_DEFAULT_PLUGINS: list[DabPlugin] = [
|
||||
SQLPlugin(),
|
||||
DDLPlugin(),
|
||||
JobsPlugin(),
|
||||
SchemasPlugin(),
|
||||
VolumesPlugin(),
|
||||
|
||||
@@ -166,28 +166,6 @@ class TestSQLFiles:
|
||||
assert has_catalog_ref, "No SQL files reference ${var.catalog}"
|
||||
|
||||
|
||||
class TestDDLFiles:
|
||||
"""DDL generation from SQLTable models."""
|
||||
|
||||
def _emit(self) -> dict[str, str]:
|
||||
from backends.databricks import emit
|
||||
|
||||
from conf import cfg
|
||||
|
||||
return emit(cfg._data)
|
||||
|
||||
def test_ddl_files_generated(self):
|
||||
files = self._emit()
|
||||
ddl_files = [k for k in files if k.startswith("bundle/ddl/")]
|
||||
assert len(ddl_files) > 100, f"Only {len(ddl_files)} DDL files generated"
|
||||
|
||||
def test_ddl_contains_create_table(self):
|
||||
files = self._emit()
|
||||
ddl_files = [k for k in files if k.startswith("bundle/ddl/")]
|
||||
for path in ddl_files[:10]:
|
||||
assert "CREATE TABLE" in files[path], f"{path} missing CREATE TABLE"
|
||||
|
||||
|
||||
class TestTargets:
|
||||
"""Target isolation."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user