update sql_generator notebook: schema-only mode, all 12 pipelines
- Switch from DuckDB file to schema-only transpilation (no data needed) - Add cclf pipeline (previously excluded due to transpile errors) - Include cclf in summary table - Remove try/except wrapper and hardcoded DB path
This commit is contained in:
@@ -20,21 +20,17 @@ def _(mo):
|
|||||||
narwhals express functions against DuckDB relations and extracting the SQL
|
narwhals express functions against DuckDB relations and extracting the SQL
|
||||||
via `.sql_query()`, then transpiling with sqlglot.
|
via `.sql_query()`, then transpiling with sqlglot.
|
||||||
|
|
||||||
Each cell below transpiles one pipeline module and displays the generated
|
Uses schema-only DuckDB (no data file needed). Each cell below transpiles
|
||||||
`CREATE OR REPLACE TABLE` statements.
|
one pipeline module and displays the generated `CREATE OR REPLACE TABLE`
|
||||||
|
statements.
|
||||||
""")
|
""")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
@app.cell(hide_code=True)
|
@app.cell(hide_code=True)
|
||||||
def _():
|
def _():
|
||||||
import duckdb
|
|
||||||
|
|
||||||
from aco.lake.transpile import transpile
|
from aco.lake.transpile import transpile
|
||||||
|
|
||||||
DB_PATH = "/home/kert/notebooks/aco.duckdb"
|
|
||||||
con = duckdb.connect(DB_PATH, read_only=True)
|
|
||||||
|
|
||||||
DIALECT = "databricks"
|
DIALECT = "databricks"
|
||||||
CATALOG = "homelab"
|
CATALOG = "homelab"
|
||||||
|
|
||||||
@@ -42,7 +38,6 @@ def _():
|
|||||||
"""Transpile a pipeline and return formatted SQL."""
|
"""Transpile a pipeline and return formatted SQL."""
|
||||||
result = transpile(
|
result = transpile(
|
||||||
pipeline,
|
pipeline,
|
||||||
con,
|
|
||||||
target_dialect=DIALECT,
|
target_dialect=DIALECT,
|
||||||
catalog=CATALOG,
|
catalog=CATALOG,
|
||||||
output_mode="ctas",
|
output_mode="ctas",
|
||||||
@@ -56,7 +51,7 @@ def _():
|
|||||||
header = f"/* {label}: {len(result)} steps, {errors} errors */\n"
|
header = f"/* {label}: {len(result)} steps, {errors} errors */\n"
|
||||||
return header + "\n\n".join(parts), len(result), errors
|
return header + "\n\n".join(parts), len(result), errors
|
||||||
|
|
||||||
return con, run_transpile
|
return (run_transpile,)
|
||||||
|
|
||||||
|
|
||||||
@app.cell(hide_code=True)
|
@app.cell(hide_code=True)
|
||||||
@@ -241,13 +236,8 @@ def _(mo):
|
|||||||
def _(mo, run_transpile):
|
def _(mo, run_transpile):
|
||||||
from aco.pipe import cclf as _cclf
|
from aco.pipe import cclf as _cclf
|
||||||
|
|
||||||
try:
|
|
||||||
_sql, _n, _e = run_transpile(_cclf.pipeline, "cclf")
|
_sql, _n, _e = run_transpile(_cclf.pipeline, "cclf")
|
||||||
_msg = f"**{_n} steps, {_e} errors**\n\n```sql\n{_sql}\n```"
|
mo.md(f"**{_n} steps, {_e} errors**\n\n```sql\n{_sql}\n```")
|
||||||
except Exception as _ex:
|
|
||||||
_msg = f"**CCLF pipeline requires CCLF data in DuckDB.**\n\nError: `{_ex}`"
|
|
||||||
|
|
||||||
mo.md(_msg)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
@@ -278,10 +268,10 @@ def _(mo):
|
|||||||
|
|
||||||
|
|
||||||
@app.cell
|
@app.cell
|
||||||
def _(con, mo):
|
def _(mo, run_transpile):
|
||||||
from aco.lake.transpile import transpile as _transpile
|
|
||||||
from aco.pipe import (
|
from aco.pipe import (
|
||||||
ahrq_measures,
|
ahrq_measures,
|
||||||
|
cclf,
|
||||||
claims_preprocessing,
|
claims_preprocessing,
|
||||||
core,
|
core,
|
||||||
data_quality,
|
data_quality,
|
||||||
@@ -305,6 +295,7 @@ def _(con, mo):
|
|||||||
("quality_measures", quality_measures),
|
("quality_measures", quality_measures),
|
||||||
("hcc_suspecting", hcc_suspecting),
|
("hcc_suspecting", hcc_suspecting),
|
||||||
("provider_attribution", provider_attribution),
|
("provider_attribution", provider_attribution),
|
||||||
|
("cclf", cclf),
|
||||||
("main", main),
|
("main", main),
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -312,13 +303,10 @@ def _(con, mo):
|
|||||||
_total_steps = 0
|
_total_steps = 0
|
||||||
_total_errors = 0
|
_total_errors = 0
|
||||||
for _name, _mod in _modules:
|
for _name, _mod in _modules:
|
||||||
_result = _transpile(
|
_, _n, _e = run_transpile(_mod.pipeline, _name)
|
||||||
_mod.pipeline, con, target_dialect="databricks", catalog="homelab"
|
_total_steps += _n
|
||||||
)
|
_total_errors += _e
|
||||||
_errs = sum(1 for v in _result.values() if v.startswith("-- ERROR"))
|
_rows.append(f"| {_name} | {_n} | {_e} |")
|
||||||
_total_steps += len(_result)
|
|
||||||
_total_errors += _errs
|
|
||||||
_rows.append(f"| {_name} | {len(_result)} | {_errs} |")
|
|
||||||
|
|
||||||
_table = "| Pipeline | Steps | Errors |\n|----------|-------|--------|\n"
|
_table = "| Pipeline | Steps | Errors |\n|----------|-------|--------|\n"
|
||||||
_table += "\n".join(_rows)
|
_table += "\n".join(_rows)
|
||||||
|
|||||||
Reference in New Issue
Block a user