ec44e937f5
Wire cli/docs.py build/serve/generate commands to run extraction scripts and Docusaurus. Add extract_cli.py (27 CLI docs from typer help), extract_openapi.py (8 REST API docs from FastAPI schema), extract_citations.py (14 pipeline pages with bib references), extract_dag.py (pipeline + table lineage Mermaid diagrams). Closes #26, #27, #28, #29, #30.
93 lines
2.4 KiB
Python
93 lines
2.4 KiB
Python
"""Generate pipeline DAG visualization for docs.
|
|
|
|
Creates a Mermaid diagram of the full lineage graph and writes it
|
|
as a Docusaurus MDX page.
|
|
|
|
Usage::
|
|
|
|
uv run python docs/scripts/extract_dag.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path("src").resolve()))
|
|
|
|
OUT_DIR = Path("docs/docs")
|
|
|
|
|
|
def main() -> None:
|
|
from aco.lake.lineage import build_lineage
|
|
|
|
graph = build_lineage()
|
|
mermaid = graph.to_mermaid()
|
|
|
|
# Also build a simplified pipeline-level DAG from PIPELINES registry
|
|
from aco.pipe import registry
|
|
|
|
pipeline_names = sorted(registry.keys())
|
|
|
|
# Build inter-pipeline dependency edges from lineage
|
|
pipeline_deps: dict[str, set[str]] = {n: set() for n in pipeline_names}
|
|
for table, deps in graph.table_edges.items():
|
|
table_pipeline = table.split(".")[0] if "." in table else table
|
|
for dep in deps:
|
|
dep_pipeline = dep.split(".")[0] if "." in dep else dep
|
|
if (
|
|
table_pipeline != dep_pipeline
|
|
and table_pipeline in pipeline_deps
|
|
and dep_pipeline in pipeline_deps
|
|
):
|
|
pipeline_deps[table_pipeline].add(dep_pipeline)
|
|
|
|
# Pipeline-level Mermaid
|
|
pipeline_mermaid_lines = ["graph LR"]
|
|
for name in pipeline_names:
|
|
safe = name.replace("-", "_")
|
|
pipeline_mermaid_lines.append(f" {safe}[{name}]")
|
|
for name, deps in sorted(pipeline_deps.items()):
|
|
safe_name = name.replace("-", "_")
|
|
for dep in sorted(deps):
|
|
safe_dep = dep.replace("-", "_")
|
|
pipeline_mermaid_lines.append(f" {safe_dep} --> {safe_name}")
|
|
pipeline_mermaid = "\n".join(pipeline_mermaid_lines)
|
|
|
|
content = f"""---
|
|
title: Pipeline DAG
|
|
sidebar_position: 13
|
|
---
|
|
|
|
# Pipeline DAG
|
|
|
|
## Pipeline Dependencies
|
|
|
|
High-level view of how pipelines depend on each other.
|
|
|
|
```mermaid
|
|
{pipeline_mermaid}
|
|
```
|
|
|
|
## Full Table Lineage
|
|
|
|
Detailed table-level dependency graph across all pipelines.
|
|
{len(graph.table_edges)} tables, {sum(len(v) for v in graph.table_edges.values())} edges.
|
|
|
|
```mermaid
|
|
{mermaid}
|
|
```
|
|
"""
|
|
|
|
out_path = OUT_DIR / "dag.md"
|
|
out_path.write_text(content)
|
|
print(f"DAG visualization -> {out_path}")
|
|
print(
|
|
f" {len(pipeline_names)} pipelines, "
|
|
f"{len(graph.table_edges)} tables"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|