add Docusaurus docs generation: API reference, CLI, REST API, citations, DAG viz

Wire cli/docs.py build/serve/generate commands to run extraction scripts
and Docusaurus. Add extract_cli.py (27 CLI docs from typer help),
extract_openapi.py (8 REST API docs from FastAPI schema),
extract_citations.py (14 pipeline pages with bib references),
extract_dag.py (pipeline + table lineage Mermaid diagrams).
Closes #26, #27, #28, #29, #30.

docs/scripts/extract_citations.py

@@ -0,0 +1,126 @@
+"""Extract Zotero-sourced regulatory citations for pipeline docs.
+
+For each pipeline, finds bib items tagged with the pipeline's module
+namespace and generates a citations page with linked references.
+
+Usage::
+
+    uv run python docs/scripts/extract_citations.py
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path("src").resolve()))
+
+OUT_DIR = Path("docs/docs/pipelines")
+
+
+def main() -> None:
+    from conf import path as _conf_path
+
+    bib_db = _conf_path("db.bib")
+    if not bib_db.exists():
+        print("bib.sqlite not found, skipping citations")
+        return
+
+    from aco.pipe import registry
+    from bib.store import Store
+
+    store = Store(str(bib_db))
+
+    if OUT_DIR.exists():
+        import shutil
+
+        shutil.rmtree(OUT_DIR)
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    cat_json = OUT_DIR / "_category_.json"
+    cat_json.write_text('{"label": "Pipelines", "position": 12}\n')
+
+    count = 0
+
+    # Overview page
+    overview = [
+        "---\ntitle: Pipelines\nsidebar_position: 1\n---\n",
+        "# Pipelines\n",
+        "| Pipeline | Steps | Description |",
+        "|----------|-------|-------------|",
+    ]
+    for name, pipeline in sorted(registry.items()):
+        overview.append(f"| [{name}](./{name}) | {len(pipeline)} | |")
+    overview.append("")
+    (OUT_DIR / "index.md").write_text("\n".join(overview))
+    count += 1
+
+    # Per-pipeline pages with citations
+    all_items = store.list_items()
+
+    for pos, (name, pipeline) in enumerate(sorted(registry.items()), start=2):
+        lines = [
+            f"---\ntitle: {name}\nsidebar_position: {pos}\n---\n",
+            f"# {name}\n",
+            f"**Steps:** {len(pipeline)}\n",
+        ]
+
+        # Step list
+        lines.append("## Steps\n")
+        for i, expr in enumerate(pipeline.exprs, 1):
+            doc = expr.fn.__doc__ or ""
+            doc_line = doc.strip().split("\n")[0] if doc else ""
+            lines.append(f"{i}. **{expr.name}**{f' — {doc_line}' if doc_line else ''}")
+        lines.append("")
+
+        # Find citations: items tagged with module:{pipeline_name} or
+        # related tags
+        module_tag = f"module:{name}"
+        cited_items = [
+            item for item in all_items if module_tag in item.tags
+        ]
+
+        # Also check for aco-related tags
+        if not cited_items:
+            aco_tag = "module:aco"
+            cited_items = [
+                item for item in all_items if aco_tag in item.tags
+            ][:5]  # Limit to top 5 general references
+
+        if cited_items:
+            lines.append("## Regulatory References\n")
+            for item in cited_items:
+                title = item.title
+                url = item.url
+                date = item.date_published or ""
+                institution = item.institution or ""
+
+                if url:
+                    lines.append(f"- [{title}]({url})")
+                else:
+                    lines.append(f"- {title}")
+
+                meta_parts = []
+                if institution:
+                    meta_parts.append(institution)
+                if date:
+                    meta_parts.append(date)
+                if meta_parts:
+                    lines.append(f"  *{' | '.join(meta_parts)}*")
+
+                abstract = item.abstract or ""
+                if abstract:
+                    short = abstract[:200] + "..." if len(abstract) > 200 else abstract
+                    lines.append(f"  {short}")
+                lines.append("")
+
+        out_path = OUT_DIR / f"{name}.md"
+        out_path.write_text("\n".join(lines))
+        count += 1
+        print(f"  {name} -> {out_path}")
+
+    print(f"\nDone: {count} pipeline docs written to {OUT_DIR}")
+
+
+if __name__ == "__main__":
+    main()

docs/scripts/extract_cli.py

@@ -0,0 +1,101 @@
+"""Extract CLI usage docs from typer commands.
+
+Walks the CLI app tree, invokes ``--help`` for each command, and
+writes Markdown files into ``docs/docs/cli/``.
+
+Usage::
+
+    uv run python docs/scripts/extract_cli.py
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from typer.testing import CliRunner
+
+# Ensure src/ is on the path
+sys.path.insert(0, str(Path("src").resolve()))
+
+from cli import app  # noqa: E402
+
+runner = CliRunner()
+
+OUT_DIR = Path("docs/docs/cli")
+
+
+def _get_commands(group_name: str = "") -> list[tuple[str, list[str]]]:
+    """Walk the typer app tree and return (name, cmd_path) pairs."""
+    commands: list[tuple[str, list[str]]] = []
+
+    # Top-level help
+    commands.append(("stack", []))
+
+    # Registered sub-commands / groups
+    for info in app.registered_commands:
+        name = info.name or info.callback.__name__
+        commands.append((name, [name]))
+
+    for group in app.registered_groups:
+        grp_name = group.name or ""
+        if not grp_name:
+            continue
+        commands.append((grp_name, [grp_name]))
+
+        # Walk sub-commands within the group
+        grp_app = group.typer_instance
+        if grp_app:
+            for sub_info in grp_app.registered_commands:
+                sub_name = sub_info.name or sub_info.callback.__name__
+                commands.append(
+                    (f"{grp_name}-{sub_name}", [grp_name, sub_name])
+                )
+            for sub_group in grp_app.registered_groups:
+                sg_name = sub_group.name or ""
+                if sg_name:
+                    commands.append(
+                        (f"{grp_name}-{sg_name}", [grp_name, sg_name])
+                    )
+
+    return commands
+
+
+def main() -> None:
+    if OUT_DIR.exists():
+        import shutil
+
+        shutil.rmtree(OUT_DIR)
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Category metadata for sidebar
+    cat_json = OUT_DIR / "_category_.json"
+    cat_json.write_text('{"label": "CLI Reference", "position": 10}\n')
+
+    commands = _get_commands()
+    count = 0
+
+    for slug, cmd_path in commands:
+        result = runner.invoke(app, [*cmd_path, "--help"])
+        if result.exit_code != 0:
+            continue
+
+        help_text = result.output.strip()
+        title = f"stack {' '.join(cmd_path)}" if cmd_path else "stack"
+
+        lines = [
+            f"---\ntitle: {title}\nsidebar_position: {count + 1}\n---\n",
+            f"# `{title}`\n",
+            f"```\n{help_text}\n```\n",
+        ]
+
+        out_path = OUT_DIR / f"{slug}.md"
+        out_path.write_text("\n".join(lines))
+        count += 1
+        print(f"  {title} -> {out_path}")
+
+    print(f"\nDone: {count} CLI docs written to {OUT_DIR}")
+
+
+if __name__ == "__main__":
+    main()

docs/scripts/extract_dag.py

@@ -0,0 +1,92 @@
+"""Generate pipeline DAG visualization for docs.
+
+Creates a Mermaid diagram of the full lineage graph and writes it
+as a Docusaurus MDX page.
+
+Usage::
+
+    uv run python docs/scripts/extract_dag.py
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path("src").resolve()))
+
+OUT_DIR = Path("docs/docs")
+
+
+def main() -> None:
+    from aco.lake.lineage import build_lineage
+
+    graph = build_lineage()
+    mermaid = graph.to_mermaid()
+
+    # Also build a simplified pipeline-level DAG from PIPELINES registry
+    from aco.pipe import registry
+
+    pipeline_names = sorted(registry.keys())
+
+    # Build inter-pipeline dependency edges from lineage
+    pipeline_deps: dict[str, set[str]] = {n: set() for n in pipeline_names}
+    for table, deps in graph.table_edges.items():
+        table_pipeline = table.split(".")[0] if "." in table else table
+        for dep in deps:
+            dep_pipeline = dep.split(".")[0] if "." in dep else dep
+            if (
+                table_pipeline != dep_pipeline
+                and table_pipeline in pipeline_deps
+                and dep_pipeline in pipeline_deps
+            ):
+                pipeline_deps[table_pipeline].add(dep_pipeline)
+
+    # Pipeline-level Mermaid
+    pipeline_mermaid_lines = ["graph LR"]
+    for name in pipeline_names:
+        safe = name.replace("-", "_")
+        pipeline_mermaid_lines.append(f"    {safe}[{name}]")
+    for name, deps in sorted(pipeline_deps.items()):
+        safe_name = name.replace("-", "_")
+        for dep in sorted(deps):
+            safe_dep = dep.replace("-", "_")
+            pipeline_mermaid_lines.append(f"    {safe_dep} --> {safe_name}")
+    pipeline_mermaid = "\n".join(pipeline_mermaid_lines)
+
+    content = f"""---
+title: Pipeline DAG
+sidebar_position: 13
+---
+
+# Pipeline DAG
+
+## Pipeline Dependencies
+
+High-level view of how pipelines depend on each other.
+
+```mermaid
+{pipeline_mermaid}
+```
+
+## Full Table Lineage
+
+Detailed table-level dependency graph across all pipelines.
+{len(graph.table_edges)} tables, {sum(len(v) for v in graph.table_edges.values())} edges.
+
+```mermaid
+{mermaid}
+```
+"""
+
+    out_path = OUT_DIR / "dag.md"
+    out_path.write_text(content)
+    print(f"DAG visualization -> {out_path}")
+    print(
+        f"  {len(pipeline_names)} pipelines, "
+        f"{len(graph.table_edges)} tables"
+    )
+
+
+if __name__ == "__main__":
+    main()

docs/scripts/extract_openapi.py

@@ -0,0 +1,171 @@
+"""Extract REST API docs from FastAPI OpenAPI schema.
+
+Reads the OpenAPI JSON from the FastAPI app and generates Markdown
+files for each route group into ``docs/docs/rest-api/``.
+
+Usage::
+
+    uv run python docs/scripts/extract_openapi.py
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path("src").resolve()))
+
+OUT_DIR = Path("docs/docs/rest-api")
+
+
+def _type_str(schema: dict) -> str:
+    """Convert a JSON Schema type to a readable string."""
+    if "$ref" in schema:
+        return schema["$ref"].rsplit("/", 1)[-1]
+    if "anyOf" in schema:
+        parts = [_type_str(s) for s in schema["anyOf"]]
+        return " | ".join(parts)
+    t = schema.get("type", "object")
+    if t == "array":
+        items = schema.get("items", {})
+        return f"list[{_type_str(items)}]"
+    return t
+
+
+def _render_endpoint(method: str, path: str, spec: dict) -> str:
+    """Render a single endpoint to markdown."""
+    lines = []
+    summary = spec.get("summary", "")
+    lines.append(f"### `{method.upper()} {path}`\n")
+    if summary:
+        lines.append(f"{summary}\n")
+
+    # Parameters
+    params = spec.get("parameters", [])
+    if params:
+        lines.append("**Parameters:**\n")
+        lines.append("| Name | In | Type | Required |")
+        lines.append("|------|-----|------|----------|")
+        for p in params:
+            schema = p.get("schema", {})
+            lines.append(
+                f"| `{p['name']}` | {p.get('in', '')} "
+                f"| {_type_str(schema)} | {p.get('required', False)} |"
+            )
+        lines.append("")
+
+    # Request body
+    body = spec.get("requestBody", {})
+    if body:
+        content = body.get("content", {})
+        for ct, ct_spec in content.items():
+            schema = ct_spec.get("schema", {})
+            lines.append(f"**Request body** (`{ct}`):\n")
+            lines.append(f"Type: `{_type_str(schema)}`\n")
+
+    # Responses
+    responses = spec.get("responses", {})
+    if responses:
+        lines.append("**Responses:**\n")
+        lines.append("| Status | Description |")
+        lines.append("|--------|-------------|")
+        for code, resp in sorted(responses.items()):
+            desc = resp.get("description", "")
+            lines.append(f"| {code} | {desc} |")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    from api.server import app  # noqa: E402
+
+    schema = app.openapi()
+
+    if OUT_DIR.exists():
+        import shutil
+
+        shutil.rmtree(OUT_DIR)
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Category metadata
+    cat_json = OUT_DIR / "_category_.json"
+    cat_json.write_text('{"label": "REST API", "position": 11}\n')
+
+    # Write full OpenAPI JSON
+    (OUT_DIR / "openapi.json").write_text(json.dumps(schema, indent=2))
+
+    # Group endpoints by first tag
+    groups: dict[str, list[tuple[str, str, dict]]] = {}
+    for path, methods in schema.get("paths", {}).items():
+        for method, spec in methods.items():
+            if method in ("parameters", "servers", "summary"):
+                continue
+            tags = spec.get("tags", ["default"])
+            tag = tags[0] if tags else "default"
+            groups.setdefault(tag, []).append((method, path, spec))
+
+    count = 0
+    # Overview page
+    overview_lines = [
+        "---\ntitle: REST API Overview\nsidebar_position: 1\n---\n",
+        "# REST API\n",
+        f"**Title:** {schema.get('info', {}).get('title', 'Stack API')}\n",
+        f"**Version:** {schema.get('info', {}).get('version', '0.1.0')}\n",
+        "## Endpoints\n",
+    ]
+    for tag, endpoints in sorted(groups.items()):
+        overview_lines.append(f"- [{tag}](./{tag}) ({len(endpoints)} endpoints)")
+    overview_lines.append(
+        "\n\nFull OpenAPI spec: [openapi.json](./openapi.json)\n"
+    )
+    (OUT_DIR / "index.md").write_text("\n".join(overview_lines))
+    count += 1
+
+    # Per-group pages
+    for pos, (tag, endpoints) in enumerate(sorted(groups.items()), start=2):
+        lines = [
+            f"---\ntitle: {tag}\nsidebar_position: {pos}\n---\n",
+            f"# {tag}\n",
+        ]
+        for method, path, spec in endpoints:
+            lines.append(_render_endpoint(method, path, spec))
+
+        out_path = OUT_DIR / f"{tag}.md"
+        out_path.write_text("\n".join(lines))
+        count += 1
+        print(f"  {tag} -> {out_path}")
+
+    # Schemas section
+    schemas = schema.get("components", {}).get("schemas", {})
+    if schemas:
+        schema_lines = [
+            "---\ntitle: Schemas\nsidebar_position: 99\n---\n",
+            "# Schemas\n",
+        ]
+        for name, s in sorted(schemas.items()):
+            schema_lines.append(f"## `{name}`\n")
+            desc = s.get("description", "")
+            if desc:
+                schema_lines.append(f"{desc}\n")
+            props = s.get("properties", {})
+            if props:
+                schema_lines.append("| Field | Type | Required |")
+                schema_lines.append("|-------|------|----------|")
+                required = set(s.get("required", []))
+                for field, fspec in props.items():
+                    schema_lines.append(
+                        f"| `{field}` | {_type_str(fspec)} "
+                        f"| {'yes' if field in required else ''} |"
+                    )
+                schema_lines.append("")
+
+        (OUT_DIR / "schemas.md").write_text("\n".join(schema_lines))
+        count += 1
+
+    print(f"\nDone: {count} REST API docs written to {OUT_DIR}")
+
+
+if __name__ == "__main__":
+    main()

docs/sidebars.js

@@ -6,6 +6,19 @@ const sidebars = {
       type: "autogenerated",
       dirName: "api",
     },
+    {
+      type: "autogenerated",
+      dirName: "cli",
+    },
+    {
+      type: "autogenerated",
+      dirName: "rest-api",
+    },
+    {
+      type: "autogenerated",
+      dirName: "pipelines",
+    },
+    "dag",
   ],
 };
 

src/cli/docs.py

@@ -2,20 +2,100 @@
 
 from __future__ import annotations
 
+import subprocess
+import sys
+from pathlib import Path
+
 import typer
 
 app = typer.Typer(no_args_is_help=True)
 
+DOCS_DIR = Path(__file__).resolve().parent.parent.parent / "docs"
+SCRIPTS_DIR = DOCS_DIR / "scripts"
+
+
+def _run_script(name: str) -> bool:
+    """Run a docs generation script via uv. Returns True on success."""
+    script = SCRIPTS_DIR / name
+    if not script.exists():
+        typer.echo(f"  skip: {name} not found")
+        return False
+
+    typer.echo(f"  running {name}...")
+
+    # griffe is needed for extract_docs
+    extras = ["--with", "griffe"] if name == "extract_docs.py" else []
+
+    result = subprocess.run(
+        ["uv", "run", *extras, "python", str(script)],
+        cwd=str(DOCS_DIR.parent),
+    )
+    return result.returncode == 0
+
+
+def _generate() -> None:
+    """Run all documentation generation scripts."""
+    typer.echo("Generating documentation content...")
+    scripts = [
+        "extract_docs.py",
+        "extract_cli.py",
+        "extract_openapi.py",
+        "extract_citations.py",
+        "extract_dag.py",
+        "export_library.py",
+    ]
+    ok = 0
+    for s in scripts:
+        if _run_script(s):
+            ok += 1
+    typer.echo(f"Generated: {ok}/{len(scripts)} scripts succeeded")
+
 
 @app.command()
-def build() -> None:
+def build(
+    skip_generate: bool = typer.Option(
+        False,
+        "--skip-generate",
+        help="Skip content generation, only run Docusaurus build.",
+    ),
+) -> None:
     """Generate documentation site."""
-    typer.echo("docs build [stub]")
+    if not skip_generate:
+        _generate()
+
+    typer.echo("Building Docusaurus site...")
+    npm = "npm.cmd" if sys.platform == "win32" else "npm"
+    result = subprocess.run(
+        [npm, "run", "build"],
+        cwd=str(DOCS_DIR),
+    )
+    if result.returncode != 0:
+        typer.echo("Docusaurus build failed. Is node/npm installed?")
+        typer.echo(f"  docs dir: {DOCS_DIR}")
+        raise typer.Exit(1)
+    typer.echo("docs build complete")
 
 
 @app.command()
 def serve(
-    port: int = typer.Option(8000, help="Port to serve on."),
+    port: int = typer.Option(3000, help="Port to serve on."),
+    skip_generate: bool = typer.Option(
+        False, "--skip-generate", help="Skip content generation."
+    ),
 ) -> None:
-    """Serve documentation locally."""
-    typer.echo(f"docs serve: port={port} [stub]")
+    """Serve documentation locally with hot reload."""
+    if not skip_generate:
+        _generate()
+
+    typer.echo(f"Starting Docusaurus dev server on port={port}...")
+    npm = "npm.cmd" if sys.platform == "win32" else "npm"
+    subprocess.run(
+        [npm, "start", "--", "--port", str(port)],
+        cwd=str(DOCS_DIR),
+    )
+
+
+@app.command()
+def generate() -> None:
+    """Generate documentation content without building the site."""
+    _generate()

tests/cli/test_cli.py

@@ -129,17 +129,17 @@ class TestValidate:
 
 
 class TestDocs:
-    def test_docs_build(self) -> None:
-        result = runner.invoke(app, ["docs", "build"])
+    def test_docs_build_help(self) -> None:
+        result = runner.invoke(app, ["docs", "build", "--help"])
         assert result.exit_code == 0
-        assert "docs build" in result.output
+        assert "skip-generate" in result.output
 
-    def test_docs_serve(self) -> None:
-        result = runner.invoke(app, ["docs", "serve"])
+    def test_docs_serve_help(self) -> None:
+        result = runner.invoke(app, ["docs", "serve", "--help"])
         assert result.exit_code == 0
-        assert "docs serve" in result.output
+        assert "port" in result.output
 
-    def test_docs_serve_custom_port(self) -> None:
-        result = runner.invoke(app, ["docs", "serve", "--port", "9000"])
+    def test_docs_generate_help(self) -> None:
+        result = runner.invoke(app, ["docs", "generate", "--help"])
         assert result.exit_code == 0
-        assert "port=9000" in result.output
+        assert "Generate" in result.output