stack/dev/scripts/gen_config.py

"""Generate all derived config files from stack.toml.

Reads [platform], [services], [images], and [ci] from stack.toml.
Dispatches to the active CI backend emitter (gitea or github)
for pipeline YAML, plus generates coredns config.

Usage::

    uv run python dev/scripts/gen_config.py            # generate all
    uv run python dev/scripts/gen_config.py --check     # verify up-to-date
    uv run python dev/scripts/gen_config.py --backend X  # override backend
"""

from __future__ import annotations

import argparse
import sys

from conf import ROOT, cfg

# ── Platform values ──────────────────────────────────────────────

DOMAIN = cfg.platform.domain
HOST_IP = cfg.platform.host_ip


def _platform_dict() -> dict:
    return {
        "registry": cfg.platform.registry,
        "org": cfg.platform.repo.split("/")[0],
        "image_prefix": cfg.platform.image_prefix,
        "domain": DOMAIN,
        "host_ip": HOST_IP,
        "repo": cfg.platform.repo,
    }


# ── Image definitions ────────────────────────────────────────────


def load_images() -> list[dict]:
    """Read [images] from stack.toml, merge each with defaults."""
    raw = dict(cfg._data.get("images", {}))
    defaults = dict(raw.pop("defaults", {}))
    images = []
    for name, overrides in raw.items():
        merged = {**defaults, **overrides, "name": name}
        images.append(merged)
    return images


def scannable(images: list[dict]) -> list[dict]:
    return [i for i in images if i.get("scan", True)]


# ── CoreDNS ──────────────────────────────────────────────────────


def gen_coredns() -> dict[str, str]:
    subdomains = cfg._data.get("platform", {}).get("subdomains", [])
    lines = [f"{HOST_IP} {DOMAIN}"]  # bare domain always first
    for sub in subdomains:
        lines.append(f"{HOST_IP} {sub}.{DOMAIN}")
    hosts = "\n".join(lines) + "\n"

    corefile = f"""{DOMAIN} {{
    hosts /etc/coredns/hosts {{
        fallthrough
    }}
    log
}}

. {{
    forward . 1.1.1.1 8.8.8.8
    cache 300
}}
"""
    return {"infra/coredns/hosts": hosts, "infra/coredns/Corefile": corefile}


# ── Backend dispatch ─────────────────────────────────────────────

STALE_DIRS: dict[str, list[str]] = {
    "github": [".gitea/workflows"],
    "gitea": [".github/workflows"],
}


def _emit_backend(backend: str) -> dict[str, str]:
    """Dispatch to the correct CI backend emitter."""
    images = load_images()
    scans = scannable(images)
    platform = _platform_dict()

    ci_data = cfg._data.get("ci", {})
    ci_cfg = ci_data.get(backend, {})
    # Propagate top-level coverage_threshold so each backend can read it.
    ci_cfg.setdefault("coverage_threshold", ci_data.get("coverage_threshold", 99))

    if backend == "github":
        from backends.github import emit
    elif backend == "gitea":
        from backends.gitea import emit
    else:
        print(f"Unknown CI backend: {backend}", file=sys.stderr)
        sys.exit(1)

    return emit(images, scans, platform, ci_cfg)


def _clean_stale(backend: str, generated_files: dict[str, str]) -> list[str]:
    """Return stale files from inactive backend directories."""
    stale_dirs = STALE_DIRS.get(backend, [])
    removed = []
    for stale_dir in stale_dirs:
        stale_path = ROOT / stale_dir
        if not stale_path.exists():
            continue
        for f in sorted(stale_path.glob("*.yml")):
            rel = str(f.relative_to(ROOT))
            if rel not in generated_files:
                content = f.read_text()
                if "generated by gen_config.py" in content:
                    removed.append(rel)
    return removed


# ── Main ─────────────────────────────────────────────────────────


def _emit_dab(*, sql: bool = False) -> dict[str, str]:
    """Generate databricks.yml (and optionally SQL/DDL) from pipeline registry.

    When *sql* is False (default), only databricks.yml is emitted — fast
    enough for --check in CI.  When True, also transpiles all pipeline
    expressions to SQL and generates DDL (1500+ files, ~4 minutes).
    """
    if "databricks" not in cfg._data:
        return {}
    from backends.databricks import emit

    return emit(cfg._data, sql=sql)


def generate(backend: str | None = None, *, dab_sql: bool = False) -> dict[str, str]:
    if backend is None:
        backend = cfg._data.get("ci", {}).get("backend", "gitea")

    files: dict[str, str] = {}
    files.update(gen_coredns())
    files.update(_emit_backend(backend))
    files.update(_emit_dab(sql=dab_sql))
    return files


def write_all(files: dict[str, str]) -> None:
    for relpath, content in sorted(files.items()):
        out = ROOT / relpath
        out.parent.mkdir(parents=True, exist_ok=True)
        out.write_text(content)
        print(f"  wrote {relpath}")


def check_all(files: dict[str, str]) -> list[str]:
    diffs = []
    for relpath, expected in sorted(files.items()):
        out = ROOT / relpath
        if not out.exists():
            diffs.append(f"{relpath}: missing")
        elif out.read_text() != expected:
            diffs.append(f"{relpath}: out of date")
    return diffs


def check_modules() -> list[str]:
    """Verify module-name and optional-dependencies stay in sync."""
    import tomllib

    pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text())
    module_names = set(
        pyproject.get("tool", {})
        .get("uv", {})
        .get("build-backend", {})
        .get("module-name", [])
    )
    opt_deps = set(pyproject.get("project", {}).get("optional-dependencies", {}).keys())
    # Groups that are aggregates or cloud providers, not Python modules
    meta_groups = {"all", "lake", "dev", "aws", "gcp", "azure", "notebooks"}

    errors = []
    for mod in sorted(module_names):
        if mod not in opt_deps and mod not in meta_groups:
            errors.append(
                f"module '{mod}' in [tool.uv.build-backend].module-name "
                f"has no matching [project.optional-dependencies] group"
            )
    for group in sorted(opt_deps - meta_groups):
        if group not in module_names:
            errors.append(
                f"optional-dep group '{group}' has no matching "
                f"module in [tool.uv.build-backend].module-name"
            )
    return errors


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Generate derived config from stack.toml [ci] backend"
    )
    parser.add_argument(
        "--check",
        action="store_true",
        help="Verify files are up-to-date (exit 1 if not)",
    )
    parser.add_argument(
        "--backend",
        default=None,
        help="Override CI backend (gitea|github)",
    )
    parser.add_argument(
        "--dab-sql",
        action="store_true",
        help="Also generate bundle/sql/ and bundle/ddl/ (slow, ~4 min)",
    )
    args = parser.parse_args()

    files = generate(args.backend, dab_sql=args.dab_sql)

    if args.check:
        diffs = check_all(files)
        mod_errors = check_modules()
        all_errors = diffs + mod_errors
        if all_errors:
            if diffs:
                print("Files out of date:")
                for d in diffs:
                    print(f"  {d}")
            if mod_errors:
                print("Module/dependency sync errors:")
                for e in mod_errors:
                    print(f"  {e}")
            print("\nRun: uv run python dev/scripts/gen_config.py")
            return 1
        print("All generated files are up-to-date.")
        print("Module names and optional-dependencies are in sync.")
        return 0

    write_all(files)

    # Clean stale files from inactive backend
    backend = args.backend or cfg._data.get("ci", {}).get("backend", "gitea")
    stale = _clean_stale(backend, files)
    for s in stale:
        (ROOT / s).unlink()
        print(f"  removed stale {s}")

    print(f"\nGenerated {len(files)} files from stack.toml (backend: {backend})")
    return 0


if __name__ == "__main__":
    sys.exit(main())