stack/dev/scripts/gen_readme.py

"""Generate README.md from README.md.j2 using live codebase data.

Collects counts, tables, and metadata from the actual codebase:
    - Service count and names from compose.yml
    - Module list from pyproject.toml optional-dependencies
    - Pipeline registry from src/aco/pipe/
    - Test count via pytest --collect-only
    - Credential count from api.auth.manifest
    - Network list from compose.yml
    - Workflow list from .gitea/workflows/

Usage::

    uv run python dev/scripts/gen_readme.py            # generate README.md
    uv run python dev/scripts/gen_readme.py --check     # verify up-to-date
    uv run python dev/scripts/gen_readme.py --dry-run   # print to stdout
"""

from __future__ import annotations

import argparse
import ast
import subprocess
import sys
import tomllib
from pathlib import Path

import yaml
from jinja2 import Environment, FileSystemLoader

ROOT = Path(__file__).resolve().parent.parent.parent

# ── Data collectors ──────────────────────────────────────────────────


def _collect_services() -> dict:
    """Parse compose.yml for service and network info."""
    compose = yaml.safe_load((ROOT / "compose.yml").read_text())
    services = sorted(compose.get("services", {}).keys())
    networks = sorted(compose.get("networks", {}).keys())
    return {"names": services, "count": len(services), "networks": networks}


def _collect_modules() -> dict:
    """Parse pyproject.toml for module list."""
    with open(ROOT / "pyproject.toml", "rb") as f:
        pyp = tomllib.load(f)

    opt_deps = pyp["project"]["optional-dependencies"]
    cloud = {"aws", "gcp", "azure"}
    aggregates = {"all", "lake"}
    modules = sorted(k for k in opt_deps if k not in cloud and k not in aggregates)

    build_modules = (
        pyp.get("tool", {})
        .get("uv", {})
        .get("build-backend", {})
        .get("module-name", [])
    )

    return {
        "names": modules,
        "count": len(modules),
        "cloud": sorted(cloud & set(opt_deps)),
        "aggregates": sorted(aggregates & set(opt_deps)),
        "build_modules": build_modules,
    }


def _collect_pipelines() -> dict:
    """Import pipeline registry and collect metadata."""
    sys.path.insert(0, str(ROOT / "src"))
    from aco.pipe import registry

    # Fallback descriptions for pipelines without module docstrings
    fallback_docs = {
        "ahrq_measures": "AHRQ quality indicators",
        "cclf": "CCLF file parsing",
        "claims_preprocessing": "Claim matching, encounter IDs",
        "cms_quality_measures": "CMS ACO quality metrics",
        "core": "Encounters, patients, providers",
        "data_quality": "Validation and completeness",
        "hcc_suspecting": "HCC risk adjustment",
        "input_layer": "Raw CCLF/BCDA ingestion",
        "main": "Full pipeline orchestration",
        "opps": "Outpatient payment system",
        "pharmacy": "Part D drug utilization",
        "provider_attribution": "Provider-patient assignment",
        "quality_measures": "Composite quality scores",
        "readmissions": "30-day readmission rates",
    }

    pipelines = []
    for name in sorted(registry.keys()):
        p = registry[name]
        doc = ""
        # Try to get the pipeline module's docstring
        mod_path = ROOT / "src" / "aco" / "pipe" / f"{name}.py"
        if mod_path.exists():
            try:
                tree = ast.parse(mod_path.read_text())
                raw = ast.get_docstring(tree) or ""
                doc = raw.split("\n")[0].rstrip(".")
            except SyntaxError:
                pass
        # Use fallback if docstring is empty
        if not doc:
            doc = fallback_docs.get(name, "")
        pipelines.append(
            {
                "name": name,
                "steps": len(p.exprs),
                "doc": doc,
            }
        )

    total_steps = sum(p["steps"] for p in pipelines)
    return {
        "list": pipelines,
        "count": len(pipelines),
        "total_steps": total_steps,
    }


def _collect_tests() -> dict:
    """Run pytest --collect-only to get test count."""
    result = subprocess.run(
        ["uv", "run", "python", "-m", "pytest", "tests/", "--collect-only", "-q"],
        capture_output=True,
        text=True,
        cwd=str(ROOT),
        timeout=60,
    )
    # Last line: "12291 tests collected in 3.03s"
    for line in result.stdout.strip().splitlines():
        if "test" in line and "collected" in line:
            count = int(line.split()[0])
            return {"count": count, "formatted": f"{count:,}"}

    return {"count": 0, "formatted": "?"}


def _collect_credentials() -> dict:
    """Count credentials from auth manifest."""
    sys.path.insert(0, str(ROOT / "src"))
    try:
        from api.auth.manifest import CREDENTIALS

        return {"count": len(CREDENTIALS)}
    except ImportError:
        return {"count": 0}


def _collect_workflows() -> dict:
    """List CI workflow files."""
    workflow_dirs = [
        ROOT / ".gitea" / "workflows",
        ROOT / ".github" / "workflows",
    ]
    workflows = []
    for d in workflow_dirs:
        if d.is_dir():
            for f in sorted(d.glob("*.yml")):
                workflows.append({"name": f.name, "path": str(f.relative_to(ROOT))})
    return {"list": workflows, "count": len(workflows)}


def _collect_src_tree() -> list[dict]:
    """Build the src/ module listing with descriptions."""
    descriptions = {
        "aco": "ACO analytics (express, pipe, table, lake, load)",
        "api": "FastAPI server + auth + diag",
        "bcda": "BCDA FHIR R4 client",
        "bib": "Zotero bibliography store, tags, metadata",
        "bls": "BLS data",
        "ccw": "CCW data dictionary",
        "cli": "CLI entry point (typer)",
        "cms": "CMS public data tables",
        "conf": "Config loader, storage abstraction, table base",
        "opps": "Outpatient Prospective Payment System",
        "perf": "Pipeline telemetry (OpenTelemetry)",
        "pfs": "Physician Fee Schedule",
        "rex": "REX fixed-width file processing",
        "sem": "Semantic coverage orchestration",
    }

    src = ROOT / "src"
    modules = []
    for d in sorted(src.iterdir()):
        if (
            d.is_dir()
            and not d.name.startswith(("_", "."))
            and not d.name.endswith(".egg-info")
        ):
            modules.append(
                {
                    "name": d.name,
                    "desc": descriptions.get(d.name, ""),
                }
            )
    return modules


def collect_all() -> dict:
    """Collect all dynamic data for template rendering."""
    services = _collect_services()
    modules = _collect_modules()
    pipelines = _collect_pipelines()
    tests = _collect_tests()
    credentials = _collect_credentials()
    workflows = _collect_workflows()
    src_modules = _collect_src_tree()

    return {
        "services": services,
        "modules": modules,
        "pipelines": pipelines,
        "tests": tests,
        "credentials": credentials,
        "workflows": workflows,
        "src_modules": src_modules,
    }


# ── Rendering ────────────────────────────────────────────────────────


def render(data: dict) -> str:
    """Render README.md.j2 with collected data."""
    env = Environment(
        loader=FileSystemLoader(str(ROOT)),
        keep_trailing_newline=True,
        trim_blocks=True,
        lstrip_blocks=True,
    )
    env.filters["backtick"] = lambda s: f"`{s}`"
    template = env.get_template("README.md.j2")
    return template.render(**data)


# ── CLI ──────────────────────────────────────────────────────────────


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--check",
        action="store_true",
        help="verify README.md matches template output (exit 1 if stale)",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="print rendered output to stdout instead of writing",
    )
    args = parser.parse_args()

    data = collect_all()
    rendered = render(data)

    if args.dry_run:
        print(rendered, end="")
        return 0

    readme = ROOT / "README.md"

    if args.check:
        current = readme.read_text(encoding="utf-8")
        if current == rendered:
            print("README.md is up-to-date.")
            return 0
        else:
            # Show first differing line
            cur_lines = current.splitlines()
            ren_lines = rendered.splitlines()
            for i, (a, b) in enumerate(zip(cur_lines, ren_lines), 1):
                if a != b:
                    print(f"README.md is stale (first diff at line {i}):")
                    print(f"  current:  {a[:100]}")
                    print(f"  expected: {b[:100]}")
                    break
            else:
                len_diff = len(cur_lines) - len(ren_lines)
                print(f"README.md is stale (length differs by {len_diff} lines)")
            print("\nRun: uv run python dev/scripts/gen_readme.py")
            return 1

    readme.write_text(rendered, encoding="utf-8")
    print(
        f"README.md generated ({data['services']['count']} services, "
        f"{data['pipelines']['count']} pipelines, "
        f"{data['tests']['formatted']} tests, "
        f"{data['modules']['count']} modules)"
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())