stack/dev/scripts/nb_fe_smoke.py

"""Frontend smoke test for the notebooks (marimo) image/service.

Loads the editor in a headless browser and fails on any console error or
page error. This is the gate that would have caught the 2026-07-09 incident:
a frontend bundle that compiled cleanly but threw "d is not a constructor"
on every notebook open, while the container healthcheck and home page stayed
green — no Python-level test can see that class of breakage.

Two modes:

  # Gate a freshly built image (CI, after `docker buildx --load`):
  python dev/scripts/nb_fe_smoke.py --image ci-test-notebooks --network ci

  # Probe a running service (post-deploy, nightly):
  python dev/scripts/nb_fe_smoke.py --url http://notebooks:2718 --network gateway

Everything crosses the docker socket (docker cp / stdin), never bind mounts:
in CI the job's filesystem lives in a volume the host daemon can't -v mount,
so mounted paths silently arrive empty (the trap that broke the apt mirror).

Stdlib only.
"""

from __future__ import annotations

import argparse
import json
import subprocess
import sys
import tempfile
import time
import uuid
from pathlib import Path

PLAYWRIGHT_IMAGE = "mcr.microsoft.com/playwright/python:v1.61.0-noble"
PLAYWRIGHT_PKG = "playwright==1.61.0"
READY_TIMEOUT_S = 120

# Console errors that are expected marimo behavior, not bundle breakage.
# CellNotInitializedError: with auto_instantiate=false the editor renders
# cached UI elements from a session snapshot whose cells aren't running in
# the kernel — every interaction with them logs this error. Appears on any
# notebook that has a __marimo__/session snapshot (e.g. right after the
# nightly integration run regenerates them).
BENIGN_ERRORS = ("CellNotInitializedError",)

# Dependency-free notebook shipped into the throwaway container. Markdown
# only, so it renders identically under any runtime config (the bare image
# has no marimo.toml, and marimo's default auto_instantiate=true would run
# code cells).
PROBE_NB = """\
import marimo

app = marimo.App()

@app.cell
def _():
    import marimo as mo
    mo.md("fe-smoke-canary-cell")
    return

if __name__ == "__main__":
    app.run()
"""

# Runs inside the playwright container; URL substituted via argv.
PROBE_SCRIPT = """
import json, sys
from playwright.sync_api import sync_playwright

url = sys.argv[1]
events = []
with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.on("console", lambda m: events.append({"type": m.type, "text": m.text}))
    page.on("pageerror", lambda e: events.append({"type": "pageerror", "text": str(e)}))
    page.goto(url, wait_until="networkidle", timeout=60000)
    page.wait_for_timeout(6000)
    body = page.inner_text("body")
    browser.close()
print("FE_SMOKE_RESULT " + json.dumps({"events": events, "body": body[:3000]}))
"""


def _run(cmd: list[str], **kw) -> subprocess.CompletedProcess:
    return subprocess.run(cmd, capture_output=True, text=True, timeout=300, **kw)


def wait_ready(url: str, network: str) -> bool:
    """Poll the target URL from a curl container on the same network."""
    deadline = time.time() + READY_TIMEOUT_S
    while time.time() < deadline:
        r = _run(
            [
                "docker",
                "run",
                "--rm",
                "--network",
                network,
                "curlimages/curl:8.11.1",
                "-s",
                "-o",
                "/dev/null",
                "-w",
                "%{http_code}",
                "--max-time",
                "5",
                url,
            ]
        )
        if r.stdout.strip() == "200":
            return True
        time.sleep(3)
    return False


def probe(url: str, network: str) -> dict:
    """Load `url` headlessly; return {events, body}."""
    shell = (
        f"pip install -q {PLAYWRIGHT_PKG} 2>/dev/null; "
        f"python3 - \"$0\" <<'PYEOF'\n{PROBE_SCRIPT}\nPYEOF"
    )
    r = subprocess.run(
        [
            "docker",
            "run",
            "--rm",
            "-i",
            "--network",
            network,
            "--entrypoint",
            "bash",
            PLAYWRIGHT_IMAGE,
            "-c",
            shell,
            url,
        ],
        capture_output=True,
        text=True,
        timeout=600,
    )
    for line in r.stdout.splitlines():
        if line.startswith("FE_SMOKE_RESULT "):
            return json.loads(line[len("FE_SMOKE_RESULT ") :])
    raise RuntimeError(
        f"probe produced no result (rc={r.returncode}):\n"
        f"{r.stdout[-1000:]}\n{r.stderr[-1000:]}"
    )


def check(result: dict, *, expect_text: str | None) -> list[str]:
    problems = [
        f"[{e['type']}] {e['text'][:300]}"
        for e in result["events"]
        if e["type"] in ("error", "pageerror")
        and not any(b in e["text"] for b in BENIGN_ERRORS)
    ]
    if expect_text and expect_text not in result["body"]:
        problems.append(
            f"expected editor content {expect_text!r} not found in page body "
            f"(got: {result['body'][:200]!r})"
        )
    return problems


def smoke_image(image: str, network: str) -> int:
    """Boot a throwaway container from `image`, probe '/', probe the editor."""
    name = f"nb-fe-smoke-{uuid.uuid4().hex[:8]}"
    with tempfile.TemporaryDirectory() as td:
        nb_dir = Path(td) / "notebooks"
        nb_dir.mkdir()
        (nb_dir / "fe_smoke_probe.py").write_text(PROBE_NB)
        try:
            r = _run(["docker", "create", "--name", name, "--network", network, image])
            if r.returncode != 0:
                print(f"docker create failed: {r.stderr[:500]}", file=sys.stderr)
                return 2
            # docker cp streams the client's file over the socket — works
            # from inside CI where -v bind mounts cannot.
            _run(["docker", "cp", str(nb_dir), f"{name}:/home/kert/"])
            _run(["docker", "start", name])
            _run(
                [
                    "docker",
                    "exec",
                    "-u",
                    "0",
                    name,
                    "chmod",
                    "-R",
                    "a+rwX",
                    "/home/kert/notebooks",
                ]
            )

            base = f"http://{name}:2718"
            if not wait_ready(f"{base}/", network):
                logs = _run(["docker", "logs", "--tail", "30", name])
                print(
                    f"service never became ready at {base}\n{logs.stdout}{logs.stderr}",
                    file=sys.stderr,
                )
                return 2
            return _verdict(
                base,
                network,
                "?file=fe_smoke_probe.py",
                expect_text="fe-smoke-canary-cell",
            )
        finally:
            _run(["docker", "rm", "-f", name])


def smoke_url(url: str, network: str, notebook: str | None) -> int:
    base = url.rstrip("/")
    if not wait_ready(f"{base}/", network):
        print(f"service not responding at {base}", file=sys.stderr)
        return 2
    query = f"?file={notebook}" if notebook else ""
    # Live service: don't assert specific content (notebook set varies),
    # only require an error-free load.
    return _verdict(base, network, query, expect_text=None)


def _verdict(
    base: str, network: str, editor_query: str, expect_text: str | None
) -> int:
    failed = False
    for label, target, expect in (
        ("home", f"{base}/", None),
        ("editor", f"{base}/{editor_query}", expect_text),
    ):
        result = probe(target, network)
        problems = check(result, expect_text=expect)
        if problems:
            failed = True
            print(f"✗ {label} {target}")
            for p in problems:
                print(f"    {p}")
        else:
            print(f"✓ {label} {target} — no console/page errors")
    return 1 if failed else 0


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    target = parser.add_mutually_exclusive_group(required=True)
    target.add_argument("--image", help="image tag to boot and gate")
    target.add_argument("--url", help="running service base URL to probe")
    parser.add_argument(
        "--network", required=True, help="docker network shared with the target"
    )
    parser.add_argument(
        "--notebook", default="sample.py", help="notebook to open in --url mode"
    )
    args = parser.parse_args()

    if args.image:
        return smoke_image(args.image, args.network)
    return smoke_url(args.url, args.network, args.notebook)


if __name__ == "__main__":
    raise SystemExit(main())