fix(notebooks): install what src/ needs — a notebooks extra drives the image

Notebooks import src/ off the bind mount, but the image installed a
hand-written package list that never had httpx, so event_study,
code_families, llm_search, bcda_v3_sandbox and api_explorer all died on
import (bls, pfs.utilization, llm.client, bcda.client); pfs.pipe's lazy
openpyxl and the base deps (pymupdf, python-docx, xlrd) were missing too.

pyproject gains a `notebooks` extra composed of the extras notebook-
reachable code uses (conf, aco, bib, cms, nlm, pfs) plus the duckdb <2
cap. A small Dockerfile stage expands it with the base dependencies into
a requirements file, folding duplicate names into one spec because `uv
add` keeps only the last spec per package (the cap would otherwise be
dropped). The image's own list keeps only image-specific packages.

Closes #1389

infra/images/notebooks.Dockerfile

@@ -66,6 +66,47 @@ WORKDIR /src
 COPY --from=fe /src/ /src/
 RUN uv build --wheel
 
+# ---- stage 2b: notebook requirements ----------------------------------------
+#
+# Notebooks import src/ (bind-mounted, on PYTHONPATH), so the image needs the
+# stack's base dependencies plus pyproject's `notebooks` extra. Expand its
+# stack[...] self-references into plain requirements here; the heavy install
+# layer below then rebuilds only when that list changes, not on every
+# pyproject edit. `uv add` keeps just the last spec it sees per package, so
+# repeated names (duckdb comes from four extras) fold into one requirement.
+
+FROM ghcr.io/astral-sh/uv:0.13.2-python3.13-trixie-slim AS nbreqs
+COPY pyproject.toml /pyproject.toml
+RUN python3 - > /notebooks-requirements.txt <<'PY'
+import re, tomllib
+
+project = tomllib.load(open("/pyproject.toml", "rb"))["project"]
+extras = project["optional-dependencies"]
+reqs, seen = list(project["dependencies"]), set()
+
+def walk(extra):
+    if extra in seen:
+        return
+    seen.add(extra)
+    for req in extras[extra]:
+        if m := re.fullmatch(r"stack\[([\w,-]+)\]", req):
+            for sub in m.group(1).split(","):
+                walk(sub)
+        else:
+            reqs.append(req)
+
+walk("notebooks")
+merged = {}
+for req in reqs:
+    name, ext, spec = re.fullmatch(r"([\w.-]+)(?:\[([^\]]*)\])?\s*([^;]*)", req).groups()
+    _, e, s = merged.setdefault(re.sub(r"[-_.]+", "-", name).lower(), [name, set(), []])
+    e.update(x.strip() for x in (ext or "").split(",") if x.strip())
+    s.extend(c.strip() for c in spec.split(",") if c.strip() and c.strip() not in s)
+for name, e, s in merged.values():
+    print(name + (f"[{','.join(sorted(e))}]" if e else "") + ",".join(s))
+PY
+RUN cat /notebooks-requirements.txt
+
 # ---- stage 3: runtime -------------------------------------------------------
 
 FROM nvidia/cuda:12.6.0-runtime-ubuntu24.04
@@ -99,28 +140,27 @@ WORKDIR /home/${USERNAME}
 
 # Bring in the fhirworx marimo wheel from the wheel stage.
 COPY --from=wheel /src/dist/ /tmp/marimo-wheel/
+COPY --from=nbreqs /notebooks-requirements.txt /tmp/notebooks-requirements.txt
 
 # No release cooldown here (unlike pyproject.toml's exclude-newer): this
 # workspace pulls cudf-polars-cu12 from pypi.nvidia.com, which publishes no
 # upload dates, so any exclude-newer filters out every release and the build
 # fails. uv has no per-package opt-out for date-less indexes.
 
-# duckdb stays on the 1.x line the rest of the stack locks (1.5.x): this
-# workspace has no lock, and every service shares the same .duckdb files,
-# so a new major here must not be the first engine to touch them.
-
 # Install python + init workspace venv. Use the local wheel for marimo so
 # the fhirworx frontend ships in the image; no upstream PyPI fetch for
-# marimo itself.
+# marimo itself. Packages the stack's code needs (duckdb, narwhals, sqlglot,
+# pyarrow, httpx, ...) come from the requirements file, not this list — a
+# name listed in both would have its spec replaced by whichever comes last.
 RUN uv python install ${PYTHON_VERSION} \
     && uv init workspace --python ${PYTHON_VERSION} \
     && cd workspace \
     && MARIMO_WHL=$(ls /tmp/marimo-wheel/marimo-*.whl | head -1) \
-    && uv add "${MARIMO_WHL}[recommended]" polars cudf-polars-cu12 pandas numpy pyarrow \
-        "pyiceberg[s3,pyarrow]>=0.7.0" "duckdb>=1.5.4,<2" "narwhals>=1.0.0" "trino>=0.328.0" \
-        "sqlglot>=26.0.0" \
+    && uv add "${MARIMO_WHL}[recommended]" polars cudf-polars-cu12 pandas numpy \
+        "pyiceberg[s3,pyarrow]>=0.7.0" "trino>=0.328.0" \
         vega_datasets pyzotero obstore s3fs \
         "sqlalchemy>=2.0.0" "psycopg[binary]>=3.2.0" \
+        -r /tmp/notebooks-requirements.txt \
         --extra-index-url https://pypi.nvidia.com
 
 # Create marimo config directory with minimal defaults.

pyproject.toml

@@ -168,6 +168,24 @@ azure = [
     "azure-storage-blob>=12.23.0",
     "adlfs>=2024.1.0",
 ]
+# What notebooks/ reaches through src/. The notebooks container bind-mounts
+# src/ and installs this extra (plus the base dependencies) — see
+# infra/images/notebooks.Dockerfile — so a dependency a notebook-reachable
+# module gains belongs in its own extra, and that extra belongs here.
+notebooks = [
+    "stack[conf]",
+    "stack[aco]",
+    "stack[bib]",
+    "stack[cms]",
+    "stack[nlm]",
+    "stack[pfs]",
+    # The image resolves without a lock, and every service shares the same
+    # .duckdb files, so a new duckdb major must not land here first.
+    "duckdb>=1.5.4,<2",
+    "fsspec>=2026.6.0",  # bcda.store; the bcda extra itself drags in streamlit
+    "openpyxl>=3.1.5",  # pfs.pipe reads CMS workbooks
+    "fastexcel>=0.19.0",  # polars.read_excel's calamine engine
+]
 all = [
     "stack[aco]",
     "stack[api]",

uv.lock

@@ -14,7 +14,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-10-20T19:35:11.505292094Z"
+exclude-newer = "2026-10-22T18:36:04.896204313Z"
 exclude-newer-span = "P7D"
 
 [options.exclude-newer-package]
@@ -4698,6 +4698,19 @@ nlm = [
     { name = "httpx" },
     { name = "pydantic" },
 ]
+notebooks = [
+    { name = "duckdb" },
+    { name = "fastexcel" },
+    { name = "fsspec" },
+    { name = "httpx" },
+    { name = "markdown-it-py" },
+    { name = "mdit-py-plugins" },
+    { name = "narwhals" },
+    { name = "openpyxl" },
+    { name = "pyarrow" },
+    { name = "pydantic" },
+    { name = "sqlglot" },
+]
 opps = [
     { name = "duckdb" },
     { name = "fsspec" },
@@ -4798,12 +4811,15 @@ requires-dist = [
     { name = "databricks-sdk", marker = "extra == 'lake'", specifier = ">=0.85.0" },
     { name = "duckdb", marker = "extra == 'aco'", specifier = ">=1.5.4" },
     { name = "duckdb", marker = "extra == 'conf'", specifier = ">=1.5.4" },
+    { name = "duckdb", marker = "extra == 'notebooks'", specifier = ">=1.5.4,<2" },
     { name = "duckdb", marker = "extra == 'opps'", specifier = ">=1.5.4" },
     { name = "duckdb", marker = "extra == 'pfs'", specifier = ">=1.5.4" },
     { name = "duckdb", marker = "extra == 'rec'", specifier = ">=1.5.4" },
     { name = "fastapi", marker = "extra == 'api'", specifier = ">=0.139.0" },
     { name = "fastapi", marker = "extra == 'llm'", specifier = ">=0.139.0" },
+    { name = "fastexcel", marker = "extra == 'notebooks'", specifier = ">=0.19.0" },
     { name = "fsspec", marker = "extra == 'bcda'", specifier = ">=2026.6.0" },
+    { name = "fsspec", marker = "extra == 'notebooks'", specifier = ">=2026.6.0" },
     { name = "fsspec", marker = "extra == 'rex'", specifier = ">=2026.6.0" },
     { name = "gcsfs", marker = "extra == 'gcp'", specifier = ">=2024.1.0" },
     { name = "google-cloud-storage", marker = "extra == 'gcp'", specifier = ">=2.18.0" },
@@ -4827,6 +4843,7 @@ requires-dist = [
     { name = "narwhals", marker = "extra == 'pfs'", specifier = ">=2.17.0" },
     { name = "narwhals", marker = "extra == 'rec'", specifier = ">=2.17.0" },
     { name = "narwhals", marker = "extra == 'rex'", specifier = ">=2.17.0" },
+    { name = "openpyxl", marker = "extra == 'notebooks'", specifier = ">=3.1.5" },
     { name = "opentelemetry-api", marker = "extra == 'perf'", specifier = ">=1.45.1" },
     { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'perf'", specifier = ">=1.25.0" },
     { name = "opentelemetry-exporter-prometheus", marker = "extra == 'perf'", specifier = ">=0.46b0" },
@@ -4861,17 +4878,20 @@ requires-dist = [
     { name = "stack", extras = ["aco"], marker = "extra == 'all'" },
     { name = "stack", extras = ["aco"], marker = "extra == 'cli'" },
     { name = "stack", extras = ["aco"], marker = "extra == 'lake'" },
+    { name = "stack", extras = ["aco"], marker = "extra == 'notebooks'" },
     { name = "stack", extras = ["api"], marker = "extra == 'all'" },
     { name = "stack", extras = ["api"], marker = "extra == 'cli'" },
     { name = "stack", extras = ["bcda"], marker = "extra == 'all'" },
     { name = "stack", extras = ["bib"], marker = "extra == 'all'" },
     { name = "stack", extras = ["bib"], marker = "extra == 'cli'" },
     { name = "stack", extras = ["bib"], marker = "extra == 'llm'" },
+    { name = "stack", extras = ["bib"], marker = "extra == 'notebooks'" },
     { name = "stack", extras = ["bib"], marker = "extra == 'prisma'" },
     { name = "stack", extras = ["bls"], marker = "extra == 'all'" },
     { name = "stack", extras = ["ccw"], marker = "extra == 'all'" },
     { name = "stack", extras = ["cli"], marker = "extra == 'all'" },
     { name = "stack", extras = ["cms"], marker = "extra == 'all'" },
+    { name = "stack", extras = ["cms"], marker = "extra == 'notebooks'" },
     { name = "stack", extras = ["conf"], marker = "extra == 'aco'" },
     { name = "stack", extras = ["conf"], marker = "extra == 'api'" },
     { name = "stack", extras = ["conf"], marker = "extra == 'bcda'" },
@@ -4879,6 +4899,7 @@ requires-dist = [
     { name = "stack", extras = ["conf"], marker = "extra == 'lead'" },
     { name = "stack", extras = ["conf"], marker = "extra == 'llm'" },
     { name = "stack", extras = ["conf"], marker = "extra == 'nlm'" },
+    { name = "stack", extras = ["conf"], marker = "extra == 'notebooks'" },
     { name = "stack", extras = ["conf"], marker = "extra == 'opps'" },
     { name = "stack", extras = ["conf"], marker = "extra == 'perf'" },
     { name = "stack", extras = ["conf"], marker = "extra == 'pfs'" },
@@ -4893,8 +4914,10 @@ requires-dist = [
     { name = "stack", extras = ["mail"], marker = "extra == 'cli'" },
     { name = "stack", extras = ["nlm"], marker = "extra == 'all'" },
     { name = "stack", extras = ["nlm"], marker = "extra == 'cli'" },
+    { name = "stack", extras = ["nlm"], marker = "extra == 'notebooks'" },
     { name = "stack", extras = ["perf"], marker = "extra == 'all'" },
     { name = "stack", extras = ["pfs"], marker = "extra == 'all'" },
+    { name = "stack", extras = ["pfs"], marker = "extra == 'notebooks'" },
     { name = "stack", extras = ["pfs"], marker = "extra == 'rec'" },
     { name = "stack", extras = ["qpp"], marker = "extra == 'all'" },
     { name = "stack", extras = ["rec"], marker = "extra == 'all'" },
@@ -4910,7 +4933,7 @@ requires-dist = [
     { name = "uvicorn", marker = "extra == 'llm'", specifier = ">=0.41.0" },
     { name = "xlrd", specifier = ">=2.0.2" },
 ]
-provides-extras = ["conf", "aco", "api", "bcda", "bib", "bls", "llm", "ccw", "lead", "cli", "nlm", "mail", "cms", "opps", "pfs", "qpp", "rec", "rex", "prisma", "perf", "sem", "lake", "aws", "gcp", "azure", "all"]
+provides-extras = ["conf", "aco", "api", "bcda", "bib", "bls", "llm", "ccw", "lead", "cli", "nlm", "mail", "cms", "opps", "pfs", "qpp", "rec", "rex", "prisma", "perf", "sem", "lake", "aws", "gcp", "azure", "notebooks", "all"]
 
 [package.metadata.requires-dev]
 dev = [