diff --git a/infra/images/notebooks.Dockerfile b/infra/images/notebooks.Dockerfile index f31ba6b..68f7a7e 100644 --- a/infra/images/notebooks.Dockerfile +++ b/infra/images/notebooks.Dockerfile @@ -66,6 +66,47 @@ WORKDIR /src COPY --from=fe /src/ /src/ RUN uv build --wheel +# ---- stage 2b: notebook requirements ---------------------------------------- +# +# Notebooks import src/ (bind-mounted, on PYTHONPATH), so the image needs the +# stack's base dependencies plus pyproject's `notebooks` extra. Expand its +# stack[...] self-references into plain requirements here; the heavy install +# layer below then rebuilds only when that list changes, not on every +# pyproject edit. `uv add` keeps just the last spec it sees per package, so +# repeated names (duckdb comes from four extras) fold into one requirement. + +FROM ghcr.io/astral-sh/uv:0.13.2-python3.13-trixie-slim AS nbreqs +COPY pyproject.toml /pyproject.toml +RUN python3 - > /notebooks-requirements.txt <<'PY' +import re, tomllib + +project = tomllib.load(open("/pyproject.toml", "rb"))["project"] +extras = project["optional-dependencies"] +reqs, seen = list(project["dependencies"]), set() + +def walk(extra): + if extra in seen: + return + seen.add(extra) + for req in extras[extra]: + if m := re.fullmatch(r"stack\[([\w,-]+)\]", req): + for sub in m.group(1).split(","): + walk(sub) + else: + reqs.append(req) + +walk("notebooks") +merged = {} +for req in reqs: + name, ext, spec = re.fullmatch(r"([\w.-]+)(?:\[([^\]]*)\])?\s*([^;]*)", req).groups() + _, e, s = merged.setdefault(re.sub(r"[-_.]+", "-", name).lower(), [name, set(), []]) + e.update(x.strip() for x in (ext or "").split(",") if x.strip()) + s.extend(c.strip() for c in spec.split(",") if c.strip() and c.strip() not in s) +for name, e, s in merged.values(): + print(name + (f"[{','.join(sorted(e))}]" if e else "") + ",".join(s)) +PY +RUN cat /notebooks-requirements.txt + # ---- stage 3: runtime ------------------------------------------------------- FROM nvidia/cuda:12.6.0-runtime-ubuntu24.04 @@ -99,28 +140,27 @@ WORKDIR /home/${USERNAME} # Bring in the fhirworx marimo wheel from the wheel stage. COPY --from=wheel /src/dist/ /tmp/marimo-wheel/ +COPY --from=nbreqs /notebooks-requirements.txt /tmp/notebooks-requirements.txt # No release cooldown here (unlike pyproject.toml's exclude-newer): this # workspace pulls cudf-polars-cu12 from pypi.nvidia.com, which publishes no # upload dates, so any exclude-newer filters out every release and the build # fails. uv has no per-package opt-out for date-less indexes. -# duckdb stays on the 1.x line the rest of the stack locks (1.5.x): this -# workspace has no lock, and every service shares the same .duckdb files, -# so a new major here must not be the first engine to touch them. - # Install python + init workspace venv. Use the local wheel for marimo so # the fhirworx frontend ships in the image; no upstream PyPI fetch for -# marimo itself. +# marimo itself. Packages the stack's code needs (duckdb, narwhals, sqlglot, +# pyarrow, httpx, ...) come from the requirements file, not this list — a +# name listed in both would have its spec replaced by whichever comes last. RUN uv python install ${PYTHON_VERSION} \ && uv init workspace --python ${PYTHON_VERSION} \ && cd workspace \ && MARIMO_WHL=$(ls /tmp/marimo-wheel/marimo-*.whl | head -1) \ - && uv add "${MARIMO_WHL}[recommended]" polars cudf-polars-cu12 pandas numpy pyarrow \ - "pyiceberg[s3,pyarrow]>=0.7.0" "duckdb>=1.5.4,<2" "narwhals>=1.0.0" "trino>=0.328.0" \ - "sqlglot>=26.0.0" \ + && uv add "${MARIMO_WHL}[recommended]" polars cudf-polars-cu12 pandas numpy \ + "pyiceberg[s3,pyarrow]>=0.7.0" "trino>=0.328.0" \ vega_datasets pyzotero obstore s3fs \ "sqlalchemy>=2.0.0" "psycopg[binary]>=3.2.0" \ + -r /tmp/notebooks-requirements.txt \ --extra-index-url https://pypi.nvidia.com # Create marimo config directory with minimal defaults. diff --git a/pyproject.toml b/pyproject.toml index eee2a4c..ffec660 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,6 +168,24 @@ azure = [ "azure-storage-blob>=12.23.0", "adlfs>=2024.1.0", ] +# What notebooks/ reaches through src/. The notebooks container bind-mounts +# src/ and installs this extra (plus the base dependencies) — see +# infra/images/notebooks.Dockerfile — so a dependency a notebook-reachable +# module gains belongs in its own extra, and that extra belongs here. +notebooks = [ + "stack[conf]", + "stack[aco]", + "stack[bib]", + "stack[cms]", + "stack[nlm]", + "stack[pfs]", + # The image resolves without a lock, and every service shares the same + # .duckdb files, so a new duckdb major must not land here first. + "duckdb>=1.5.4,<2", + "fsspec>=2026.6.0", # bcda.store; the bcda extra itself drags in streamlit + "openpyxl>=3.1.5", # pfs.pipe reads CMS workbooks + "fastexcel>=0.19.0", # polars.read_excel's calamine engine +] all = [ "stack[aco]", "stack[api]", diff --git a/uv.lock b/uv.lock index d4522e0..581cc95 100644 --- a/uv.lock +++ b/uv.lock @@ -14,7 +14,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-10-20T19:35:11.505292094Z" +exclude-newer = "2026-10-22T18:36:04.896204313Z" exclude-newer-span = "P7D" [options.exclude-newer-package] @@ -4698,6 +4698,19 @@ nlm = [ { name = "httpx" }, { name = "pydantic" }, ] +notebooks = [ + { name = "duckdb" }, + { name = "fastexcel" }, + { name = "fsspec" }, + { name = "httpx" }, + { name = "markdown-it-py" }, + { name = "mdit-py-plugins" }, + { name = "narwhals" }, + { name = "openpyxl" }, + { name = "pyarrow" }, + { name = "pydantic" }, + { name = "sqlglot" }, +] opps = [ { name = "duckdb" }, { name = "fsspec" }, @@ -4798,12 +4811,15 @@ requires-dist = [ { name = "databricks-sdk", marker = "extra == 'lake'", specifier = ">=0.85.0" }, { name = "duckdb", marker = "extra == 'aco'", specifier = ">=1.5.4" }, { name = "duckdb", marker = "extra == 'conf'", specifier = ">=1.5.4" }, + { name = "duckdb", marker = "extra == 'notebooks'", specifier = ">=1.5.4,<2" }, { name = "duckdb", marker = "extra == 'opps'", specifier = ">=1.5.4" }, { name = "duckdb", marker = "extra == 'pfs'", specifier = ">=1.5.4" }, { name = "duckdb", marker = "extra == 'rec'", specifier = ">=1.5.4" }, { name = "fastapi", marker = "extra == 'api'", specifier = ">=0.139.0" }, { name = "fastapi", marker = "extra == 'llm'", specifier = ">=0.139.0" }, + { name = "fastexcel", marker = "extra == 'notebooks'", specifier = ">=0.19.0" }, { name = "fsspec", marker = "extra == 'bcda'", specifier = ">=2026.6.0" }, + { name = "fsspec", marker = "extra == 'notebooks'", specifier = ">=2026.6.0" }, { name = "fsspec", marker = "extra == 'rex'", specifier = ">=2026.6.0" }, { name = "gcsfs", marker = "extra == 'gcp'", specifier = ">=2024.1.0" }, { name = "google-cloud-storage", marker = "extra == 'gcp'", specifier = ">=2.18.0" }, @@ -4827,6 +4843,7 @@ requires-dist = [ { name = "narwhals", marker = "extra == 'pfs'", specifier = ">=2.17.0" }, { name = "narwhals", marker = "extra == 'rec'", specifier = ">=2.17.0" }, { name = "narwhals", marker = "extra == 'rex'", specifier = ">=2.17.0" }, + { name = "openpyxl", marker = "extra == 'notebooks'", specifier = ">=3.1.5" }, { name = "opentelemetry-api", marker = "extra == 'perf'", specifier = ">=1.45.1" }, { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'perf'", specifier = ">=1.25.0" }, { name = "opentelemetry-exporter-prometheus", marker = "extra == 'perf'", specifier = ">=0.46b0" }, @@ -4861,17 +4878,20 @@ requires-dist = [ { name = "stack", extras = ["aco"], marker = "extra == 'all'" }, { name = "stack", extras = ["aco"], marker = "extra == 'cli'" }, { name = "stack", extras = ["aco"], marker = "extra == 'lake'" }, + { name = "stack", extras = ["aco"], marker = "extra == 'notebooks'" }, { name = "stack", extras = ["api"], marker = "extra == 'all'" }, { name = "stack", extras = ["api"], marker = "extra == 'cli'" }, { name = "stack", extras = ["bcda"], marker = "extra == 'all'" }, { name = "stack", extras = ["bib"], marker = "extra == 'all'" }, { name = "stack", extras = ["bib"], marker = "extra == 'cli'" }, { name = "stack", extras = ["bib"], marker = "extra == 'llm'" }, + { name = "stack", extras = ["bib"], marker = "extra == 'notebooks'" }, { name = "stack", extras = ["bib"], marker = "extra == 'prisma'" }, { name = "stack", extras = ["bls"], marker = "extra == 'all'" }, { name = "stack", extras = ["ccw"], marker = "extra == 'all'" }, { name = "stack", extras = ["cli"], marker = "extra == 'all'" }, { name = "stack", extras = ["cms"], marker = "extra == 'all'" }, + { name = "stack", extras = ["cms"], marker = "extra == 'notebooks'" }, { name = "stack", extras = ["conf"], marker = "extra == 'aco'" }, { name = "stack", extras = ["conf"], marker = "extra == 'api'" }, { name = "stack", extras = ["conf"], marker = "extra == 'bcda'" }, @@ -4879,6 +4899,7 @@ requires-dist = [ { name = "stack", extras = ["conf"], marker = "extra == 'lead'" }, { name = "stack", extras = ["conf"], marker = "extra == 'llm'" }, { name = "stack", extras = ["conf"], marker = "extra == 'nlm'" }, + { name = "stack", extras = ["conf"], marker = "extra == 'notebooks'" }, { name = "stack", extras = ["conf"], marker = "extra == 'opps'" }, { name = "stack", extras = ["conf"], marker = "extra == 'perf'" }, { name = "stack", extras = ["conf"], marker = "extra == 'pfs'" }, @@ -4893,8 +4914,10 @@ requires-dist = [ { name = "stack", extras = ["mail"], marker = "extra == 'cli'" }, { name = "stack", extras = ["nlm"], marker = "extra == 'all'" }, { name = "stack", extras = ["nlm"], marker = "extra == 'cli'" }, + { name = "stack", extras = ["nlm"], marker = "extra == 'notebooks'" }, { name = "stack", extras = ["perf"], marker = "extra == 'all'" }, { name = "stack", extras = ["pfs"], marker = "extra == 'all'" }, + { name = "stack", extras = ["pfs"], marker = "extra == 'notebooks'" }, { name = "stack", extras = ["pfs"], marker = "extra == 'rec'" }, { name = "stack", extras = ["qpp"], marker = "extra == 'all'" }, { name = "stack", extras = ["rec"], marker = "extra == 'all'" }, @@ -4910,7 +4933,7 @@ requires-dist = [ { name = "uvicorn", marker = "extra == 'llm'", specifier = ">=0.41.0" }, { name = "xlrd", specifier = ">=2.0.2" }, ] -provides-extras = ["conf", "aco", "api", "bcda", "bib", "bls", "llm", "ccw", "lead", "cli", "nlm", "mail", "cms", "opps", "pfs", "qpp", "rec", "rex", "prisma", "perf", "sem", "lake", "aws", "gcp", "azure", "all"] +provides-extras = ["conf", "aco", "api", "bcda", "bib", "bls", "llm", "ccw", "lead", "cli", "nlm", "mail", "cms", "opps", "pfs", "qpp", "rec", "rex", "prisma", "perf", "sem", "lake", "aws", "gcp", "azure", "notebooks", "all"] [package.metadata.requires-dev] dev = [