fix(llm,notebooks): code_families sections 5/7b run in the notebooks container — lazy pool/sqlalchemy imports in llm, guarded cells, LLM_DB_PASSWORD + psycopg for the notebooks service (refs #720)

compose.yml

@@ -282,6 +282,10 @@ services:
       - storage
     environment:
       - PYTHONPATH=/home/kert/src
+      # code_families.py section 5 counts family-anchored chunks in pgvector
+      # (#720); the container sits on the storage network with postgres.
+      - LLM_DB_PASSWORD=${LLM_DB_PASSWORD}
+      - LLM_PG_HOST=postgres
       - RUSTFS_ENDPOINT=http://rustfs:9000
       - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY}
       - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY}

infra/images/notebooks.Dockerfile

@@ -111,6 +111,7 @@ RUN uv python install ${PYTHON_VERSION} \
         "pyiceberg[s3,pyarrow]>=0.7.0" "duckdb>=1.0.0" "narwhals>=1.0.0" "trino>=0.328.0" \
         "sqlglot>=26.0.0" \
         vega_datasets pyzotero obstore s3fs \
+        "sqlalchemy>=2.0.0" "psycopg[binary]>=3.2.0" \
         --extra-index-url https://pypi.nvidia.com
 
 # Create marimo config directory with minimal defaults.

notebooks/code_families.py

@@ -13,8 +13,7 @@ def _():
 
 @app.cell(hide_code=True)
 def _(mo):
-    mo.md(
-        """
+    mo.md("""
     # Code families as first-class objects
 
     A physician fee schedule code is not a number — it is a **bundle of logical elements**:
@@ -23,8 +22,7 @@ def _(mo):
     into tables you can query and cite: elements → extraction → lineage → families → anchors
     → guidance → public reaction. Every number on this page is read live from the replica and
     the bibliography; every claim links to the Federal Register paragraph it came from.
-        """
-    )
+    """)
     return
 
 
@@ -86,7 +84,18 @@ def _():
     def not_built(cmd):
         return f"_Not built yet — run `{cmd}` and republish the replica._"
 
-    return NOTES, REPLICA_PATH, alt, con, fr_md, not_built, pl, plain_years, q, store
+    return (
+        NOTES,
+        REPLICA_PATH,
+        alt,
+        con,
+        fr_md,
+        not_built,
+        pl,
+        plain_years,
+        q,
+        store,
+    )
 
 
 @app.cell(hide_code=True)
@@ -241,7 +250,7 @@ def _(code_picker, fr_md, mo, pl, store):
             ]
         )
     _view
-    return code, elements
+    return (code,)
 
 
 @app.cell(hide_code=True)
@@ -597,12 +606,15 @@ def _(code, mo, not_built, pl, store):
         )
     else:
         try:
+            # sqlalchemy + psycopg only (the notebooks image, #720) — not
+            # llm.index, which pulls in langchain.
+            from sqlalchemy import create_engine as _sa_engine
             from sqlalchemy import text as _sa_text
 
             from llm.config import load as _load_llm_cfg
-            from llm.index import _engine as _llm_engine
+            from llm.config import pg_url as _pg_url
 
-            _eng = _llm_engine(_load_llm_cfg())
+            _eng = _sa_engine(_pg_url(_load_llm_cfg()))
             with _eng.begin() as _conn:
                 _chunk_rows = _conn.execute(
                     _sa_text(
@@ -615,6 +627,14 @@ def _(code, mo, not_built, pl, store):
                     ),
                     {"key": _key},
                 ).fetchall()
+        except ImportError as e:
+            _panels.append(
+                mo.md(
+                    "_pgvector client not installed in this notebook environment "
+                    f"(`sqlalchemy`/`psycopg` — {e}); chunk-per-collection counts "
+                    "are skipped._"
+                )
+            )
         except Exception as e:  # noqa: BLE001 — degrade, never crash the page
             _panels.append(mo.md(f"_pgvector unavailable: {e}_"))
         else:
@@ -899,10 +919,17 @@ def _(alt, code, con, mo, not_built, pl):
 @app.cell(hide_code=True)
 def _(code, mo, pl):
     # ── 7b. What the chat sees ──
+    _import_err = ""
+    try:
+        # Replica-only reads — no LLM_DB_PASSWORD, no pool, no langchain
+        # (llm's pool exports resolve lazily; #720).
         from llm.config import load as _load_llm_cfg
         from llm.lineage import lineage_evidence as _lineage_evidence
-
-    _cfg = _load_llm_cfg()  # replica-only reads — no LLM_DB_PASSWORD, no pool
+    except ImportError as _e:  # the notebooks image lacks the `llm` extra
+        _import_err = str(_e)
+        _ev = None
+    else:
+        _cfg = _load_llm_cfg()
         _ev = _lineage_evidence(f"history of {code}", _cfg)
 
     _intro = mo.md(
@@ -924,6 +951,11 @@ def _(code, mo, pl):
             [
                 _intro,
                 mo.md(
+                    f"_The chat's lineage module could not be imported here ({_import_err}); "
+                    "this section needs the `llm` package's base dependencies._"
+                )
+                if _import_err
+                else mo.md(
                     "_No lineage events for this code — run "
                     "`stack pfs lineage --all-payable --write`._"
                 ),

src/llm/__init__.py

@@ -15,8 +15,25 @@ Architecture::
 Spec: docs/superpowers/specs/2026-07-16-llm-module-design.md
 """
 
+from __future__ import annotations
+
+from typing import Any
+
 from llm.config import LlmConfig as LlmConfig
 from llm.config import load as load
 from llm.config import pg_url as pg_url
-from llm.pool import HostPool as HostPool
-from llm.pool import PoolEmbeddings as PoolEmbeddings
+
+# The pool pulls in langchain; ``llm.config``, ``llm.links`` and
+# ``llm.lineage`` do not need it and are imported by the notebooks
+# container, which installs the workspace without the ``llm`` extra
+# (#720). Resolve the pool exports lazily so ``import llm.config`` never
+# imports langchain.
+__all__ = ["HostPool", "LlmConfig", "PoolEmbeddings", "load", "pg_url"]
+
+
+def __getattr__(name: str) -> Any:
+    if name in {"HostPool", "PoolEmbeddings"}:
+        from llm import pool
+
+        return getattr(pool, name)
+    raise AttributeError(f"module 'llm' has no attribute {name!r}")

src/llm/evidence.py

@@ -21,7 +21,6 @@ from pathlib import Path
 from typing import Any, Sequence
 
 import duckdb
-from sqlalchemy import text
 
 from llm.config import LlmConfig
 from llm.links import as_source
@@ -279,7 +278,16 @@ def valuation_evidence(question: str, cfg: LlmConfig) -> ValuationEvidence | Non
     )
 
 
-_CITED_SQL = text(
+def _sa_text(sql: str) -> Any:
+    """``sqlalchemy.text`` imported at call time: sqlalchemy is part of the
+    ``llm`` extra, and the notebooks container imports this module for the
+    replica-only paths (``valuation``, ``manual_sources``) without it (#720)."""
+    from sqlalchemy import text
+
+    return text(sql)
+
+
+_CITED_SQL = (
     "SELECT document, cmetadata FROM ( "
     "SELECT e.document, e.cmetadata, "
     "row_number() OVER ( "
@@ -305,7 +313,7 @@ _CITED_SQL = text(
 #: for rules or sequence number for comments/corpus; stage two then ranks
 #: the surviving (one-per-item) rows per family by date and keeps the
 #: newest :window.
-_FAMILY_CITED_SQL = text(
+_FAMILY_CITED_SQL = (
     "SELECT document, cmetadata FROM ( "
     "SELECT document, cmetadata, family, item_key, ordkey, "
     "row_number() OVER ( "
@@ -341,7 +349,7 @@ _FAMILY_CITED_SQL = text(
 #: One chunk per docket instead: the newest chunk (by date, then seq) among
 #: a docket's chunks whose ``codes`` metadata mention any of the wanted
 #: codes, across every matching docket, newest docket year first.
-_DOCKET_CITED_SQL = text(
+_DOCKET_CITED_SQL = (
     "SELECT document, cmetadata FROM ( "
     "SELECT e.document, e.cmetadata, "
     "row_number() OVER ( "
@@ -438,7 +446,8 @@ def _collect_by_docket(
     try:
         with engine.begin() as conn:
             rows = conn.execute(
-                _DOCKET_CITED_SQL, {"collection": collection, "codes": upper}
+                _sa_text(_DOCKET_CITED_SQL),
+                {"collection": collection, "codes": upper},
             ).fetchall()
     except Exception as e:  # noqa: BLE001
         log.warning("docket-cited sources skipped (%s): %s", collection, e)
@@ -526,7 +535,7 @@ def code_cited_sources(
     )
     code_hits = _collect(
         engine,
-        _CITED_SQL,
+        _sa_text(_CITED_SQL),
         "codes",
         codes,
         per=per_code,
@@ -544,7 +553,7 @@ def code_cited_sources(
         out += _interleave(
             _collect(
                 engine,
-                _FAMILY_CITED_SQL,
+                _sa_text(_FAMILY_CITED_SQL),
                 "families",
                 families,
                 per=per_code,

tests/llm/test_init_lazy.py

@@ -0,0 +1,42 @@
+"""#720: the notebooks container installs the workspace without the
+``llm`` extra (no langchain, no sqlalchemy). ``llm.config``,
+``llm.links`` and ``llm.lineage`` must import there anyway — the pool
+exports are resolved lazily, and ``llm.evidence`` only touches
+sqlalchemy when a pgvector query actually runs."""
+
+from __future__ import annotations
+
+import importlib
+import subprocess
+import sys
+
+import pytest
+
+_PROBE = """
+import sys, builtins
+_real = builtins.__import__
+def _fake(name, *a, **k):
+    if name.split('.')[0] in {'langchain_core', 'langchain_postgres', 'langchain_ollama', 'sqlalchemy'}:
+        raise ModuleNotFoundError(name)
+    return _real(name, *a, **k)
+builtins.__import__ = _fake
+import llm.config, llm.links, llm.lineage, llm.evidence  # must not need langchain/sqlalchemy
+import llm
+print('ok', llm.LlmConfig.__name__)
+"""
+
+
+def test_config_links_lineage_evidence_import_without_langchain_or_sqlalchemy():
+    r = subprocess.run(
+        [sys.executable, "-c", _PROBE], capture_output=True, text=True, check=False
+    )
+    assert r.returncode == 0, r.stderr[-800:]
+    assert "ok LlmConfig" in r.stdout
+
+
+def test_pool_exports_resolve_lazily():
+    llm = importlib.import_module("llm")
+    assert llm.HostPool.__name__ == "HostPool"
+    assert llm.PoolEmbeddings.__name__ == "PoolEmbeddings"
+    with pytest.raises(AttributeError):
+        _ = llm.NoSuchName

tests/notebooks/test_code_families_nb.py

@@ -90,3 +90,20 @@ def test_section5_tolerates_missing_llm_db_password(monkeypatch):
     assert family_tables, "expected the bib 'family:*' tag table to render"
     rows = {r["name"]: r["count"] for r in family_tables[0].data.to_dicts()}
     assert rows.get("family:CCM", 0) > 0
+
+
+def test_section_7b_tolerates_missing_llm_deps(monkeypatch):
+    """#720: the notebooks container installs the workspace without the
+    ``llm`` extra. When ``llm.lineage`` cannot be imported, section 7b
+    must render an explanatory note, never a traceback — the rest of the
+    notebook still runs."""
+    import sys
+
+    monkeypatch.setitem(sys.modules, "llm.lineage", None)  # ImportError on import
+    monkeypatch.delenv("LLM_DB_PASSWORD", raising=False)
+    mod = _load()
+    outputs, _defs = mod.app.run()
+    rendered = "\n".join(o._repr_html_() for o in outputs if hasattr(o, "_repr_html_"))
+    assert "7b. What the chat sees" in rendered
+    assert "lineage module could not be imported here" in rendered
+    assert "Traceback" not in rendered