feat(llm): host VRAM annotations + recency/model-tier config knobs (refs #654)

src/llm/config.py

@@ -1,16 +1,25 @@
 """[llm] configuration — stack.toml section + env overrides.
 
 Env contract:
-    LLM_OLLAMA_HOSTS     comma-separated Ollama base URLs (beats [llm].ollama)
+    LLM_OLLAMA_HOSTS     comma-separated Ollama base URLs, each optionally
+                         annotated with declared VRAM: ``http://rig:11434@24``
+                         (beats [llm].ollama)
     LLM_PG_HOST          beats [llm].pg_host (containers set this to "postgres")
     LLM_DB_PASSWORD      required for pg_url(); lives in .env, never stack.toml
     LLM_BUILD_ANN_INDEX  beats [llm].build_ann_index ("1"/"true"/"yes" = True)
+
+Ollama has no GPU-size API, so VRAM is *declared* per host; liveness is
+probed at request time (``HostPool.check``). Together they implement
+"generate on the largest GPU that is up right now".
 """
 
 from __future__ import annotations
 
 import os
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from typing import Any
+
+_DEFAULT_K_PER_KIND = {"comment": 8, "rule": 4, "corpus": 4}
 
 
 @dataclass(frozen=True)
@@ -24,6 +33,38 @@ class LlmConfig:
     pg_port: int
     pg_db: str
     pg_user: str
+    host_vram: dict[str, float] = field(default_factory=dict)
+    instruct_model_large: str = ""
+    large_min_vram_gb: float = 20.0
+    chat_num_ctx: int = 8192
+    recency_half_life_days: float = 365.0
+    recency_weight: float = 0.3
+    k_per_kind: dict[str, int] = field(
+        default_factory=lambda: dict(_DEFAULT_K_PER_KIND)
+    )
+    top_n: int = 8
+
+
+def parse_hosts(spec: str) -> tuple[tuple[str, float], ...]:
+    """``"http://a:1@24, http://b:1"`` → ``(("http://a:1", 24.0), ("http://b:1", 0.0))``.
+
+    A missing annotation means "unknown size" (0.0), which sorts last for
+    generation but still serves embeds.
+    """
+    out: list[tuple[str, float]] = []
+    for part in spec.split(","):
+        part = part.strip()
+        if not part:
+            continue
+        url, vram = part, "0"
+        if "@" in part:
+            url, vram = part.rsplit("@", 1)
+        out.append((url.strip().rstrip("/"), float(vram or 0)))
+    return tuple(out)
+
+
+def _opt(section: Any, key: str, default: Any) -> Any:
+    return section[key] if key in section else default
 
 
 def load() -> LlmConfig:
@@ -32,19 +73,30 @@ def load() -> LlmConfig:
 
     section = cfg.llm
     hosts_env = os.environ.get("LLM_OLLAMA_HOSTS", "")
-    if hosts_env:
-        hosts = tuple(h.strip() for h in hosts_env.split(",") if h.strip())
-    else:
-        hosts = (str(section.ollama),)
+    pairs = parse_hosts(hosts_env if hosts_env else str(section.ollama))
     ann_env = os.environ.get("LLM_BUILD_ANN_INDEX")
     if ann_env is not None:
         build_ann_index = ann_env.strip().lower() in ("1", "true", "yes")
     else:
         build_ann_index = bool(section.build_ann_index)
+    k_raw = _opt(section, "k_per_kind", None)
+    k_per_kind = (
+        {str(k): int(v) for k, v in dict(k_raw._data).items()}
+        if k_raw is not None
+        else dict(_DEFAULT_K_PER_KIND)
+    )
     return LlmConfig(
-        ollama_hosts=hosts,
+        ollama_hosts=tuple(url for url, _ in pairs),
+        host_vram={url: vram for url, vram in pairs},
         embed_model=str(section.embed_model),
         instruct_model=str(section.instruct_model),
+        instruct_model_large=str(_opt(section, "instruct_model_large", "")),
+        large_min_vram_gb=float(_opt(section, "large_min_vram_gb", 20.0)),
+        chat_num_ctx=int(_opt(section, "chat_num_ctx", 8192)),
+        recency_half_life_days=float(_opt(section, "recency_half_life_days", 365.0)),
+        recency_weight=float(_opt(section, "recency_weight", 0.3)),
+        k_per_kind=k_per_kind,
+        top_n=int(_opt(section, "top_n", 8)),
         embed_dim=int(section.embed_dim),
         build_ann_index=build_ann_index,
         pg_host=os.environ.get("LLM_PG_HOST", str(section.pg_host)),

tests/llm/test_config.py

@@ -51,3 +51,39 @@ class TestPgUrl:
         cfg = llm_config.load()
         with pytest.raises(RuntimeError, match="LLM_DB_PASSWORD"):
             llm_config.pg_url(cfg)
+
+
+class TestParseHosts:
+    def test_bare_and_annotated(self):
+        out = llm_config.parse_hosts(
+            "http://ollama:11434, http://rig.local:11434@24 ,http://nb:11434@12"
+        )
+        assert out == (
+            ("http://ollama:11434", 0.0),
+            ("http://rig.local:11434", 24.0),
+            ("http://nb:11434", 12.0),
+        )
+
+    def test_trailing_slash_stripped_and_empty_parts_dropped(self):
+        assert llm_config.parse_hosts("http://a:1/@8,,") == (("http://a:1", 8.0),)
+
+
+class TestNewKnobs:
+    def test_defaults_when_section_lacks_keys(self, monkeypatch):
+        monkeypatch.delenv("LLM_OLLAMA_HOSTS", raising=False)
+        cfg = llm_config.load()
+        assert cfg.large_min_vram_gb == 20.0
+        assert cfg.chat_num_ctx == 8192
+        assert cfg.recency_half_life_days == 365.0
+        assert cfg.recency_weight == 0.3
+        assert cfg.k_per_kind == {"comment": 8, "rule": 4, "corpus": 4}
+        assert cfg.top_n == 8
+        assert isinstance(cfg.instruct_model_large, str)
+
+    def test_env_hosts_populate_vram_map(self, monkeypatch):
+        monkeypatch.setenv(
+            "LLM_OLLAMA_HOSTS", "http://rig:11434@24,http://laptop:11434@12"
+        )
+        cfg = llm_config.load()
+        assert cfg.ollama_hosts == ("http://rig:11434", "http://laptop:11434")
+        assert cfg.host_vram == {"http://rig:11434": 24.0, "http://laptop:11434": 12.0}