feat(llm): linked sources with kind/date, since filter, /hosts fleet view in chat UI
This commit is contained in:
@@ -10,6 +10,7 @@ Run: ``uvicorn llm.api:app`` (or ``stack llm serve``).
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from importlib import resources
|
||||
from typing import Iterator
|
||||
|
||||
@@ -18,9 +19,16 @@ from fastapi.responses import HTMLResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
app = FastAPI(
|
||||
title="llm chat", description="RAG chat over CMS comments.", version="0.1.0"
|
||||
title="llm chat",
|
||||
description=(
|
||||
"RAG chat over the CMS rulemaking library — comments, FR rules, "
|
||||
"reference corpus."
|
||||
),
|
||||
version="0.2.0",
|
||||
)
|
||||
|
||||
_ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
|
||||
# OTel instrumentation — no-op if perf not installed or telemetry disabled.
|
||||
try:
|
||||
from perf import init as _perf_init
|
||||
@@ -34,6 +42,7 @@ except ImportError: # pragma: no cover — perf always installed in the image
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
question: str
|
||||
since: str | None = None
|
||||
|
||||
|
||||
def _page() -> str:
|
||||
@@ -57,7 +66,7 @@ def whoami(x_auth_request_user: str = Header(default="")) -> dict:
|
||||
return {"user": x_auth_request_user}
|
||||
|
||||
|
||||
def _sse(question: str) -> Iterator[str]:
|
||||
def _sse(question: str, since: str = "") -> Iterator[str]:
|
||||
from llm import config as llm_config
|
||||
from llm.pool import HostPool
|
||||
from llm.rag import stream_answer
|
||||
@@ -65,7 +74,7 @@ def _sse(question: str) -> Iterator[str]:
|
||||
cfg = llm_config.load()
|
||||
pool = HostPool.from_config(cfg)
|
||||
try:
|
||||
for event in stream_answer(question, cfg=cfg, pool=pool):
|
||||
for event in stream_answer(question, cfg=cfg, pool=pool, since=since):
|
||||
yield f"data: {json.dumps(event)}\n\n"
|
||||
except Exception as exc: # surface to the transcript, don't 500 mid-stream
|
||||
yield f"data: {json.dumps({'type': 'error', 'message': str(exc)})}\n\n"
|
||||
@@ -76,4 +85,39 @@ def chat(req: ChatRequest) -> StreamingResponse:
|
||||
question = req.question.strip()
|
||||
if not question:
|
||||
raise HTTPException(status_code=400, detail="empty question")
|
||||
return StreamingResponse(_sse(question), media_type="text/event-stream")
|
||||
since = (req.since or "").strip()
|
||||
if since and not _ISO_DATE.match(since):
|
||||
raise HTTPException(status_code=400, detail="since must be YYYY-MM-DD")
|
||||
return StreamingResponse(_sse(question, since), media_type="text/event-stream")
|
||||
|
||||
|
||||
@app.get("/hosts")
|
||||
def hosts() -> dict:
|
||||
"""The Ollama fleet as the service sees it right now: declared VRAM,
|
||||
liveness, models, and the host + model a chat would use."""
|
||||
from llm import config as llm_config
|
||||
from llm.pool import HostPool, pick_model
|
||||
|
||||
cfg = llm_config.load()
|
||||
pool = HostPool.from_config(cfg)
|
||||
declared = [{"host": h, "vram_gb": pool.vram(h)} for h in pool.hosts]
|
||||
try:
|
||||
live = set(pool.check(cfg.instruct_model))
|
||||
except RuntimeError as exc:
|
||||
return {
|
||||
"hosts": [{**d, "live": False, "models": []} for d in declared],
|
||||
"generation": None,
|
||||
"error": str(exc),
|
||||
}
|
||||
status = {row["host"]: row for row in pool.status()}
|
||||
rows = [
|
||||
{
|
||||
**d,
|
||||
"live": d["host"] in live,
|
||||
"models": status.get(d["host"], {}).get("models", []),
|
||||
}
|
||||
for d in declared
|
||||
]
|
||||
with pool.acquire_generation() as host:
|
||||
model = pick_model(cfg, pool, host)
|
||||
return {"hosts": rows, "generation": {"host": host, "model": model}}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Comment Chat — fhirworx</title>
|
||||
<title>Library Chat</title>
|
||||
<link rel="icon" type="image/png" sizes="32x32" href="//dashboard.fhirworx.io/fav32.png">
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
@@ -104,6 +104,17 @@
|
||||
}
|
||||
button:hover:not(:disabled) { background: #253748; }
|
||||
button:disabled { opacity: .45; cursor: default; }
|
||||
form label.recent {
|
||||
display: flex; align-items: center; gap: 6px; font-size: 13px;
|
||||
color: var(--muted-fg); white-space: nowrap;
|
||||
}
|
||||
.src a { color: var(--primary); text-decoration: underline dotted; }
|
||||
.src .kind {
|
||||
font-family: var(--font-mono); font-size: 11px; padding: 1px 5px; border-radius: 3px;
|
||||
background: color-mix(in srgb, var(--primary) 12%, transparent); margin-right: 6px;
|
||||
}
|
||||
.src .date { color: var(--muted-fg); margin-left: 6px; }
|
||||
.meta { font-size: 12px; color: var(--muted-fg); margin-top: 4px; font-family: var(--font-mono); }
|
||||
.hint {
|
||||
margin: auto; max-width: 500px; text-align: center; color: var(--muted-fg);
|
||||
font-style: italic;
|
||||
@@ -112,16 +123,18 @@
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Comment Chat</h1>
|
||||
<span class="sub">grounded in indexed CMS rulemaking comments</span>
|
||||
<h1>Library Chat</h1>
|
||||
<span class="sub">grounded in the indexed library — comments, FR rules, references</span>
|
||||
<span class="who" id="who"></span>
|
||||
</header>
|
||||
<div id="log">
|
||||
<div class="hint">Ask a question about the indexed CMS rulemaking comments.
|
||||
Answers are drawn only from the comments and cite their ids.</div>
|
||||
<div class="hint">Ask a question about CMS rulemaking. Answers draw on the
|
||||
indexed comments, Federal Register rules and reference library, prefer the
|
||||
most recent comments, and link each citation to its source passage.</div>
|
||||
</div>
|
||||
<form id="form">
|
||||
<textarea id="q" placeholder="e.g. What do commenters say about telehealth originating sites?" autofocus></textarea>
|
||||
<label class="recent"><input type="checkbox" id="recent"> last 12 months</label>
|
||||
<button id="send" type="submit">Send</button>
|
||||
</form>
|
||||
|
||||
@@ -152,7 +165,7 @@
|
||||
// Render answer text, wrapping [CMS-...] citations in styled spans.
|
||||
function paint(el, text) {
|
||||
el.innerHTML = '';
|
||||
const re = /\[([A-Z0-9-]+)\]/g;
|
||||
const re = /\[([^\[\]\n]{2,90})\]/g;
|
||||
let last = 0, m;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
if (m.index > last) el.appendChild(document.createTextNode(text.slice(last, m.index)));
|
||||
@@ -164,19 +177,44 @@
|
||||
if (last < text.length) el.appendChild(document.createTextNode(text.slice(last)));
|
||||
}
|
||||
|
||||
function renderSources(wrap, sources) {
|
||||
if (!sources || !sources.length) return;
|
||||
function renderSources(wrap, ev) {
|
||||
const sources = ev.sources || [];
|
||||
if (ev.model) {
|
||||
const meta = document.createElement('div');
|
||||
meta.className = 'meta';
|
||||
meta.textContent = ev.model + ' @ ' + (ev.host || '').replace(/^https?:\/\//, '');
|
||||
wrap.appendChild(meta);
|
||||
}
|
||||
if (!sources.length) return;
|
||||
const d = document.createElement('details');
|
||||
d.className = 'sources';
|
||||
const s = document.createElement('summary');
|
||||
s.textContent = sources.length + ' source comment' + (sources.length > 1 ? 's' : '');
|
||||
s.textContent = sources.length + ' source' + (sources.length > 1 ? 's' : '');
|
||||
d.appendChild(s);
|
||||
for (const src of sources) {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'src';
|
||||
const kind = document.createElement('span');
|
||||
kind.className = 'kind'; kind.textContent = src.kind || 'comment';
|
||||
el.appendChild(kind);
|
||||
const id = document.createElement('b');
|
||||
id.textContent = '[' + src.comment_id + ']';
|
||||
if (src.url) {
|
||||
const a = document.createElement('a');
|
||||
a.href = src.url; a.target = '_blank'; a.rel = 'noopener';
|
||||
a.textContent = '[' + src.label + ']';
|
||||
id.appendChild(a);
|
||||
} else {
|
||||
id.textContent = '[' + src.label + ']';
|
||||
}
|
||||
el.appendChild(id);
|
||||
if (src.date) {
|
||||
const dt = document.createElement('span');
|
||||
dt.className = 'date'; dt.textContent = src.date;
|
||||
el.appendChild(dt);
|
||||
}
|
||||
if (src.title && src.kind !== 'comment') {
|
||||
el.appendChild(document.createTextNode(' — ' + src.title));
|
||||
}
|
||||
el.appendChild(document.createTextNode(' ' + src.snippet));
|
||||
d.appendChild(el);
|
||||
}
|
||||
@@ -191,11 +229,13 @@
|
||||
const { wrap, b } = bubble('bot');
|
||||
let answer = '';
|
||||
send.disabled = true;
|
||||
const recent = document.getElementById('recent').checked;
|
||||
const since = recent ? new Date(Date.now() - 365 * 864e5).toISOString().slice(0, 10) : null;
|
||||
try {
|
||||
const resp = await fetch('chat', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ question })
|
||||
body: JSON.stringify({ question, since })
|
||||
});
|
||||
if (!resp.ok) throw new Error('request failed (' + resp.status + ')');
|
||||
const reader = resp.body.getReader();
|
||||
@@ -212,7 +252,7 @@
|
||||
if (!line) continue;
|
||||
const ev = JSON.parse(line);
|
||||
if (ev.type === 'token') { answer += ev.text; paint(b, answer); log.scrollTop = log.scrollHeight; }
|
||||
else if (ev.type === 'sources') renderSources(wrap, ev.sources);
|
||||
else if (ev.type === 'sources') renderSources(wrap, ev);
|
||||
else if (ev.type === 'error') { b.className = 'bubble err'; b.textContent = 'Error: ' + ev.message; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ class TestIndex:
|
||||
r = client.get("/")
|
||||
assert r.status_code == 200
|
||||
assert "text/html" in r.headers["content-type"]
|
||||
assert "Comment Chat" in r.text
|
||||
assert "Library Chat" in r.text
|
||||
assert 'id="form"' in r.text
|
||||
|
||||
|
||||
@@ -63,3 +63,63 @@ class TestChat:
|
||||
assert r.status_code == 200
|
||||
assert '"type": "error"' in r.text
|
||||
assert "ollama down" in r.text
|
||||
|
||||
|
||||
class TestChatSince:
|
||||
@patch("llm.rag.stream_answer")
|
||||
def test_since_forwarded(self, mock_stream):
|
||||
mock_stream.return_value = iter([{"type": "done"}])
|
||||
r = client.post("/chat", json={"question": "q", "since": "2025-09-01"})
|
||||
assert r.status_code == 200
|
||||
assert mock_stream.call_args.kwargs["since"] == "2025-09-01"
|
||||
|
||||
def test_bad_since_400(self):
|
||||
r = client.post("/chat", json={"question": "q", "since": "last year"})
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
def _cfg(**kw):
|
||||
from llm.config import LlmConfig
|
||||
|
||||
base = dict(
|
||||
ollama_hosts=("http://h1:11434", "http://h2:11434"),
|
||||
host_vram={"http://h1:11434": 12, "http://h2:11434": 24},
|
||||
embed_model="e",
|
||||
instruct_model="chat",
|
||||
embed_dim=768,
|
||||
build_ann_index=False,
|
||||
pg_host="x",
|
||||
pg_port=5432,
|
||||
pg_db="llm",
|
||||
pg_user="llm",
|
||||
)
|
||||
base.update(kw)
|
||||
return LlmConfig(**base)
|
||||
|
||||
|
||||
class TestHosts:
|
||||
@patch("llm.pool.pick_model", return_value="big")
|
||||
@patch("llm.pool.HostPool.check", return_value=["http://h2:11434"])
|
||||
@patch("llm.pool.HostPool.status")
|
||||
@patch("llm.config.load")
|
||||
def test_reports_fleet_and_pick(self, mock_load, mock_status, _check, _pm):
|
||||
mock_load.return_value = _cfg()
|
||||
mock_status.return_value = [
|
||||
{"host": "http://h2:11434", "vram_gb": 24.0, "models": ["chat:latest"]}
|
||||
]
|
||||
r = client.get("/hosts")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["generation"] == {"host": "http://h2:11434", "model": "big"}
|
||||
assert [(h["host"], h["live"]) for h in body["hosts"]] == [
|
||||
("http://h1:11434", False),
|
||||
("http://h2:11434", True),
|
||||
]
|
||||
|
||||
@patch("llm.pool.HostPool.check", side_effect=RuntimeError("no Ollama host"))
|
||||
@patch("llm.config.load")
|
||||
def test_no_live_hosts(self, mock_load, _check):
|
||||
mock_load.return_value = _cfg(ollama_hosts=("http://h1:11434",))
|
||||
r = client.get("/hosts")
|
||||
assert r.json()["generation"] is None
|
||||
assert "no Ollama host" in r.json()["error"]
|
||||
|
||||
Reference in New Issue
Block a user