feat(llm): SSO-guarded RAG chat UI at llm.fhirworx.io (P34)
New llm FastAPI service (src/llm/api.py + rag.py + web/chat.html): grounded streaming chat over indexed comments with cited sources. Own image, compose service, Traefik reef entry with git-sso, llm subdomain registered. Dashboard tile + README row. stack llm serve CLI.
This commit is contained in:
kert
@@ -202,6 +202,55 @@ jobs:
|
||||
--sha "${{ github.sha }}" \
|
||||
--ref "${{ github.ref }}" || true
|
||||
|
||||
llm:
|
||||
runs-on: ubuntu-latest
|
||||
if: >-
|
||||
contains(github.event.head_commit.modified, 'infra/images/llm.Dockerfile') ||
|
||||
contains(github.event.head_commit.modified, 'src') ||
|
||||
contains(github.event.head_commit.modified, 'pyproject.toml')
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- name: Install crane
|
||||
run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
||||
|
||||
- name: Log in to registry
|
||||
run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
||||
env:
|
||||
CRANE_INSECURE: "true"
|
||||
|
||||
|
||||
- name: Install trivy
|
||||
run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
||||
|
||||
- name: Scan llm
|
||||
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o llm-scan.json local/llm:build
|
||||
|
||||
- name: Compute short SHA
|
||||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build llm
|
||||
run: docker build -f infra/images/llm.Dockerfile -t local/llm:build .
|
||||
|
||||
- name: Push llm
|
||||
run: |
|
||||
docker save local/llm:build -o /tmp/llm.tar
|
||||
crane push /tmp/llm.tar git:3000/homelab/stack/llm:${{ env.SHORT_SHA }} --insecure
|
||||
crane push /tmp/llm.tar git:3000/homelab/stack/llm:latest --insecure
|
||||
|
||||
- name: File failure issue
|
||||
if: failure()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
|
||||
run: |
|
||||
uv sync --no-dev --quiet 2>/dev/null || true
|
||||
uv run python -m api.diag.ci \
|
||||
--workflow "Deploy" --job "llm" \
|
||||
--run "${{ github.run_number }}" \
|
||||
--sha "${{ github.sha }}" \
|
||||
--ref "${{ github.ref }}" || true
|
||||
|
||||
mc:
|
||||
runs-on: ubuntu-latest
|
||||
if: >-
|
||||
@@ -246,7 +295,7 @@ jobs:
|
||||
|
||||
report:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [notebooks, zotero, docs, api, mc]
|
||||
needs: [notebooks, zotero, docs, api, llm, mc]
|
||||
if: always() && !cancelled()
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
@@ -67,6 +67,15 @@ jobs:
|
||||
crane push /tmp/api.tar git:3000/homelab/stack/api:hardened --insecure
|
||||
crane push /tmp/api.tar git:3000/homelab/stack/api:latest --insecure
|
||||
|
||||
- name: Build llm
|
||||
run: docker build --no-cache -f infra/images/llm.Dockerfile -t local/llm:build .
|
||||
|
||||
- name: Push llm
|
||||
run: |
|
||||
docker save local/llm:build -o /tmp/llm.tar
|
||||
crane push /tmp/llm.tar git:3000/homelab/stack/llm:hardened --insecure
|
||||
crane push /tmp/llm.tar git:3000/homelab/stack/llm:latest --insecure
|
||||
|
||||
- name: Build mc
|
||||
run: docker build --no-cache -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||
|
||||
@@ -88,12 +97,15 @@ jobs:
|
||||
- name: Scan api
|
||||
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o api-scan.json local/api:build
|
||||
|
||||
- name: Scan llm
|
||||
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o llm-scan.json local/llm:build
|
||||
|
||||
- name: Close resolved or file new vuln issues
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
|
||||
run: |
|
||||
uv sync --no-dev
|
||||
for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json; do
|
||||
for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json llm-scan.json; do
|
||||
if [ -f "$f" ]; then
|
||||
uv run python -m api.diag.vuln --close "$f" || \
|
||||
uv run python -m api.diag.vuln "$f" || true
|
||||
|
||||
@@ -16,6 +16,9 @@ on:
|
||||
- 'infra/images/api.Dockerfile'
|
||||
- 'src/**'
|
||||
- 'pyproject.toml'
|
||||
- 'infra/images/llm.Dockerfile'
|
||||
- 'src/**'
|
||||
- 'pyproject.toml'
|
||||
- 'infra/rustfs/**'
|
||||
- 'infra/images/mc.Dockerfile'
|
||||
pull_request:
|
||||
@@ -30,6 +33,9 @@ on:
|
||||
- 'infra/images/api.Dockerfile'
|
||||
- 'src/**'
|
||||
- 'pyproject.toml'
|
||||
- 'infra/images/llm.Dockerfile'
|
||||
- 'src/**'
|
||||
- 'pyproject.toml'
|
||||
- 'infra/rustfs/**'
|
||||
- 'infra/images/mc.Dockerfile'
|
||||
|
||||
@@ -149,6 +155,34 @@ jobs:
|
||||
--sha "${{ github.sha }}" \
|
||||
--ref "${{ github.ref }}" || true
|
||||
|
||||
llm:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- name: Hadolint llm
|
||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||
with:
|
||||
dockerfile: infra/images/llm.Dockerfile
|
||||
|
||||
|
||||
|
||||
- name: Build llm
|
||||
run: docker build -f infra/images/llm.Dockerfile -t local/llm:build .
|
||||
|
||||
- name: File failure issue
|
||||
if: failure()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
|
||||
run: |
|
||||
uv sync --no-dev --quiet 2>/dev/null || true
|
||||
uv run python -m api.diag.ci \
|
||||
--workflow "Infra CI" --job "llm" \
|
||||
--run "${{ github.run_number }}" \
|
||||
--sha "${{ github.sha }}" \
|
||||
--ref "${{ github.ref }}" || true
|
||||
|
||||
mc:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
@@ -68,6 +68,15 @@ jobs:
|
||||
crane push /tmp/api.tar git:3000/homelab/stack/api:${{ env.SHORT_SHA }} --insecure
|
||||
crane push /tmp/api.tar git:3000/homelab/stack/api:latest --insecure
|
||||
|
||||
- name: Build llm
|
||||
run: docker build -f infra/images/llm.Dockerfile -t local/llm:build .
|
||||
|
||||
- name: Push llm
|
||||
run: |
|
||||
docker save local/llm:build -o /tmp/llm.tar
|
||||
crane push /tmp/llm.tar git:3000/homelab/stack/llm:${{ env.SHORT_SHA }} --insecure
|
||||
crane push /tmp/llm.tar git:3000/homelab/stack/llm:latest --insecure
|
||||
|
||||
- name: Build mc
|
||||
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||
|
||||
@@ -89,12 +98,15 @@ jobs:
|
||||
- name: Scan api
|
||||
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o api-scan.json local/api:build
|
||||
|
||||
- name: Scan llm
|
||||
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o llm-scan.json local/llm:build
|
||||
|
||||
- name: Report vulnerabilities
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
|
||||
run: |
|
||||
uv sync --no-dev
|
||||
for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json; do
|
||||
for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json llm-scan.json; do
|
||||
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
|
||||
done
|
||||
|
||||
|
||||
16
README.md
16
README.md
@@ -2,7 +2,7 @@
|
||||
|
||||
|
||||
|
||||
Healthcare analytics platform on self-hosted infrastructure. Replaces dbt SQL models with narwhals DataFrame-agnostic expression functions, backed by DuckDB locally and Iceberg/Trino in the lakehouse. 29 services, 14 data pipelines, 13,847 tests at 99% coverage.
|
||||
Healthcare analytics platform on self-hosted infrastructure. Replaces dbt SQL models with narwhals DataFrame-agnostic expression functions, backed by DuckDB locally and Iceberg/Trino in the lakehouse. 32 services, 14 data pipelines, 14,010 tests at 99% coverage.
|
||||
|
||||
## Quick start
|
||||
|
||||
@@ -37,7 +37,7 @@ pip install stack[all] # everything
|
||||
pip install stack[aco,aws] # ACO analytics with AWS storage
|
||||
```
|
||||
|
||||
17 modules available as optional extras: `aco`, `api`, `bcda`, `bib`, `bls`, `ccw`, `cli`, `cms`, `conf`, `mail`, `opps`, `perf`, `pfs`, `prisma`, `rec`, `rex`, `sem`. Cloud providers: `aws`, `azure`, `gcp`. Aggregates: `all`, `lake`.
|
||||
18 modules available as optional extras: `aco`, `api`, `bcda`, `bib`, `bls`, `ccw`, `cli`, `cms`, `conf`, `llm`, `mail`, `opps`, `perf`, `pfs`, `prisma`, `rec`, `rex`, `sem`. Cloud providers: `aws`, `azure`, `gcp`. Aggregates: `all`, `lake`.
|
||||
|
||||
## Services
|
||||
|
||||
@@ -47,6 +47,7 @@ All services route through Traefik at `*.fhirworx.io`.
|
||||
|---------|-----|---------|
|
||||
| API | `api` (internal) | FastAPI — health, pipelines, bib endpoints |
|
||||
| Notebooks | `notebooks.fhirworx.io` | Marimo notebooks (GPU-accelerated) |
|
||||
| LLM Chat | `llm.fhirworx.io` | SSO-guarded RAG chat over CMS comments (grounded, cited) |
|
||||
| Docs | `docs.fhirworx.io` | Docusaurus — API reference + CMS bibliography |
|
||||
| Gitea | `git.fhirworx.io` | Git server, container registry, LFS |
|
||||
| Zotero | `zotero.fhirworx.io` | Reference manager (KasmVNC desktop) |
|
||||
@@ -154,7 +155,7 @@ The generator reads image definitions from `stack.toml [images]`, dispatches to
|
||||
|
||||
### Workflows
|
||||
|
||||
8 workflows generated per backend:
|
||||
10 workflows generated per backend:
|
||||
|
||||
| Workflow | Trigger | What it does |
|
||||
|----------|---------|--------------|
|
||||
@@ -194,7 +195,7 @@ Daily automated pipeline (`pkg-supply-chain.yml`) that:
|
||||
|
||||
## Credential management
|
||||
|
||||
All 16 service credentials derive from a single 256-bit root key via HKDF-SHA256. No passwords stored in `.env` — they regenerate deterministically from root key + commit SHA on each deploy.
|
||||
All 17 service credentials derive from a single 256-bit root key via HKDF-SHA256. No passwords stored in `.env` — they regenerate deterministically from root key + commit SHA on each deploy.
|
||||
|
||||
```
|
||||
ROOT_KEY + commit_sha -> HKDF-SHA256 -> all credentials -> .env + backends
|
||||
@@ -418,7 +419,7 @@ The hook classifies staged files and runs only what is relevant:
|
||||
| `src/sem/*.py` | `tests/sem/` + `test_ast_coverage.py` | Module tests + structural invariants |
|
||||
| `src/aco/*.py` + `src/sem/*.py` | `tests/aco/` + `tests/sem/` + structural | Both module test dirs |
|
||||
| `tests/bib/test_sync.py` | `tests/bib/` | Changed test dir |
|
||||
| `pyproject.toml` or `conftest.py` | Full suite (13,847+ tests) | Infrastructure change |
|
||||
| `pyproject.toml` or `conftest.py` | Full suite (14,010+ tests) | Infrastructure change |
|
||||
| `notebooks/pfs_calcs.py` | `marimo check` + notebook execution | Notebook validation only |
|
||||
| `README.md` only | Nothing | No testable changes |
|
||||
|
||||
@@ -470,7 +471,7 @@ Dynamic values pulled from: `compose.yml` (services), `pyproject.toml` (modules)
|
||||
|
||||
```
|
||||
stack/
|
||||
├── compose.yml Docker Compose (29 services)
|
||||
├── compose.yml Docker Compose (32 services)
|
||||
├── stack.toml Centralised configuration
|
||||
├── pyproject.toml Python project (uv, optional deps per module)
|
||||
├── src/
|
||||
@@ -483,6 +484,7 @@ stack/
|
||||
│ ├── cli/ CLI entry point (typer)
|
||||
│ ├── cms/ CMS public data tables
|
||||
│ ├── conf/ Config loader, storage abstraction, table base
|
||||
│ ├── llm/
|
||||
│ ├── mail/
|
||||
│ ├── opps/ Outpatient Prospective Payment System
|
||||
│ ├── perf/ Pipeline telemetry (OpenTelemetry)
|
||||
@@ -513,7 +515,7 @@ stack/
|
||||
│ ├── hooks/ Git hooks (tracked, core.hooksPath = dev/hooks)
|
||||
│ ├── seeds/ Reference data (BCDA samples, CMS docs)
|
||||
│ └── pipelines/ CI-agnostic pipeline specs
|
||||
├── tests/ 13,847 tests at 99% coverage
|
||||
├── tests/ 14,010 tests at 99% coverage
|
||||
├── notebooks/ Marimo notebooks
|
||||
├── docs/ Docusaurus site
|
||||
├── data/ DuckDB, bib.sqlite, BCDA/CMS data, zotero (gitignored)
|
||||
|
||||
@@ -51,6 +51,7 @@ All services route through Traefik at `*.fhirworx.io`.
|
||||
|---------|-----|---------|
|
||||
| API | `api` (internal) | FastAPI — health, pipelines, bib endpoints |
|
||||
| Notebooks | `notebooks.fhirworx.io` | Marimo notebooks (GPU-accelerated) |
|
||||
| LLM Chat | `llm.fhirworx.io` | SSO-guarded RAG chat over CMS comments (grounded, cited) |
|
||||
| Docs | `docs.fhirworx.io` | Docusaurus — API reference + CMS bibliography |
|
||||
| Gitea | `git.fhirworx.io` | Git server, container registry, LFS |
|
||||
| Zotero | `zotero.fhirworx.io` | Reference manager (KasmVNC desktop) |
|
||||
|
||||
33
compose.yml
33
compose.yml
@@ -663,6 +663,39 @@ services:
|
||||
- no-new-privileges:true
|
||||
restart: unless-stopped
|
||||
|
||||
# SSO-guarded RAG chat UI (P34). Routed at llm.fhirworx.io via Traefik's
|
||||
# git-sso middleware ($reef in infra/traefik/dynamic/services.yml). Joins
|
||||
# gateway (Traefik), storage (postgres:5432 / pgvector), and data
|
||||
# (ollama:11434). Query-only — the batch indexer runs on the host.
|
||||
llm:
|
||||
image: ${IMAGE_PREFIX:-fhirworx}/llm:${COMMIT_SHA:-latest}
|
||||
pull_policy: if_not_present
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/images/llm.Dockerfile
|
||||
container_name: llm
|
||||
networks:
|
||||
- gateway
|
||||
- storage
|
||||
- data
|
||||
- observability
|
||||
environment:
|
||||
- LLM_OLLAMA_HOSTS=http://ollama:11434
|
||||
- LLM_PG_HOST=postgres
|
||||
- LLM_DB_PASSWORD=${LLM_DB_PASSWORD}
|
||||
- OTEL_SERVICE_NAME=llm
|
||||
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
ollama:
|
||||
condition: service_started
|
||||
labels:
|
||||
- "promtail=true"
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
restart: unless-stopped
|
||||
|
||||
# IMAP → bib poller. Pulls UNSEEN mail from `cmsupdates@mail.fhirworx.io`
|
||||
# every MAIL_POLL_INTERVAL seconds and upserts each as a Source item.
|
||||
# Idempotent (server-side `\Seen` flag), so frequency is purely a
|
||||
|
||||
@@ -56,6 +56,7 @@ HOSTS_SUBDOMAINS = [
|
||||
"nessie",
|
||||
"trino",
|
||||
"polaris",
|
||||
"llm",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -274,6 +274,7 @@ SUBDOMAINS = [
|
||||
"s3console",
|
||||
"traefik",
|
||||
"auth",
|
||||
"llm",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -16,3 +16,4 @@
|
||||
192.168.1.192 loki.fhirworx.io
|
||||
192.168.1.192 s3.fhirworx.io
|
||||
192.168.1.192 s3console.fhirworx.io
|
||||
192.168.1.192 llm.fhirworx.io
|
||||
|
||||
32
infra/images/llm.Dockerfile
Normal file
32
infra/images/llm.Dockerfile
Normal file
@@ -0,0 +1,32 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Local package registry (Gitea) — set via --build-arg to pull from mirror
|
||||
ARG PYPI_INDEX_URL=""
|
||||
|
||||
# Patch base image CVEs + install curl for the healthcheck.
|
||||
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY pyproject.toml uv.lock README.md ./
|
||||
COPY src/ src/
|
||||
|
||||
# The chat service needs the `llm` extra (fastapi/uvicorn/langchain/pgvector).
|
||||
ENV UV_PYTHON_PREFERENCE=only-system \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=.venv \
|
||||
UV_INDEX_URL=${PYPI_INDEX_URL}
|
||||
RUN uv sync --no-dev --extra llm && uv pip install -e .
|
||||
|
||||
# Config (conf reads stack.toml by walking up from CWD)
|
||||
COPY stack.toml ./
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
|
||||
CMD curl -sf --max-time 4 http://localhost:8000/health || exit 1
|
||||
|
||||
CMD ["uv", "run", "--no-sync", "uvicorn", "llm.api:app", \
|
||||
"--host", "0.0.0.0", "--port", "8000", \
|
||||
"--workers", "1", "--log-level", "info"]
|
||||
@@ -31,6 +31,17 @@
|
||||
</div>
|
||||
|
||||
<div class="grid">
|
||||
<a data-subdomain="llm" target="_blank" class="tile notebook">
|
||||
<span class="badge gpu">GPU</span>
|
||||
<span class="status"></span>
|
||||
<span class="icon">💬</span>
|
||||
<div class="tile-eyebrow">Chat</div>
|
||||
<div class="tile-title">Comment Chat</div>
|
||||
<p class="tile-desc">RAG chat over indexed CMS rulemaking comments, with cited sources</p>
|
||||
<span class="tile-port"></span>
|
||||
<div class="stripe"></div>
|
||||
</a>
|
||||
|
||||
<a data-subdomain="traefik" target="_blank" class="tile proxy">
|
||||
<span class="status"></span>
|
||||
<span class="icon">🔗</span>
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
"zotero" (dict "port" "8080" "theme" true "mw" "git-sso,secure-headers")
|
||||
"webdav" (dict "port" "8080" "theme" false "mw" "secure-headers")
|
||||
"api" (dict "port" "8000" "theme" false "mw" "secure-headers")
|
||||
"llm" (dict "port" "8000" "theme" true "mw" "git-sso,secure-headers")
|
||||
"nessie" (dict "port" "19120" "theme" false "mw" "git-sso,infra-headers")
|
||||
"trino" (dict "port" "8080" "theme" true "mw" "git-sso,infra-headers")
|
||||
"polaris" (dict "port" "8181" "theme" false "mw" "git-sso,infra-headers")
|
||||
|
||||
@@ -53,6 +53,8 @@ llm = [
|
||||
"stack[conf]",
|
||||
"stack[bib]",
|
||||
"httpx>=0.28.1",
|
||||
"fastapi>=0.139.0",
|
||||
"uvicorn>=0.41.0",
|
||||
"langchain-core>=0.3.0",
|
||||
"langchain-ollama>=0.2.0",
|
||||
"langchain-postgres>=0.0.12",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""stack llm — local RAG over comments + corpus (P33: index only)."""
|
||||
"""stack llm — local RAG over comments + corpus (index + chat serve)."""
|
||||
|
||||
import typer
|
||||
|
||||
@@ -44,3 +44,14 @@ def index(
|
||||
f"indexed={stats['indexed']} skipped={stats['skipped']} "
|
||||
f"chunks={stats['chunks']}"
|
||||
)
|
||||
|
||||
|
||||
@app.command()
|
||||
def serve(
|
||||
host: str = typer.Option("127.0.0.1", help="Bind address."),
|
||||
port: int = typer.Option(8000, help="Port."),
|
||||
) -> None:
|
||||
"""Serve the SSO-guarded chat UI (llm.api:app)."""
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run("llm.api:app", host=host, port=port, log_level="info")
|
||||
|
||||
79
src/llm/api.py
Normal file
79
src/llm/api.py
Normal file
@@ -0,0 +1,79 @@
|
||||
"""FastAPI app for the llm chat UI (P34).
|
||||
|
||||
Served at llm.fhirworx.io behind Traefik's git-sso middleware — every request
|
||||
that reaches this app is already authenticated by Gitea SSO, so the app only
|
||||
reads the forwarded identity header for display, it does not re-authenticate.
|
||||
|
||||
Run: ``uvicorn llm.api:app`` (or ``stack llm serve``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from importlib import resources
|
||||
from typing import Iterator
|
||||
|
||||
from fastapi import FastAPI, Header, HTTPException
|
||||
from fastapi.responses import HTMLResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
app = FastAPI(
|
||||
title="llm chat", description="RAG chat over CMS comments.", version="0.1.0"
|
||||
)
|
||||
|
||||
# OTel instrumentation — no-op if perf not installed or telemetry disabled.
|
||||
try:
|
||||
from perf import init as _perf_init
|
||||
from perf.middleware import instrument as _perf_instrument
|
||||
|
||||
_perf_init()
|
||||
_perf_instrument(app)
|
||||
except ImportError: # pragma: no cover — perf always installed in the image
|
||||
pass
|
||||
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
question: str
|
||||
|
||||
|
||||
def _page() -> str:
|
||||
return resources.files("llm").joinpath("web/chat.html").read_text()
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict:
|
||||
"""Liveness probe for the container healthcheck."""
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index() -> str:
|
||||
return _page()
|
||||
|
||||
|
||||
@app.get("/whoami")
|
||||
def whoami(x_auth_request_user: str = Header(default="")) -> dict:
|
||||
"""The Gitea username Traefik forwarded (empty if unset)."""
|
||||
return {"user": x_auth_request_user}
|
||||
|
||||
|
||||
def _sse(question: str) -> Iterator[str]:
|
||||
from llm import config as llm_config
|
||||
from llm.pool import HostPool
|
||||
from llm.rag import stream_answer
|
||||
|
||||
cfg = llm_config.load()
|
||||
pool = HostPool.from_config(cfg)
|
||||
try:
|
||||
for event in stream_answer(question, cfg=cfg, pool=pool):
|
||||
yield f"data: {json.dumps(event)}\n\n"
|
||||
except Exception as exc: # surface to the transcript, don't 500 mid-stream
|
||||
yield f"data: {json.dumps({'type': 'error', 'message': str(exc)})}\n\n"
|
||||
|
||||
|
||||
@app.post("/chat")
|
||||
def chat(req: ChatRequest) -> StreamingResponse:
|
||||
question = req.question.strip()
|
||||
if not question:
|
||||
raise HTTPException(status_code=400, detail="empty question")
|
||||
return StreamingResponse(_sse(question), media_type="text/event-stream")
|
||||
94
src/llm/rag.py
Normal file
94
src/llm/rag.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""RAG chain for the chat UI: retrieve comments, stream a grounded answer.
|
||||
|
||||
Single-shot and stateless — each question is retrieved and answered on its
|
||||
own. Retrieval hits pgvector's ``comments`` collection via the embedding
|
||||
pool; generation streams the configured instruct model from an Ollama host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Iterator
|
||||
|
||||
import httpx
|
||||
|
||||
from llm.config import LlmConfig
|
||||
from llm.pool import HostPool
|
||||
|
||||
_TIMEOUT = httpx.Timeout(300.0, connect=5.0)
|
||||
|
||||
_SYSTEM = (
|
||||
"You answer questions about public comments submitted to CMS rulemaking "
|
||||
"dockets. Use ONLY the comment excerpts provided below. When you use an "
|
||||
"excerpt, cite its id in square brackets, e.g. [CMS-2017-0092-1306]. If "
|
||||
"the excerpts do not contain the answer, say you don't have information "
|
||||
"on that in the indexed comments — do not invent facts."
|
||||
)
|
||||
|
||||
|
||||
def retrieve(
|
||||
question: str, *, cfg: LlmConfig, pool: HostPool, k: int = 6
|
||||
) -> list[dict]:
|
||||
"""Top-k comment chunks for ``question`` as source dicts."""
|
||||
from llm.index import vectorstore
|
||||
|
||||
store = vectorstore("comments", cfg, pool)
|
||||
hits = store.similarity_search_with_score(question, k=k)
|
||||
sources = []
|
||||
for doc, score in hits:
|
||||
md = doc.metadata or {}
|
||||
sources.append(
|
||||
{
|
||||
"comment_id": md.get("comment_id") or md.get("item_key", ""),
|
||||
"docket": md.get("docket", ""),
|
||||
"snippet": doc.page_content[:500].strip(),
|
||||
"score": round(float(score), 4),
|
||||
}
|
||||
)
|
||||
return sources
|
||||
|
||||
|
||||
def build_messages(question: str, sources: list[dict]) -> list[dict]:
|
||||
"""Grounded chat messages: system rules + question with excerpts."""
|
||||
if sources:
|
||||
context = "\n\n".join(f"[{s['comment_id']}] {s['snippet']}" for s in sources)
|
||||
else:
|
||||
context = "(no relevant comments found)"
|
||||
user = f"Comment excerpts:\n\n{context}\n\nQuestion: {question}"
|
||||
return [
|
||||
{"role": "system", "content": _SYSTEM},
|
||||
{"role": "user", "content": user},
|
||||
]
|
||||
|
||||
|
||||
def stream_answer(question: str, *, cfg: LlmConfig, pool: HostPool) -> Iterator[dict]:
|
||||
"""Retrieve, then stream a grounded answer.
|
||||
|
||||
Yields ``{"type":"token","text":…}`` events as the model generates, then
|
||||
one ``{"type":"sources",…}`` and a final ``{"type":"done"}``.
|
||||
"""
|
||||
sources = retrieve(question, cfg=cfg, pool=pool)
|
||||
pool.check(cfg.instruct_model)
|
||||
messages = build_messages(question, sources)
|
||||
with pool.acquire() as host, httpx.Client(timeout=_TIMEOUT) as client:
|
||||
with client.stream(
|
||||
"POST",
|
||||
f"{host}/api/chat",
|
||||
json={
|
||||
"model": cfg.instruct_model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
},
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
for line in resp.iter_lines():
|
||||
if not line:
|
||||
continue
|
||||
data = json.loads(line)
|
||||
chunk = data.get("message", {}).get("content", "")
|
||||
if chunk:
|
||||
yield {"type": "token", "text": chunk}
|
||||
if data.get("done"):
|
||||
break
|
||||
yield {"type": "sources", "sources": sources}
|
||||
yield {"type": "done"}
|
||||
167
src/llm/web/chat.html
Normal file
167
src/llm/web/chat.html
Normal file
@@ -0,0 +1,167 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Comment Chat — fhirworx</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #0f1115; --panel: #171a21; --line: #262b36; --fg: #e6e9ef;
|
||||
--muted: #8b93a7; --accent: #6ea8fe; --user: #1f6feb22; --bot: #171a21;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0; font: 15px/1.55 system-ui, -apple-system, "Segoe UI", sans-serif;
|
||||
color: var(--fg); background: var(--bg); height: 100vh; display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
header {
|
||||
padding: 12px 18px; border-bottom: 1px solid var(--line);
|
||||
display: flex; align-items: baseline; gap: 12px; flex: 0 0 auto;
|
||||
}
|
||||
header h1 { font-size: 15px; margin: 0; font-weight: 600; }
|
||||
header .who { color: var(--muted); font-size: 13px; margin-left: auto; }
|
||||
#log {
|
||||
flex: 1 1 auto; overflow-y: auto; padding: 20px; display: flex;
|
||||
flex-direction: column; gap: 16px; max-width: 820px; width: 100%;
|
||||
margin: 0 auto;
|
||||
}
|
||||
.msg { display: flex; flex-direction: column; gap: 6px; }
|
||||
.msg .role { font-size: 12px; color: var(--muted); text-transform: uppercase; letter-spacing: .04em; }
|
||||
.bubble { padding: 12px 14px; border-radius: 10px; white-space: pre-wrap; word-wrap: break-word; }
|
||||
.user .bubble { background: var(--user); align-self: flex-end; max-width: 80%; }
|
||||
.bot .bubble { background: var(--bot); border: 1px solid var(--line); }
|
||||
.sources { margin-top: 4px; font-size: 13px; }
|
||||
.sources summary { cursor: pointer; color: var(--muted); }
|
||||
.src { border-left: 2px solid var(--line); padding: 4px 10px; margin: 8px 0; color: var(--muted); }
|
||||
.src b { color: var(--accent); font-weight: 600; }
|
||||
.err { color: #ff7b72; }
|
||||
form {
|
||||
flex: 0 0 auto; border-top: 1px solid var(--line); padding: 14px 18px;
|
||||
display: flex; gap: 10px; max-width: 820px; width: 100%; margin: 0 auto;
|
||||
}
|
||||
textarea {
|
||||
flex: 1; resize: none; height: 44px; padding: 11px 12px; border-radius: 8px;
|
||||
border: 1px solid var(--line); background: var(--panel); color: var(--fg);
|
||||
font: inherit;
|
||||
}
|
||||
button {
|
||||
padding: 0 18px; border: 0; border-radius: 8px; background: var(--accent);
|
||||
color: #06090f; font-weight: 600; cursor: pointer;
|
||||
}
|
||||
button:disabled { opacity: .5; cursor: default; }
|
||||
.hint { color: var(--muted); text-align: center; margin: auto; max-width: 460px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Comment Chat</h1>
|
||||
<span class="who" id="who"></span>
|
||||
</header>
|
||||
<div id="log">
|
||||
<div class="hint">Ask a question about the indexed CMS rulemaking comments.
|
||||
Answers are grounded in the comments and cite their ids.</div>
|
||||
</div>
|
||||
<form id="form">
|
||||
<textarea id="q" placeholder="e.g. What do commenters say about telehealth originating sites?" autofocus></textarea>
|
||||
<button id="send" type="submit">Send</button>
|
||||
</form>
|
||||
|
||||
<script>
|
||||
const log = document.getElementById('log');
|
||||
const form = document.getElementById('form');
|
||||
const box = document.getElementById('q');
|
||||
const send = document.getElementById('send');
|
||||
|
||||
fetch('whoami').then(r => r.json()).then(d => {
|
||||
if (d.user) document.getElementById('who').textContent = 'signed in as ' + d.user;
|
||||
}).catch(() => {});
|
||||
|
||||
function bubble(role) {
|
||||
const wrap = document.createElement('div');
|
||||
wrap.className = 'msg ' + role;
|
||||
const label = document.createElement('div');
|
||||
label.className = 'role';
|
||||
label.textContent = role === 'user' ? 'you' : 'assistant';
|
||||
const b = document.createElement('div');
|
||||
b.className = 'bubble';
|
||||
wrap.appendChild(label); wrap.appendChild(b);
|
||||
log.appendChild(wrap);
|
||||
log.scrollTop = log.scrollHeight;
|
||||
return { wrap, b };
|
||||
}
|
||||
|
||||
function renderSources(wrap, sources) {
|
||||
if (!sources || !sources.length) return;
|
||||
const d = document.createElement('details');
|
||||
d.className = 'sources';
|
||||
const s = document.createElement('summary');
|
||||
s.textContent = sources.length + ' source comment' + (sources.length > 1 ? 's' : '');
|
||||
d.appendChild(s);
|
||||
for (const src of sources) {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'src';
|
||||
const id = document.createElement('b');
|
||||
id.textContent = '[' + src.comment_id + ']';
|
||||
el.appendChild(id);
|
||||
el.appendChild(document.createTextNode(' ' + src.snippet));
|
||||
d.appendChild(el);
|
||||
}
|
||||
wrap.appendChild(d);
|
||||
log.scrollTop = log.scrollHeight;
|
||||
}
|
||||
|
||||
async function ask(question) {
|
||||
const hint = log.querySelector('.hint');
|
||||
if (hint) hint.remove();
|
||||
bubble('user').b.textContent = question;
|
||||
const { wrap, b } = bubble('bot');
|
||||
send.disabled = true;
|
||||
try {
|
||||
const resp = await fetch('chat', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ question })
|
||||
});
|
||||
if (!resp.ok) throw new Error('request failed (' + resp.status + ')');
|
||||
const reader = resp.body.getReader();
|
||||
const dec = new TextDecoder();
|
||||
let buf = '';
|
||||
for (;;) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buf += dec.decode(value, { stream: true });
|
||||
const parts = buf.split('\n\n');
|
||||
buf = parts.pop();
|
||||
for (const part of parts) {
|
||||
const line = part.replace(/^data: /, '').trim();
|
||||
if (!line) continue;
|
||||
const ev = JSON.parse(line);
|
||||
if (ev.type === 'token') { b.textContent += ev.text; log.scrollTop = log.scrollHeight; }
|
||||
else if (ev.type === 'sources') renderSources(wrap, ev.sources);
|
||||
else if (ev.type === 'error') { b.className = 'bubble err'; b.textContent = 'Error: ' + ev.message; }
|
||||
}
|
||||
}
|
||||
if (!b.textContent) b.textContent = '(no answer)';
|
||||
} catch (e) {
|
||||
b.className = 'bubble err';
|
||||
b.textContent = 'Error: ' + e.message;
|
||||
} finally {
|
||||
send.disabled = false;
|
||||
box.focus();
|
||||
}
|
||||
}
|
||||
|
||||
form.addEventListener('submit', e => {
|
||||
e.preventDefault();
|
||||
const q = box.value.trim();
|
||||
if (!q) return;
|
||||
box.value = '';
|
||||
ask(q);
|
||||
});
|
||||
box.addEventListener('keydown', e => {
|
||||
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); form.requestSubmit(); }
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -24,7 +24,7 @@ subdomains = [
|
||||
"dashboard", "docs", "git", "ci", "notebooks", "zotero",
|
||||
"webdav", "api", "nessie", "trino", "polaris",
|
||||
"grafana", "prometheus", "tempo", "loki",
|
||||
"s3", "s3console",
|
||||
"s3", "s3console", "llm",
|
||||
]
|
||||
|
||||
[services]
|
||||
@@ -281,6 +281,11 @@ dockerfile = "infra/images/api.Dockerfile"
|
||||
context = "."
|
||||
path_filter = ["infra/images/api.Dockerfile", "src/**", "pyproject.toml"]
|
||||
|
||||
[images.llm]
|
||||
dockerfile = "infra/images/llm.Dockerfile"
|
||||
context = "."
|
||||
path_filter = ["infra/images/llm.Dockerfile", "src/**", "pyproject.toml"]
|
||||
|
||||
[images.mc]
|
||||
dockerfile = "infra/images/mc.Dockerfile"
|
||||
context = "infra/rustfs/"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Exercise cli/llm.py's `index` command with mocked backends."""
|
||||
"""Exercise cli/llm.py's `index` and `serve` commands with mocked backends."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -32,7 +32,7 @@ class TestIndexComments:
|
||||
mock_from_config.return_value = pool
|
||||
mock_index_docs.return_value = _STATS
|
||||
|
||||
result = runner.invoke(app, [])
|
||||
result = runner.invoke(app, ["index"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
mock_iter.assert_called_once_with(store, docket="")
|
||||
@@ -62,7 +62,7 @@ class TestIndexCorpus:
|
||||
mock_from_config.return_value = MagicMock()
|
||||
mock_index_docs.return_value = _STATS
|
||||
|
||||
result = runner.invoke(app, ["--collection", "corpus"])
|
||||
result = runner.invoke(app, ["index", "--collection", "corpus"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
mock_iter.assert_called_once_with(store)
|
||||
@@ -78,7 +78,7 @@ class TestIndexBadCollection:
|
||||
mock_load.return_value = MagicMock()
|
||||
mock_bib.return_value = MagicMock()
|
||||
|
||||
result = runner.invoke(app, ["--collection", "bogus"])
|
||||
result = runner.invoke(app, ["index", "--collection", "bogus"])
|
||||
|
||||
assert result.exit_code == 2
|
||||
assert "collection must be 'comments' or 'corpus'" in result.output
|
||||
@@ -99,8 +99,18 @@ class TestIndexLimit:
|
||||
mock_from_config.return_value = MagicMock()
|
||||
mock_index_docs.return_value = _STATS
|
||||
|
||||
result = runner.invoke(app, ["--limit", "2"])
|
||||
result = runner.invoke(app, ["index", "--limit", "2"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
docs_arg = mock_index_docs.call_args.args[0]
|
||||
assert list(docs_arg) == ["doc0", "doc1"]
|
||||
|
||||
|
||||
class TestServe:
|
||||
@patch("uvicorn.run")
|
||||
def test_serve_starts_uvicorn(self, mock_run):
|
||||
result = runner.invoke(app, ["serve", "--host", "0.0.0.0", "--port", "9000"])
|
||||
assert result.exit_code == 0
|
||||
mock_run.assert_called_once_with(
|
||||
"llm.api:app", host="0.0.0.0", port=9000, log_level="info"
|
||||
)
|
||||
|
||||
65
tests/llm/test_api.py
Normal file
65
tests/llm/test_api.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""llm.api — FastAPI chat app."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from llm.api import app
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
class TestHealth:
|
||||
def test_ok(self):
|
||||
r = client.get("/health")
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"status": "ok"}
|
||||
|
||||
|
||||
class TestIndex:
|
||||
def test_serves_chat_page(self):
|
||||
r = client.get("/")
|
||||
assert r.status_code == 200
|
||||
assert "text/html" in r.headers["content-type"]
|
||||
assert "Comment Chat" in r.text
|
||||
assert 'id="form"' in r.text
|
||||
|
||||
|
||||
class TestWhoami:
|
||||
def test_reads_forwarded_header(self):
|
||||
r = client.get("/whoami", headers={"X-Auth-Request-User": "kert"})
|
||||
assert r.json() == {"user": "kert"}
|
||||
|
||||
def test_empty_when_absent(self):
|
||||
r = client.get("/whoami")
|
||||
assert r.json() == {"user": ""}
|
||||
|
||||
|
||||
class TestChat:
|
||||
def test_empty_question_400(self):
|
||||
r = client.post("/chat", json={"question": " "})
|
||||
assert r.status_code == 400
|
||||
|
||||
@patch("llm.rag.stream_answer")
|
||||
def test_streams_sse_events(self, mock_stream):
|
||||
mock_stream.return_value = iter(
|
||||
[
|
||||
{"type": "token", "text": "Hi"},
|
||||
{"type": "sources", "sources": []},
|
||||
{"type": "done"},
|
||||
]
|
||||
)
|
||||
r = client.post("/chat", json={"question": "hello"})
|
||||
assert r.status_code == 200
|
||||
assert "text/event-stream" in r.headers["content-type"]
|
||||
body = r.text
|
||||
assert 'data: {"type": "token", "text": "Hi"}' in body
|
||||
assert '"type": "done"' in body
|
||||
|
||||
@patch("llm.rag.stream_answer")
|
||||
def test_error_becomes_event_not_500(self, mock_stream):
|
||||
mock_stream.side_effect = RuntimeError("ollama down")
|
||||
r = client.post("/chat", json={"question": "hello"})
|
||||
assert r.status_code == 200
|
||||
assert '"type": "error"' in r.text
|
||||
assert "ollama down" in r.text
|
||||
125
tests/llm/test_rag.py
Normal file
125
tests/llm/test_rag.py
Normal file
@@ -0,0 +1,125 @@
|
||||
"""llm.rag — retrieval + grounded streaming answer."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
from llm.config import LlmConfig
|
||||
from llm.rag import build_messages, retrieve, stream_answer
|
||||
|
||||
CFG = LlmConfig(
|
||||
ollama_hosts=("http://h1:11434",),
|
||||
embed_model="embed",
|
||||
instruct_model="chat",
|
||||
embed_dim=768,
|
||||
pg_host="x",
|
||||
pg_port=5432,
|
||||
pg_db="llm",
|
||||
pg_user="llm",
|
||||
build_ann_index=True,
|
||||
)
|
||||
|
||||
|
||||
def _doc(text, **md):
|
||||
return Document(page_content=text, metadata=md)
|
||||
|
||||
|
||||
class TestRetrieve:
|
||||
@patch("llm.index.vectorstore")
|
||||
def test_maps_hits_to_sources(self, mock_vs):
|
||||
store = mock_vs.return_value
|
||||
store.similarity_search_with_score.return_value = [
|
||||
(
|
||||
_doc(
|
||||
"Telehealth comment.",
|
||||
comment_id="CMS-2017-0092-1306",
|
||||
docket="CMS-2017-0092",
|
||||
item_key="K1",
|
||||
),
|
||||
0.21,
|
||||
),
|
||||
]
|
||||
pool = MagicMock()
|
||||
out = retrieve("telehealth", cfg=CFG, pool=pool, k=3)
|
||||
assert out == [
|
||||
{
|
||||
"comment_id": "CMS-2017-0092-1306",
|
||||
"docket": "CMS-2017-0092",
|
||||
"snippet": "Telehealth comment.",
|
||||
"score": 0.21,
|
||||
}
|
||||
]
|
||||
store.similarity_search_with_score.assert_called_once_with("telehealth", k=3)
|
||||
|
||||
@patch("llm.index.vectorstore")
|
||||
def test_falls_back_to_item_key_when_no_comment_id(self, mock_vs):
|
||||
store = mock_vs.return_value
|
||||
store.similarity_search_with_score.return_value = [
|
||||
(_doc("x", item_key="K9", docket="D"), 0.5)
|
||||
]
|
||||
out = retrieve("q", cfg=CFG, pool=MagicMock())
|
||||
assert out[0]["comment_id"] == "K9"
|
||||
|
||||
|
||||
class TestBuildMessages:
|
||||
def test_includes_ids_and_abstention_rule(self):
|
||||
sources = [
|
||||
{
|
||||
"comment_id": "CMS-2017-0092-1",
|
||||
"docket": "D",
|
||||
"snippet": "reduce documentation",
|
||||
"score": 0.1,
|
||||
}
|
||||
]
|
||||
msgs = build_messages("why?", sources)
|
||||
assert msgs[0]["role"] == "system"
|
||||
assert "only" in msgs[0]["content"].lower()
|
||||
assert "don't have information" in msgs[0]["content"].lower()
|
||||
assert "[CMS-2017-0092-1]" in msgs[1]["content"]
|
||||
assert "why?" in msgs[1]["content"]
|
||||
|
||||
def test_no_sources_marks_empty_context(self):
|
||||
msgs = build_messages("q", [])
|
||||
assert "no relevant comments" in msgs[1]["content"].lower()
|
||||
|
||||
|
||||
class TestStreamAnswer:
|
||||
@patch("llm.rag.httpx.Client")
|
||||
@patch("llm.rag.retrieve")
|
||||
def test_yields_tokens_then_sources_then_done(self, mock_retrieve, MockClient):
|
||||
mock_retrieve.return_value = [
|
||||
{"comment_id": "C1", "docket": "D", "snippet": "s", "score": 0.1}
|
||||
]
|
||||
lines = [
|
||||
'{"message":{"content":"Doc"},"done":false}',
|
||||
"", # keep-alive blank line — must be skipped, not parsed
|
||||
'{"message":{"content":"tors"},"done":false}',
|
||||
'{"message":{"content":""},"done":true}',
|
||||
]
|
||||
stream_cm = MockClient.return_value.__enter__.return_value.stream.return_value
|
||||
resp = stream_cm.__enter__.return_value
|
||||
resp.iter_lines.return_value = iter(lines)
|
||||
pool = MagicMock()
|
||||
|
||||
events = list(stream_answer("q", cfg=CFG, pool=pool))
|
||||
|
||||
pool.check.assert_called_once_with("chat")
|
||||
assert events[0] == {"type": "token", "text": "Doc"}
|
||||
assert events[1] == {"type": "token", "text": "tors"}
|
||||
assert events[-2] == {
|
||||
"type": "sources",
|
||||
"sources": [
|
||||
{"comment_id": "C1", "docket": "D", "snippet": "s", "score": 0.1}
|
||||
],
|
||||
}
|
||||
assert events[-1] == {"type": "done"}
|
||||
|
||||
@patch("llm.rag.httpx.Client")
|
||||
@patch("llm.rag.retrieve")
|
||||
def test_http_error_propagates(self, mock_retrieve, MockClient):
|
||||
mock_retrieve.return_value = []
|
||||
stream_cm = MockClient.return_value.__enter__.return_value.stream.return_value
|
||||
resp = stream_cm.__enter__.return_value
|
||||
resp.raise_for_status.side_effect = RuntimeError("ollama down")
|
||||
with __import__("pytest").raises(RuntimeError, match="ollama down"):
|
||||
list(stream_answer("q", cfg=CFG, pool=MagicMock()))
|
||||
4
uv.lock
generated
4
uv.lock
generated
@@ -4026,6 +4026,7 @@ lake = [
|
||||
]
|
||||
llm = [
|
||||
{ name = "duckdb" },
|
||||
{ name = "fastapi" },
|
||||
{ name = "httpx" },
|
||||
{ name = "langchain-core" },
|
||||
{ name = "langchain-ollama" },
|
||||
@@ -4035,6 +4036,7 @@ llm = [
|
||||
{ name = "psycopg", extra = ["binary"] },
|
||||
{ name = "pydantic" },
|
||||
{ name = "sqlalchemy" },
|
||||
{ name = "uvicorn" },
|
||||
]
|
||||
mail = [
|
||||
{ name = "httpx" },
|
||||
@@ -4140,6 +4142,7 @@ requires-dist = [
|
||||
{ name = "duckdb", marker = "extra == 'pfs'", specifier = ">=1.0.0" },
|
||||
{ name = "duckdb", marker = "extra == 'rec'", specifier = ">=1.0.0" },
|
||||
{ name = "fastapi", marker = "extra == 'api'", specifier = ">=0.139.0" },
|
||||
{ name = "fastapi", marker = "extra == 'llm'", specifier = ">=0.139.0" },
|
||||
{ name = "fsspec", marker = "extra == 'bcda'", specifier = ">=2024.1.0" },
|
||||
{ name = "fsspec", marker = "extra == 'rex'", specifier = ">=2024.1.0" },
|
||||
{ name = "gcsfs", marker = "extra == 'gcp'", specifier = ">=2024.1.0" },
|
||||
@@ -4230,6 +4233,7 @@ requires-dist = [
|
||||
{ name = "typer", marker = "extra == 'rec'", specifier = ">=0.24.1" },
|
||||
{ name = "uvicorn", marker = "extra == 'api'", specifier = ">=0.41.0" },
|
||||
{ name = "uvicorn", marker = "extra == 'cli'", specifier = ">=0.41.0" },
|
||||
{ name = "uvicorn", marker = "extra == 'llm'", specifier = ">=0.41.0" },
|
||||
{ name = "xlrd", specifier = ">=2.0.2" },
|
||||
]
|
||||
provides-extras = ["conf", "aco", "api", "bcda", "bib", "bls", "llm", "ccw", "cli", "mail", "cms", "opps", "pfs", "rec", "rex", "prisma", "perf", "sem", "lake", "aws", "gcp", "azure", "all"]
|
||||
|
||||
Reference in New Issue
Block a user