homelab/stack
1
0
Fork 0
Code Issues 262 Pull Requests 2 Actions Packages Projects 1 Releases Wiki Activity

chore(llm): qwen2.5 model tiers, recency + retrieval knobs (refs #654)

Browse Source
This commit is contained in:
kert
2026-09-03 10:50:13 -04:00
parent af595366dd
commit 3218eb7845
2 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/superpowers/specs/2026-07-16-llm-module-design.md
View File

@@ -1,6 +1,6 @@
# llm module — local RAG over rulemaking comments + Zotero corpus # llm module — local RAG over rulemaking comments + Zotero corpus
**Status:** Designed — awaiting implementation (milestones P33–P35) **Status:** Designed — awaiting implementation (milestones P33–P35). Superseded in part by `2026-09-03-llm-corpus-recency-links-gpu-design.md` (retrieval scope, ranking, evidence links, GPU routing).
**Date:** 2026-07-16 **Date:** 2026-07-16
**Related:** #253 (extraction, closed), #254/#255/#256 (P30, open — consumers of **Related:** #253 (extraction, closed), #254/#255/#256 (P30, open — consumers of
this module), `.claude/specs/2026-04-23-comments-extraction-design.md` this module), `.claude/specs/2026-04-23-comments-extraction-design.md`

14
stack.toml
View File

@@ -104,7 +104,10 @@ timeout = 120.0
[llm] [llm]
ollama = "http://127.0.0.1:11434" # host-side default; containers override via LLM_OLLAMA_HOSTS ollama = "http://127.0.0.1:11434" # host-side default; containers override via LLM_OLLAMA_HOSTS
embed_model = "nomic-embed-text" # 768-dim; bake-off (P33 #564) may revise embed_model = "nomic-embed-text" # 768-dim; bake-off (P33 #564) may revise
instruct_model = "llama3.1:8b" # bake-off may revise instruct_model = "qwen2.5:14b" # baseline, pulled on every fleet host
instruct_model_large = "qwen2.5:32b" # used when the chosen host declares >= large_min_vram_gb
large_min_vram_gb = 20
chat_num_ctx = 8192 # passed per request; no Modelfile ctx variants
embed_dim = 768 embed_dim = 768
# HNSW index build. Safe since the postgres image ships an AVX-512-free # HNSW index build. Safe since the postgres image ships an AVX-512-free
# pgvector rebuild (infra/images/postgresql.Dockerfile, #580); before that it # pgvector rebuild (infra/images/postgresql.Dockerfile, #580); before that it
@@ -114,6 +117,15 @@ pg_host = "127.0.0.1" # host-side default; containers set LLM_P
pg_port = 5432 pg_port = 5432
pg_db = "llm" pg_db = "llm"
pg_user = "llm" pg_user = "llm"
# Retrieval: similarity × recency blend (half-life in days), sources kept
recency_half_life_days = 365
recency_weight = 0.3
top_n = 8
[llm.k_per_kind] # over-fetched ×3 per kind, then re-ranked
comment = 8
rule = 4
corpus = 4
[s3] [s3]
endpoint = "http://rustfs:9000" endpoint = "http://rustfs:9000"