35dc05c4d0
A UMLS licence key (UMLS_API_KEY, read via conf.secret("nlm.api_key"))
now drives a new `nlm` module and `stack nlm` CLI:
- `stack nlm releases` lists the current file of every UTS release
stream (14 today: RxNorm full/prescribable/weekly + RxNav-in-a-Box,
SNOMED CT US/International/Spanish/CORE/transitive closure/ICD-10-CM
map, UMLS full/Metathesaurus/MRCONSO).
- `stack nlm fetch [--stream …]` downloads them through the UTS
/download endpoint to `<nlm.dir>/<stream>/<version>/<file>`, with a
sidecar (what, where from, bytes, sha256, when). A finished file is
skipped; an interrupted one resumes from its .part with a Range
request; a non-zip answer (a login page) is refused.
- `stack nlm vsac` saves every VSAC value-set definition (18.6k
versions, paged FHIR search) as gzipped JSON plus index.jsonl, then
$expands each one — large expansions are paged and joined — with
backoff on 429/5xx, 4 workers, resumable, failures to errors.jsonl.
The key rides as a query parameter (UTS) or basic auth (VSAC);
nlm.scrub() strips it from every error before it can reach a log.
nlm.dir defaults to the gitignored data/nlm: the content is
UMLS-licensed and must never be published.
Live-checked: listing, an authenticated download with sidecar, and a
768-code value set expanded in pages of 25 matching the one-shot
expansion exactly. Tests mock httpx (100% of the new module).
329 lines
11 KiB
TOML
329 lines
11 KiB
TOML
# stack.toml — centralised configuration for the stack platform.
|
||
#
|
||
# Paths are relative to the repository root unless absolute.
|
||
# Secrets live in .env (never here).
|
||
#
|
||
# Context switching
|
||
# -----------------
|
||
# [context.active] selects the environment profile. Override at
|
||
# runtime with STACK_CONTEXT=lake (env var beats this file).
|
||
#
|
||
# Each context sets db_backend and storage_backend. Modules call
|
||
# conf.context() to decide *which* engine or filesystem to use;
|
||
# the [db], [storage], and [lake] sections supply the concrete
|
||
# connection details for the selected backend.
|
||
|
||
[platform]
|
||
domain = "fhirworx.io"
|
||
registry = "git.fhirworx.io"
|
||
image_prefix = "fhirworx"
|
||
repo = "homelab/stack"
|
||
ci_email = "ci@fhirworx.io"
|
||
host_ip = "192.168.1.192"
|
||
subdomains = [
|
||
"dashboard", "docs", "git", "ci", "notebooks", "zotero",
|
||
"webdav", "api", "nessie", "trino", "polaris",
|
||
"grafana", "prometheus", "tempo", "loki",
|
||
"s3", "s3console", "llm",
|
||
]
|
||
|
||
[services]
|
||
git = "http://git:3000"
|
||
rustfs = "http://rustfs:9000"
|
||
nessie = "http://nessie:19120"
|
||
polaris = "http://polaris:8181"
|
||
trino = "http://trino:8080"
|
||
grafana = "http://grafana:3000"
|
||
ollama = "http://ollama:11434"
|
||
|
||
[context]
|
||
active = "local"
|
||
|
||
[context.local]
|
||
db_backend = "duckdb"
|
||
storage_backend = "local"
|
||
|
||
[context.lake]
|
||
db_backend = "iceberg"
|
||
storage_backend = "s3"
|
||
catalog = "nessie"
|
||
|
||
[context.databricks]
|
||
db_backend = "databricks"
|
||
storage_backend = "dbfs"
|
||
|
||
[context.trino]
|
||
db_backend = "trino"
|
||
storage_backend = "s3"
|
||
|
||
[context.aws]
|
||
db_backend = "iceberg"
|
||
storage_backend = "s3"
|
||
catalog = "glue"
|
||
compute = "ecs"
|
||
|
||
[context.gcp]
|
||
db_backend = "iceberg"
|
||
storage_backend = "gcs"
|
||
catalog = "bigquery"
|
||
compute = "cloud-run"
|
||
|
||
[context.azure]
|
||
db_backend = "iceberg"
|
||
storage_backend = "abfs"
|
||
catalog = "unity"
|
||
compute = "container-apps"
|
||
|
||
[db]
|
||
aco = "data/aco.duckdb"
|
||
bib = "data/bib.sqlite"
|
||
zotero = "data/zotero/data/zotero.sqlite"
|
||
|
||
[comments]
|
||
seal_quiet_days = 30 # days after a docket's comment close date before an empty pull seals it
|
||
|
||
[storage]
|
||
bib = "data/bib/storage"
|
||
zotero = "data/zotero/data/storage"
|
||
bcda = "data/bcda"
|
||
rex = "data/rex"
|
||
pfs = "data/pfs"
|
||
rec = "data/rec"
|
||
cms_log = "data/cms/log.jsonl"
|
||
bcda_log = "data/bcda/log.jsonl"
|
||
|
||
[generate]
|
||
table_out = "src/aco/table"
|
||
base_import = "aco.table.base"
|
||
|
||
[bcda]
|
||
sandbox = "https://sandbox.bcda.cms.gov"
|
||
production = "https://api.bcda.cms.gov"
|
||
max_retries = 3
|
||
retry_interval = 1.0
|
||
token_lifetime = 1200
|
||
timeout = 120.0
|
||
|
||
[llm]
|
||
ollama = "http://127.0.0.1:11434" # host-side default; containers override via LLM_OLLAMA_HOSTS
|
||
embed_model = "nomic-embed-text" # 768-dim; bake-off (P33 #564) may revise
|
||
instruct_model = "qwen2.5:14b" # baseline, pulled on every fleet host
|
||
instruct_model_large = "qwen2.5:32b" # used when the chosen host declares >= large_min_vram_gb
|
||
large_min_vram_gb = 20
|
||
chat_num_ctx = 8192 # passed per request; no Modelfile ctx variants
|
||
chat_temperature = 0.2 # grounded answers cite labels verbatim; low temperature keeps them literal
|
||
chat_max_tokens = 1200 # num_predict: a history answer ran to 5,000 tokens (3 min) without a cap
|
||
chat_chars_per_token = 2.4 # measured 2.5–2.9 on FR/regulatory text; sizes the excerpt budget so prompt + reply fit chat_num_ctx (overflow is caught and rebuilt)
|
||
embed_dim = 768
|
||
# HNSW index build. Safe since the postgres image ships an AVX-512-free
|
||
# pgvector rebuild (infra/images/postgresql.Dockerfile, #580); before that it
|
||
# SIGILLed this Zen2 host. Set false to fall back to exact search.
|
||
build_ann_index = true
|
||
pg_host = "127.0.0.1" # host-side default; containers set LLM_PG_HOST=postgres
|
||
pg_port = 5432
|
||
pg_db = "llm"
|
||
pg_user = "llm"
|
||
# Retrieval: similarity × recency blend (half-life in days), sources kept
|
||
recency_half_life_days = 365
|
||
recency_weight = 0.3
|
||
top_n = 8
|
||
timeline_per_era = 2 # era_balance: hits kept per rule year on the first pass (mode="timeline")
|
||
timeline_overfetch = 6 # era_balance: k multiplier per kind for timeline mode (vs the recent-mode ×3)
|
||
duckdb_replica = "data/replica/aco.ro.duckdb" # read-only DuckDB replica the chat reads valuations from
|
||
valuation_years = 4 # final-rule vintages shown per code (plus the newest NPRM)
|
||
code_cited_per_code = 3 # excerpts literally citing each detected code (or family)
|
||
code_cited_collections = ["rules", "comments", "corpus"] # searched in this order
|
||
code_cited_max = 12 # cited excerpts kept after round-robin interleave across collections; <= 0 = unlimited
|
||
lineage_max_rows = 25 # collapsed lineage events kept in the prompt block (the SSE payload always carries every collapsed row)
|
||
lineage_on_demand_max = 3 # detected codes per turn allowed to fall back to pfs.lineage.lineage() when pfs.code_event has no rows for them
|
||
lineage_sources_max = 8 # FR paragraphs fetched as sources for events selected into the lineage prompt block, outside code_cited_max (Ruling B10: + up to 2 element-diff anchors, hard cap 10 total)
|
||
chat_codes_max = 24 # Ruling B11: per-turn cap on detected codes (explicit codes first, then family order) — applied before valuation/lineage/code-cited queries
|
||
valuation_rows_max = 24 # Ruling B11: valuation rows kept in the prompt block (explicit codes first, then newest vintage); the SSE payload always carries every row
|
||
|
||
[llm.k_per_kind] # over-fetched ×3 per kind, then re-ranked
|
||
comment = 8
|
||
rule = 4
|
||
corpus = 4
|
||
|
||
[s3]
|
||
endpoint = "http://rustfs:9000"
|
||
region = "us-east-1"
|
||
|
||
[lake]
|
||
warehouse = "s3://lakehouse/"
|
||
|
||
[lake.nessie]
|
||
catalog_uri = "http://nessie:19120/iceberg/"
|
||
|
||
[lake.polaris]
|
||
catalog_uri = "http://polaris:8181/api/catalog"
|
||
|
||
[lake.databricks]
|
||
catalog_uri = "" # set via DATABRICKS_HOST env var
|
||
warehouse = "main" # Unity Catalog name
|
||
catalog = "aco" # default catalog for table references
|
||
|
||
[lake.trino]
|
||
host = "trino"
|
||
port = 8080
|
||
catalog = "iceberg"
|
||
|
||
# DuckLake — the concurrent reference-data store chosen in the M3 spike
|
||
# (docs/superpowers/specs/2026-07-10-ducklake-vs-iceberg-decision.md).
|
||
# The postgres catalog serializes concurrent writers; data is Parquet on
|
||
# RustFS. Password comes from the POSTGRES_PASSWORD env var; the catalog
|
||
# host is compose-internal, so lake writers run inside a data-network
|
||
# container.
|
||
[lake.ducklake]
|
||
catalog = "postgres:dbname=ducklake host=postgres user=postgres"
|
||
# Read-only role for notebook readers (password: DUCKLAKE_RO_PASSWORD).
|
||
catalog_ro = "postgres:dbname=ducklake host=postgres user=ducklake_ro"
|
||
data_path = "s3://lakehouse/ducklake/"
|
||
s3_endpoint = "rustfs:9000"
|
||
|
||
# ── Databricks Asset Bundle generation ────────────────────────────
|
||
# gen_config.py reads this to produce databricks.yml.
|
||
# Add a new pipeline module → commit → databricks.yml updates automatically.
|
||
|
||
[databricks]
|
||
bundle_name = "stack"
|
||
package_name = "stack"
|
||
entry_point = "cli"
|
||
schedule = "0 0 6 * * ?"
|
||
timezone = "America/New_York"
|
||
|
||
[databricks.clusters.default]
|
||
spark_version = "15.4.x-scala2.12"
|
||
node_type_id = "i3.xlarge"
|
||
num_workers = 1
|
||
|
||
[databricks.targets.dev]
|
||
mode = "development"
|
||
default = true
|
||
catalog = "aco_dev"
|
||
|
||
[databricks.targets.staging]
|
||
catalog = "aco_staging"
|
||
|
||
[databricks.targets.prod]
|
||
mode = "production"
|
||
catalog = "aco"
|
||
|
||
[databricks.warehouse]
|
||
name = "stack-sql-warehouse"
|
||
size = "SMALL"
|
||
auto_stop_mins = 10
|
||
|
||
[databricks.secrets]
|
||
scope = "stack"
|
||
mapping = [
|
||
{ env = "BCDA_CLIENT_ID", key = "bcda-client-id" },
|
||
{ env = "BCDA_CLIENT_SECRET", key = "bcda-client-secret" },
|
||
]
|
||
|
||
[databricks.governance]
|
||
phi_schemas = ["core", "claims_preprocessing", "cclf", "input_layer"]
|
||
|
||
[databricks.governance.groups.analysts]
|
||
schemas = ["readmissions", "quality_measures", "cms_quality_measures", "ahrq_measures"]
|
||
privileges = ["SELECT"]
|
||
|
||
[databricks.governance.groups.pipeline_svc]
|
||
schemas = ["*"]
|
||
privileges = ["ALL_PRIVILEGES"]
|
||
|
||
[databricks.quality]
|
||
monitored_schemas = ["core", "claims_preprocessing", "data_quality"]
|
||
output_schema = "_monitoring"
|
||
|
||
[default]
|
||
target = "local" # duckdb | databricks | trino | lake
|
||
|
||
[nlm]
|
||
# UMLS / SNOMED CT / RxNorm releases and VSAC value sets. UMLS-licensed:
|
||
# never publish (docs site, package registry, Zotero, commits).
|
||
dir = "data/nlm"
|
||
api_key = "" # override via UMLS_API_KEY env var
|
||
|
||
[api]
|
||
host = "0.0.0.0"
|
||
port = 8000
|
||
secret = "" # override via STACK_API_SECRET env var
|
||
workers = 1
|
||
|
||
[ci]
|
||
backend = "gitea"
|
||
coverage_threshold = 99
|
||
|
||
[ci.github]
|
||
runner = "ubuntu-latest"
|
||
registry = "ghcr.io"
|
||
uv_version = "latest"
|
||
|
||
[ci.gitea]
|
||
runner_labels = ["ubuntu-latest"]
|
||
registry_secret = "REGISTRY_TOKEN"
|
||
uv_version = "latest"
|
||
ci_registry = "git:3000" # internal HTTP endpoint (avoids self-signed TLS)
|
||
|
||
[telemetry]
|
||
enabled = false
|
||
endpoint = "http://otel-collector:4317"
|
||
fallback = "traces/spans.jsonl"
|
||
sample_rate = 1.0
|
||
service_name = "stack"
|
||
|
||
[lint]
|
||
line_length = 88
|
||
|
||
# ── Container images ──────────────────────────────────────────────
|
||
# Single source of truth for all built images.
|
||
# gen_config.py reads this to generate pipeline YAML, infra-ci
|
||
# hadolint steps, trivy scans, S3 uploads, and vuln report loops.
|
||
# Add a new image here → commit → all pipelines update automatically.
|
||
|
||
[images.defaults]
|
||
scan = true
|
||
hadolint = true
|
||
trivy_severity = "HIGH,CRITICAL"
|
||
trivy_exit_code = 0
|
||
|
||
# Repo-root context: the Dockerfile COPYs infra/marimo/theme, which only
|
||
# exists at repo root (see .dockerignore carve-outs). Building from notebooks/
|
||
# fails with `"/infra/marimo/theme": not found`.
|
||
[images.notebooks]
|
||
dockerfile = "infra/images/notebooks.Dockerfile"
|
||
context = "."
|
||
path_filter = ["notebooks/**", "infra/marimo/**", "infra/images/notebooks.Dockerfile"]
|
||
|
||
# Repo-root context: the Dockerfile COPYs the desktop-bridge watchdog +
|
||
# healthcheck from infra/images/zotero/ (carved out in .dockerignore).
|
||
[images.zotero]
|
||
dockerfile = "infra/images/zotero.Dockerfile"
|
||
context = "."
|
||
path_filter = ["infra/images/zotero.Dockerfile", "infra/images/zotero/**"]
|
||
|
||
[images.docs]
|
||
dockerfile = "infra/images/docs.Dockerfile"
|
||
context = "."
|
||
path_filter = ["docs/**", "infra/images/docs.Dockerfile"]
|
||
prep_commands = [
|
||
"mkdir -p data",
|
||
"cp /host-data/bib.sqlite data/ 2>/dev/null || true",
|
||
]
|
||
|
||
[images.api]
|
||
dockerfile = "infra/images/api.Dockerfile"
|
||
context = "."
|
||
path_filter = ["infra/images/api.Dockerfile", "src/**", "pyproject.toml"]
|
||
|
||
[images.llm]
|
||
dockerfile = "infra/images/llm.Dockerfile"
|
||
context = "."
|
||
path_filter = ["infra/images/llm.Dockerfile", "src/**", "pyproject.toml"]
|
||
|
||
[images.mc]
|
||
dockerfile = "infra/images/mc.Dockerfile"
|
||
context = "infra/rustfs/"
|
||
path_filter = ["infra/rustfs/**", "infra/images/mc.Dockerfile"]
|