319 lines
9.6 KiB
TOML
319 lines
9.6 KiB
TOML
# stack.toml — centralised configuration for the stack platform.
|
||
#
|
||
# Paths are relative to the repository root unless absolute.
|
||
# Secrets live in .env (never here).
|
||
#
|
||
# Context switching
|
||
# -----------------
|
||
# [context.active] selects the environment profile. Override at
|
||
# runtime with STACK_CONTEXT=lake (env var beats this file).
|
||
#
|
||
# Each context sets db_backend and storage_backend. Modules call
|
||
# conf.context() to decide *which* engine or filesystem to use;
|
||
# the [db], [storage], and [lake] sections supply the concrete
|
||
# connection details for the selected backend.
|
||
|
||
[platform]
|
||
domain = "fhirworx.io"
|
||
registry = "git.fhirworx.io"
|
||
image_prefix = "fhirworx"
|
||
repo = "homelab/stack"
|
||
ci_email = "ci@fhirworx.io"
|
||
host_ip = "192.168.1.192"
|
||
subdomains = [
|
||
"dashboard", "docs", "git", "ci", "notebooks", "zotero",
|
||
"webdav", "api", "nessie", "trino", "polaris",
|
||
"grafana", "prometheus", "tempo", "loki",
|
||
"s3", "s3console", "llm",
|
||
]
|
||
|
||
[services]
|
||
git = "http://git:3000"
|
||
rustfs = "http://rustfs:9000"
|
||
nessie = "http://nessie:19120"
|
||
polaris = "http://polaris:8181"
|
||
trino = "http://trino:8080"
|
||
grafana = "http://grafana:3000"
|
||
ollama = "http://ollama:11434"
|
||
|
||
[context]
|
||
active = "local"
|
||
|
||
[context.local]
|
||
db_backend = "duckdb"
|
||
storage_backend = "local"
|
||
|
||
[context.lake]
|
||
db_backend = "iceberg"
|
||
storage_backend = "s3"
|
||
catalog = "nessie"
|
||
|
||
[context.databricks]
|
||
db_backend = "databricks"
|
||
storage_backend = "dbfs"
|
||
|
||
[context.trino]
|
||
db_backend = "trino"
|
||
storage_backend = "s3"
|
||
|
||
[context.aws]
|
||
db_backend = "iceberg"
|
||
storage_backend = "s3"
|
||
catalog = "glue"
|
||
compute = "ecs"
|
||
|
||
[context.gcp]
|
||
db_backend = "iceberg"
|
||
storage_backend = "gcs"
|
||
catalog = "bigquery"
|
||
compute = "cloud-run"
|
||
|
||
[context.azure]
|
||
db_backend = "iceberg"
|
||
storage_backend = "abfs"
|
||
catalog = "unity"
|
||
compute = "container-apps"
|
||
|
||
[db]
|
||
aco = "data/aco.duckdb"
|
||
bib = "data/bib.sqlite"
|
||
zotero = "data/zotero/data/zotero.sqlite"
|
||
|
||
[comments]
|
||
seal_quiet_days = 30 # days after a docket's comment close date before an empty pull seals it
|
||
|
||
[storage]
|
||
bib = "data/bib/storage"
|
||
zotero = "data/zotero/data/storage"
|
||
bcda = "data/bcda"
|
||
rex = "data/rex"
|
||
pfs = "data/pfs"
|
||
rec = "data/rec"
|
||
cms_log = "data/cms/log.jsonl"
|
||
bcda_log = "data/bcda/log.jsonl"
|
||
|
||
[generate]
|
||
table_out = "src/aco/table"
|
||
base_import = "aco.table.base"
|
||
|
||
[bcda]
|
||
sandbox = "https://sandbox.bcda.cms.gov"
|
||
production = "https://api.bcda.cms.gov"
|
||
max_retries = 3
|
||
retry_interval = 1.0
|
||
token_lifetime = 1200
|
||
timeout = 120.0
|
||
|
||
[llm]
|
||
ollama = "http://127.0.0.1:11434" # host-side default; containers override via LLM_OLLAMA_HOSTS
|
||
embed_model = "nomic-embed-text" # 768-dim; bake-off (P33 #564) may revise
|
||
instruct_model = "qwen2.5:14b" # baseline, pulled on every fleet host
|
||
instruct_model_large = "qwen2.5:32b" # used when the chosen host declares >= large_min_vram_gb
|
||
large_min_vram_gb = 20
|
||
chat_num_ctx = 8192 # passed per request; no Modelfile ctx variants
|
||
embed_dim = 768
|
||
# HNSW index build. Safe since the postgres image ships an AVX-512-free
|
||
# pgvector rebuild (infra/images/postgresql.Dockerfile, #580); before that it
|
||
# SIGILLed this Zen2 host. Set false to fall back to exact search.
|
||
build_ann_index = true
|
||
pg_host = "127.0.0.1" # host-side default; containers set LLM_PG_HOST=postgres
|
||
pg_port = 5432
|
||
pg_db = "llm"
|
||
pg_user = "llm"
|
||
# Retrieval: similarity × recency blend (half-life in days), sources kept
|
||
recency_half_life_days = 365
|
||
recency_weight = 0.3
|
||
top_n = 8
|
||
timeline_per_era = 2 # era_balance: hits kept per rule year on the first pass (mode="timeline")
|
||
timeline_overfetch = 6 # era_balance: k multiplier per kind for timeline mode (vs the recent-mode ×3)
|
||
duckdb_replica = "data/replica/aco.ro.duckdb" # read-only DuckDB replica the chat reads valuations from
|
||
valuation_years = 4 # final-rule vintages shown per code (plus the newest NPRM)
|
||
code_cited_per_code = 3 # excerpts literally citing each detected code (or family)
|
||
code_cited_collections = ["rules", "comments", "corpus"] # searched in this order
|
||
code_cited_max = 12 # cited excerpts kept after round-robin interleave across collections; <= 0 = unlimited
|
||
lineage_max_rows = 25 # collapsed lineage events kept in the prompt block (the SSE payload always carries every collapsed row)
|
||
lineage_on_demand_max = 3 # detected codes per turn allowed to fall back to pfs.lineage.lineage() when pfs.code_event has no rows for them
|
||
lineage_sources_max = 8 # FR paragraphs fetched as sources for events selected into the lineage prompt block, outside code_cited_max (Ruling B10: + up to 2 element-diff anchors, hard cap 10 total)
|
||
|
||
[llm.k_per_kind] # over-fetched ×3 per kind, then re-ranked
|
||
comment = 8
|
||
rule = 4
|
||
corpus = 4
|
||
|
||
[s3]
|
||
endpoint = "http://rustfs:9000"
|
||
region = "us-east-1"
|
||
|
||
[lake]
|
||
warehouse = "s3://lakehouse/"
|
||
|
||
[lake.nessie]
|
||
catalog_uri = "http://nessie:19120/iceberg/"
|
||
|
||
[lake.polaris]
|
||
catalog_uri = "http://polaris:8181/api/catalog"
|
||
|
||
[lake.databricks]
|
||
catalog_uri = "" # set via DATABRICKS_HOST env var
|
||
warehouse = "main" # Unity Catalog name
|
||
catalog = "aco" # default catalog for table references
|
||
|
||
[lake.trino]
|
||
host = "trino"
|
||
port = 8080
|
||
catalog = "iceberg"
|
||
|
||
# DuckLake — the concurrent reference-data store chosen in the M3 spike
|
||
# (docs/superpowers/specs/2026-07-10-ducklake-vs-iceberg-decision.md).
|
||
# The postgres catalog serializes concurrent writers; data is Parquet on
|
||
# RustFS. Password comes from the POSTGRES_PASSWORD env var; the catalog
|
||
# host is compose-internal, so lake writers run inside a data-network
|
||
# container.
|
||
[lake.ducklake]
|
||
catalog = "postgres:dbname=ducklake host=postgres user=postgres"
|
||
# Read-only role for notebook readers (password: DUCKLAKE_RO_PASSWORD).
|
||
catalog_ro = "postgres:dbname=ducklake host=postgres user=ducklake_ro"
|
||
data_path = "s3://lakehouse/ducklake/"
|
||
s3_endpoint = "rustfs:9000"
|
||
|
||
# ── Databricks Asset Bundle generation ────────────────────────────
|
||
# gen_config.py reads this to produce databricks.yml.
|
||
# Add a new pipeline module → commit → databricks.yml updates automatically.
|
||
|
||
[databricks]
|
||
bundle_name = "stack"
|
||
package_name = "stack"
|
||
entry_point = "cli"
|
||
schedule = "0 0 6 * * ?"
|
||
timezone = "America/New_York"
|
||
|
||
[databricks.clusters.default]
|
||
spark_version = "15.4.x-scala2.12"
|
||
node_type_id = "i3.xlarge"
|
||
num_workers = 1
|
||
|
||
[databricks.targets.dev]
|
||
mode = "development"
|
||
default = true
|
||
catalog = "aco_dev"
|
||
|
||
[databricks.targets.staging]
|
||
catalog = "aco_staging"
|
||
|
||
[databricks.targets.prod]
|
||
mode = "production"
|
||
catalog = "aco"
|
||
|
||
[databricks.warehouse]
|
||
name = "stack-sql-warehouse"
|
||
size = "SMALL"
|
||
auto_stop_mins = 10
|
||
|
||
[databricks.secrets]
|
||
scope = "stack"
|
||
mapping = [
|
||
{ env = "BCDA_CLIENT_ID", key = "bcda-client-id" },
|
||
{ env = "BCDA_CLIENT_SECRET", key = "bcda-client-secret" },
|
||
]
|
||
|
||
[databricks.governance]
|
||
phi_schemas = ["core", "claims_preprocessing", "cclf", "input_layer"]
|
||
|
||
[databricks.governance.groups.analysts]
|
||
schemas = ["readmissions", "quality_measures", "cms_quality_measures", "ahrq_measures"]
|
||
privileges = ["SELECT"]
|
||
|
||
[databricks.governance.groups.pipeline_svc]
|
||
schemas = ["*"]
|
||
privileges = ["ALL_PRIVILEGES"]
|
||
|
||
[databricks.quality]
|
||
monitored_schemas = ["core", "claims_preprocessing", "data_quality"]
|
||
output_schema = "_monitoring"
|
||
|
||
[default]
|
||
target = "local" # duckdb | databricks | trino | lake
|
||
|
||
[api]
|
||
host = "0.0.0.0"
|
||
port = 8000
|
||
secret = "" # override via STACK_API_SECRET env var
|
||
workers = 1
|
||
|
||
[ci]
|
||
backend = "gitea"
|
||
coverage_threshold = 99
|
||
|
||
[ci.github]
|
||
runner = "ubuntu-latest"
|
||
registry = "ghcr.io"
|
||
uv_version = "latest"
|
||
|
||
[ci.gitea]
|
||
runner_labels = ["ubuntu-latest"]
|
||
registry_secret = "REGISTRY_TOKEN"
|
||
uv_version = "latest"
|
||
ci_registry = "git:3000" # internal HTTP endpoint (avoids self-signed TLS)
|
||
|
||
[telemetry]
|
||
enabled = false
|
||
endpoint = "http://otel-collector:4317"
|
||
fallback = "traces/spans.jsonl"
|
||
sample_rate = 1.0
|
||
service_name = "stack"
|
||
|
||
[lint]
|
||
line_length = 88
|
||
|
||
# ── Container images ──────────────────────────────────────────────
|
||
# Single source of truth for all built images.
|
||
# gen_config.py reads this to generate pipeline YAML, infra-ci
|
||
# hadolint steps, trivy scans, S3 uploads, and vuln report loops.
|
||
# Add a new image here → commit → all pipelines update automatically.
|
||
|
||
[images.defaults]
|
||
scan = true
|
||
hadolint = true
|
||
trivy_severity = "HIGH,CRITICAL"
|
||
trivy_exit_code = 0
|
||
|
||
# Repo-root context: the Dockerfile COPYs infra/marimo/theme, which only
|
||
# exists at repo root (see .dockerignore carve-outs). Building from notebooks/
|
||
# fails with `"/infra/marimo/theme": not found`.
|
||
[images.notebooks]
|
||
dockerfile = "infra/images/notebooks.Dockerfile"
|
||
context = "."
|
||
path_filter = ["notebooks/**", "infra/marimo/**", "infra/images/notebooks.Dockerfile"]
|
||
|
||
# Repo-root context: the Dockerfile COPYs the desktop-bridge watchdog +
|
||
# healthcheck from infra/images/zotero/ (carved out in .dockerignore).
|
||
[images.zotero]
|
||
dockerfile = "infra/images/zotero.Dockerfile"
|
||
context = "."
|
||
path_filter = ["infra/images/zotero.Dockerfile", "infra/images/zotero/**"]
|
||
|
||
[images.docs]
|
||
dockerfile = "infra/images/docs.Dockerfile"
|
||
context = "."
|
||
path_filter = ["docs/**", "infra/images/docs.Dockerfile"]
|
||
prep_commands = [
|
||
"mkdir -p data",
|
||
"cp /host-data/bib.sqlite data/ 2>/dev/null || true",
|
||
]
|
||
|
||
[images.api]
|
||
dockerfile = "infra/images/api.Dockerfile"
|
||
context = "."
|
||
path_filter = ["infra/images/api.Dockerfile", "src/**", "pyproject.toml"]
|
||
|
||
[images.llm]
|
||
dockerfile = "infra/images/llm.Dockerfile"
|
||
context = "."
|
||
path_filter = ["infra/images/llm.Dockerfile", "src/**", "pyproject.toml"]
|
||
|
||
[images.mc]
|
||
dockerfile = "infra/images/mc.Dockerfile"
|
||
context = "infra/rustfs/"
|
||
path_filter = ["infra/rustfs/**", "infra/images/mc.Dockerfile"]
|
||
scan = false
|