stack/stack.toml

# stack.toml — centralised configuration for the stack platform.
#
# Paths are relative to the repository root unless absolute.
# Secrets live in .env (never here).
#
# Context switching
# -----------------
# [context.active] selects the environment profile.  Override at
# runtime with  STACK_CONTEXT=lake  (env var beats this file).
#
# Each context sets db_backend and storage_backend.  Modules call
# conf.context() to decide *which* engine or filesystem to use;
# the [db], [storage], and [lake] sections supply the concrete
# connection details for the selected backend.

[platform]
domain = "fhirworx.io"
registry = "git.fhirworx.io"
image_prefix = "fhirworx"
repo = "homelab/stack"
ci_email = "ci@fhirworx.io"
host_ip = "192.168.1.192"
subdomains = [
    "dashboard", "docs", "git", "ci", "notebooks", "zotero",
    "webdav", "api", "nessie", "trino", "polaris",
    "grafana", "prometheus", "tempo", "loki",
    "s3", "s3console", "llm",
]

[services]
git = "http://git:3000"
rustfs = "http://rustfs:9000"
nessie = "http://nessie:19120"
polaris = "http://polaris:8181"
trino = "http://trino:8080"
grafana = "http://grafana:3000"
ollama = "http://ollama:11434"

[context]
active = "local"

[context.local]
db_backend = "duckdb"
storage_backend = "local"

[context.lake]
db_backend = "iceberg"
storage_backend = "s3"
catalog = "nessie"

[context.databricks]
db_backend = "databricks"
storage_backend = "dbfs"

[context.trino]
db_backend = "trino"
storage_backend = "s3"

[context.aws]
db_backend = "iceberg"
storage_backend = "s3"
catalog = "glue"
compute = "ecs"

[context.gcp]
db_backend = "iceberg"
storage_backend = "gcs"
catalog = "bigquery"
compute = "cloud-run"

[context.azure]
db_backend = "iceberg"
storage_backend = "abfs"
catalog = "unity"
compute = "container-apps"

[db]
aco = "data/aco.duckdb"
bib = "data/bib.sqlite"
zotero = "data/zotero/data/zotero.sqlite"

[comments]
seal_quiet_days = 30   # days after a docket's comment close date before an empty pull seals it

[storage]
bib = "data/bib/storage"
zotero = "data/zotero/data/storage"
bcda = "data/bcda"
rex = "data/rex"
pfs = "data/pfs"
rec = "data/rec"
cms_log = "data/cms/log.jsonl"
bcda_log = "data/bcda/log.jsonl"

[generate]
table_out = "src/aco/table"
base_import = "aco.table.base"

[bcda]
sandbox = "https://sandbox.bcda.cms.gov"
production = "https://api.bcda.cms.gov"
max_retries = 3
retry_interval = 1.0
token_lifetime = 1200
timeout = 120.0

[llm]
ollama = "http://127.0.0.1:11434"     # host-side default; containers override via LLM_OLLAMA_HOSTS
embed_model = "nomic-embed-text"       # 768-dim; bake-off (P33 #564) may revise
instruct_model = "qwen2.5:14b"         # baseline, pulled on every fleet host
instruct_model_large = "qwen2.5:32b"   # used when the chosen host declares >= large_min_vram_gb
large_min_vram_gb = 20
chat_num_ctx = 8192                    # passed per request; no Modelfile ctx variants
chat_temperature = 0.2                 # grounded answers cite labels verbatim; low temperature keeps them literal
embed_dim = 768
# HNSW index build. Safe since the postgres image ships an AVX-512-free
# pgvector rebuild (infra/images/postgresql.Dockerfile, #580); before that it
# SIGILLed this Zen2 host. Set false to fall back to exact search.
build_ann_index = true
pg_host = "127.0.0.1"                  # host-side default; containers set LLM_PG_HOST=postgres
pg_port = 5432
pg_db = "llm"
pg_user = "llm"
# Retrieval: similarity × recency blend (half-life in days), sources kept
recency_half_life_days = 365
recency_weight = 0.3
top_n = 8
timeline_per_era = 2                     # era_balance: hits kept per rule year on the first pass (mode="timeline")
timeline_overfetch = 6                   # era_balance: k multiplier per kind for timeline mode (vs the recent-mode ×3)
duckdb_replica = "data/replica/aco.ro.duckdb"   # read-only DuckDB replica the chat reads valuations from
valuation_years = 4                      # final-rule vintages shown per code (plus the newest NPRM)
code_cited_per_code = 3                  # excerpts literally citing each detected code (or family)
code_cited_collections = ["rules", "comments", "corpus"]  # searched in this order
code_cited_max = 12                      # cited excerpts kept after round-robin interleave across collections; <= 0 = unlimited
lineage_max_rows = 25                    # collapsed lineage events kept in the prompt block (the SSE payload always carries every collapsed row)
lineage_on_demand_max = 3                # detected codes per turn allowed to fall back to pfs.lineage.lineage() when pfs.code_event has no rows for them
lineage_sources_max = 8                  # FR paragraphs fetched as sources for events selected into the lineage prompt block, outside code_cited_max (Ruling B10: + up to 2 element-diff anchors, hard cap 10 total)
chat_codes_max = 24                      # Ruling B11: per-turn cap on detected codes (explicit codes first, then family order) — applied before valuation/lineage/code-cited queries
valuation_rows_max = 24                  # Ruling B11: valuation rows kept in the prompt block (explicit codes first, then newest vintage); the SSE payload always carries every row

[llm.k_per_kind]                       # over-fetched ×3 per kind, then re-ranked
comment = 8
rule = 4
corpus = 4

[s3]
endpoint = "http://rustfs:9000"
region = "us-east-1"

[lake]
warehouse = "s3://lakehouse/"

[lake.nessie]
catalog_uri = "http://nessie:19120/iceberg/"

[lake.polaris]
catalog_uri = "http://polaris:8181/api/catalog"

[lake.databricks]
catalog_uri = ""   # set via DATABRICKS_HOST env var
warehouse = "main" # Unity Catalog name
catalog = "aco"    # default catalog for table references

[lake.trino]
host = "trino"
port = 8080
catalog = "iceberg"

# DuckLake — the concurrent reference-data store chosen in the M3 spike
# (docs/superpowers/specs/2026-07-10-ducklake-vs-iceberg-decision.md).
# The postgres catalog serializes concurrent writers; data is Parquet on
# RustFS. Password comes from the POSTGRES_PASSWORD env var; the catalog
# host is compose-internal, so lake writers run inside a data-network
# container.
[lake.ducklake]
catalog = "postgres:dbname=ducklake host=postgres user=postgres"
# Read-only role for notebook readers (password: DUCKLAKE_RO_PASSWORD).
catalog_ro = "postgres:dbname=ducklake host=postgres user=ducklake_ro"
data_path = "s3://lakehouse/ducklake/"
s3_endpoint = "rustfs:9000"

# ── Databricks Asset Bundle generation ────────────────────────────
# gen_config.py reads this to produce databricks.yml.
# Add a new pipeline module → commit → databricks.yml updates automatically.

[databricks]
bundle_name = "stack"
package_name = "stack"
entry_point = "cli"
schedule = "0 0 6 * * ?"
timezone = "America/New_York"

[databricks.clusters.default]
spark_version = "15.4.x-scala2.12"
node_type_id = "i3.xlarge"
num_workers = 1

[databricks.targets.dev]
mode = "development"
default = true
catalog = "aco_dev"

[databricks.targets.staging]
catalog = "aco_staging"

[databricks.targets.prod]
mode = "production"
catalog = "aco"

[databricks.warehouse]
name = "stack-sql-warehouse"
size = "SMALL"
auto_stop_mins = 10

[databricks.secrets]
scope = "stack"
mapping = [
    { env = "BCDA_CLIENT_ID", key = "bcda-client-id" },
    { env = "BCDA_CLIENT_SECRET", key = "bcda-client-secret" },
]

[databricks.governance]
phi_schemas = ["core", "claims_preprocessing", "cclf", "input_layer"]

[databricks.governance.groups.analysts]
schemas = ["readmissions", "quality_measures", "cms_quality_measures", "ahrq_measures"]
privileges = ["SELECT"]

[databricks.governance.groups.pipeline_svc]
schemas = ["*"]
privileges = ["ALL_PRIVILEGES"]

[databricks.quality]
monitored_schemas = ["core", "claims_preprocessing", "data_quality"]
output_schema = "_monitoring"

[default]
target = "local" # duckdb | databricks | trino | lake

[api]
host = "0.0.0.0"
port = 8000
secret = ""  # override via STACK_API_SECRET env var
workers = 1

[ci]
backend = "gitea"
coverage_threshold = 99

[ci.github]
runner = "ubuntu-latest"
registry = "ghcr.io"
uv_version = "latest"

[ci.gitea]
runner_labels = ["ubuntu-latest"]
registry_secret = "REGISTRY_TOKEN"
uv_version = "latest"
ci_registry = "git:3000"  # internal HTTP endpoint (avoids self-signed TLS)

[telemetry]
enabled = false
endpoint = "http://otel-collector:4317"
fallback = "traces/spans.jsonl"
sample_rate = 1.0
service_name = "stack"

[lint]
line_length = 88

# ── Container images ──────────────────────────────────────────────
# Single source of truth for all built images.
# gen_config.py reads this to generate pipeline YAML, infra-ci
# hadolint steps, trivy scans, S3 uploads, and vuln report loops.
# Add a new image here → commit → all pipelines update automatically.

[images.defaults]
scan = true
hadolint = true
trivy_severity = "HIGH,CRITICAL"
trivy_exit_code = 0

# Repo-root context: the Dockerfile COPYs infra/marimo/theme, which only
# exists at repo root (see .dockerignore carve-outs). Building from notebooks/
# fails with `"/infra/marimo/theme": not found`.
[images.notebooks]
dockerfile = "infra/images/notebooks.Dockerfile"
context = "."
path_filter = ["notebooks/**", "infra/marimo/**", "infra/images/notebooks.Dockerfile"]

# Repo-root context: the Dockerfile COPYs the desktop-bridge watchdog +
# healthcheck from infra/images/zotero/ (carved out in .dockerignore).
[images.zotero]
dockerfile = "infra/images/zotero.Dockerfile"
context = "."
path_filter = ["infra/images/zotero.Dockerfile", "infra/images/zotero/**"]

[images.docs]
dockerfile = "infra/images/docs.Dockerfile"
context = "."
path_filter = ["docs/**", "infra/images/docs.Dockerfile"]
prep_commands = [
    "mkdir -p data",
    "cp /host-data/bib.sqlite data/ 2>/dev/null || true",
]

[images.api]
dockerfile = "infra/images/api.Dockerfile"
context = "."
path_filter = ["infra/images/api.Dockerfile", "src/**", "pyproject.toml"]

[images.llm]
dockerfile = "infra/images/llm.Dockerfile"
context = "."
path_filter = ["infra/images/llm.Dockerfile", "src/**", "pyproject.toml"]

[images.mc]
dockerfile = "infra/images/mc.Dockerfile"
context = "infra/rustfs/"
path_filter = ["infra/rustfs/**", "infra/images/mc.Dockerfile"]
scan = false