8cd4633dac
cloud/databricks/README.md: what the platform guarantees (ephemeral container, Python 3.11 default → uv pin 3.12, app.yaml command/env with valueFrom, injected service-principal credentials, user authorization via X-Forwarded-Access-Token and opt-in scopes, Medium/Large compute, 15 s SIGTERM, 10 MB per source file, restricted egress policies), the permission split for a user who cannot create or manage catalogs (admin provisions catalog/schema/volume/warehouse/secrets once; attaching a uc_securable grants USE CATALOG/USE SCHEMA automatically; only CREATE TABLE/MODIFY/SELECT on the schema need a schema manager), the compose → Databricks mapping (store on a UC volume via fsspec, Delta tables through the SQL warehouse, no DuckDB file, no second port), the eight small code changes the port needs, optimisation notes (app as control plane, size per phase, cache statics, one connection per session, never pull a table into the app, sync only what the app imports), a runbook, phases 1–4 and the open questions to verify in the target workspace. Sources listed. cloud/databricks/bcda-app/: a bundle of its own (the main bundle creates schemas and a volume, which needs catalog rights) with the app resource, its warehouse/secret/volume resources and permissions, app.yaml, .python-version and a requirements stub. refs #1139
105 lines
3.2 KiB
YAML
105 lines
3.2 KiB
YAML
# BCDA manager as a Databricks App — a bundle of its own.
|
|
#
|
|
# Deliberately separate from the repo's main bundle (bundle/, generated by
|
|
# dev/scripts/gen_config.py), which CREATES schemas and a volume and so needs
|
|
# catalog-level rights. This one only references objects an admin provisioned
|
|
# once (cloud/databricks/README.md §2.1) and asks for grants to the app's
|
|
# service principal. Deploy from this directory:
|
|
#
|
|
# databricks bundle validate -t dev && databricks bundle deploy -t dev && databricks bundle run bcda_manager -t dev
|
|
bundle:
|
|
name: bcda-manager
|
|
|
|
variables:
|
|
catalog:
|
|
description: Existing catalog (no CREATE CATALOG needed)
|
|
default: aco_dev
|
|
schema:
|
|
description: Existing schema holding the store volume and bcda.* tables
|
|
default: bcda
|
|
volume:
|
|
description: Existing managed volume used as the store root
|
|
default: bcda_store
|
|
warehouse_id:
|
|
description: SQL warehouse the app may use
|
|
lookup:
|
|
warehouse: stack-sql-warehouse
|
|
secret_scope:
|
|
default: stack
|
|
users_group:
|
|
description: Group that may open the app (Data page runs as them)
|
|
default: bcda-analysts
|
|
|
|
# Only what the app imports. The whole repo is far over the 10 MB per-file
|
|
# limit (dev/seeds ndjson, uv.lock is fine) and most of it is irrelevant.
|
|
sync:
|
|
paths:
|
|
- ../../..
|
|
include:
|
|
- ../../../src/**
|
|
- ../../../pyproject.toml
|
|
- ../../../uv.lock
|
|
- ../../../stack.toml
|
|
- ../../../README.md
|
|
- ../../../dev/seeds/bcda_v3_*.csv
|
|
- ../../../assets/icons/fav32.png
|
|
- ./app.yaml
|
|
- ./requirements.txt
|
|
- ./.python-version
|
|
exclude:
|
|
- ../../../data/**
|
|
- ../../../dev/vendor/**
|
|
- ../../../tuva/**
|
|
- ../../../infra/**
|
|
- ../../../docs/**
|
|
- ../../../notebooks/**
|
|
- ../../../.venv/**
|
|
|
|
resources:
|
|
apps:
|
|
bcda_manager:
|
|
name: bcda-manager-${bundle.target}
|
|
description: "BCDA v3 manager — exports, store, _since reconciliation, load, crosswalk"
|
|
source_code_path: ../../..
|
|
# MEDIUM (2 vCPU / 6 GB) for the UI; switch to LARGE while a baseline
|
|
# pull runs in-app (README §5), back afterwards.
|
|
compute_size: MEDIUM
|
|
resources:
|
|
- name: sql_warehouse
|
|
sql_warehouse:
|
|
id: ${var.warehouse_id}
|
|
permission: CAN_USE
|
|
- name: bcda_client_id
|
|
secret:
|
|
scope: ${var.secret_scope}
|
|
key: bcda-client-id
|
|
permission: READ
|
|
- name: bcda_client_secret
|
|
secret:
|
|
scope: ${var.secret_scope}
|
|
key: bcda-client-secret
|
|
permission: READ
|
|
- name: bcda_store
|
|
uc_securable:
|
|
securable_type: VOLUME
|
|
securable_full_name: ${var.catalog}.${var.schema}.${var.volume}
|
|
permission: WRITE_VOLUME
|
|
# The Data page queries as the signed-in user; sql:restricted-query is
|
|
# enough for read-only browsing. Admins may restrict allowed scopes.
|
|
user_api_scopes:
|
|
- sql:restricted-query
|
|
permissions:
|
|
- group_name: ${var.users_group}
|
|
level: CAN_USE
|
|
|
|
targets:
|
|
dev:
|
|
mode: development
|
|
default: true
|
|
prod:
|
|
mode: production
|
|
variables:
|
|
catalog: aco
|
|
workspace:
|
|
root_path: /Workspace/Shared/.bundle/${bundle.name}/${bundle.target}
|