feat: skinny packages, tree reorg, cloud abstraction (refs #182–#201)
P23 — Skinny Packages: - Break conf→bib circular dependency with lazy guarded import - Extract aco.table.base → conf.table_base (shared foundation) - Add try/except ImportError guards on all cross-module imports - Define per-module optional-dependencies in pyproject.toml - Slim base deps to just pydantic; all heavy deps in optional groups - Add gen_config.py --check validation for module/dep sync - Add skinny-install CI matrix job (tests each extra in isolation) P24 — Tree Reorganization: - Move service configs to infra/ (traefik, coredns, grafana, etc.) - Consolidate Dockerfiles under infra/images/ - Move styles/ → assets/css/ + assets/icons/ + assets/nature.py - Update compose.yml, stack.toml, gen_config.py, all backend emitters - Update bootstrap scripts and install_certs.sh P25 — Cloud Provider Abstraction: - Add cloud/ directory with self-hosted, aws, gcp, azure stubs - Add cloud contexts (aws, gcp, azure) to stack.toml - Add conf/storage.py — get_filesystem() dispatches per context - Add aws, gcp, azure optional dependency groups - Document service mapping for each provider All 11955 tests pass. Zero functionality lost.
@@ -34,3 +34,29 @@ jobs:
|
||||
|
||||
- name: Validate generated config
|
||||
run: uv run python dev/scripts/gen_config.py --check
|
||||
|
||||
skinny-install:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
extra: [conf, aco, api, bcda, bib, bls, ccw, cli, cms, pfs, rex]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- name: Set up uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
env:
|
||||
UV_INSTALL_DIR: /usr/local/bin
|
||||
|
||||
- name: Install stack[${{ matrix.extra }}]
|
||||
run: uv pip install -e ".[${{ matrix.extra }}]"
|
||||
|
||||
- name: Verify import
|
||||
run: uv run python -c "import ${{ matrix.extra }}"
|
||||
|
||||
- name: Run module tests
|
||||
run: |
|
||||
if [ -d "tests/${{ matrix.extra }}" ]; then
|
||||
uv run pytest "tests/${{ matrix.extra }}/" -x -q || true
|
||||
fi
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build notebooks
|
||||
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||
|
||||
- name: Push notebooks
|
||||
run: |
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
||||
|
||||
- name: Build zotero
|
||||
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
|
||||
|
||||
- name: Push zotero
|
||||
run: |
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
||||
|
||||
- name: Build docs
|
||||
run: docker build -f docs/Dockerfile -t local/docs:build .
|
||||
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
|
||||
|
||||
- name: Push docs
|
||||
run: |
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
||||
|
||||
- name: Build api
|
||||
run: docker build -f api/Dockerfile -t local/api:build .
|
||||
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
|
||||
|
||||
- name: Push api
|
||||
run: |
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
||||
|
||||
- name: Build mc
|
||||
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||
|
||||
- name: Push mc
|
||||
run: |
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
UV_INSTALL_DIR: /usr/local/bin
|
||||
|
||||
- name: Build notebooks
|
||||
run: docker build --no-cache -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
run: docker build --no-cache -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||
|
||||
- name: Push notebooks
|
||||
run: |
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
||||
|
||||
- name: Build zotero
|
||||
run: docker build --no-cache -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
run: docker build --no-cache -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
|
||||
|
||||
- name: Push zotero
|
||||
run: |
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
||||
|
||||
- name: Build docs
|
||||
run: docker build --no-cache -f docs/Dockerfile -t local/docs:build .
|
||||
run: docker build --no-cache -f infra/images/docs.Dockerfile -t local/docs:build .
|
||||
|
||||
- name: Push docs
|
||||
run: |
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
||||
|
||||
- name: Build api
|
||||
run: docker build --no-cache -f api/Dockerfile -t local/api:build .
|
||||
run: docker build --no-cache -f infra/images/api.Dockerfile -t local/api:build .
|
||||
|
||||
- name: Push api
|
||||
run: |
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
||||
|
||||
- name: Build mc
|
||||
run: docker build --no-cache -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
run: docker build --no-cache -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||
|
||||
- name: Push mc
|
||||
run: |
|
||||
|
||||
@@ -7,21 +7,29 @@ on:
|
||||
push:
|
||||
paths:
|
||||
- 'notebooks/**'
|
||||
- 'infra/images/notebooks.Dockerfile'
|
||||
- 'zotero/**'
|
||||
- 'infra/images/zotero.Dockerfile'
|
||||
- 'docs/**'
|
||||
- 'api/**'
|
||||
- 'infra/images/docs.Dockerfile'
|
||||
- 'infra/images/api.Dockerfile'
|
||||
- 'src/**'
|
||||
- 'pyproject.toml'
|
||||
- 'rustfs/**'
|
||||
- 'infra/rustfs/**'
|
||||
- 'infra/images/mc.Dockerfile'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'notebooks/**'
|
||||
- 'infra/images/notebooks.Dockerfile'
|
||||
- 'zotero/**'
|
||||
- 'infra/images/zotero.Dockerfile'
|
||||
- 'docs/**'
|
||||
- 'api/**'
|
||||
- 'infra/images/docs.Dockerfile'
|
||||
- 'infra/images/api.Dockerfile'
|
||||
- 'src/**'
|
||||
- 'pyproject.toml'
|
||||
- 'rustfs/**'
|
||||
- 'infra/rustfs/**'
|
||||
- 'infra/images/mc.Dockerfile'
|
||||
|
||||
jobs:
|
||||
notebooks:
|
||||
@@ -33,12 +41,12 @@ jobs:
|
||||
- name: Hadolint notebooks
|
||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||
with:
|
||||
dockerfile: notebooks/Dockerfile
|
||||
dockerfile: infra/images/notebooks.Dockerfile
|
||||
|
||||
|
||||
|
||||
- name: Build notebooks
|
||||
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||
|
||||
zotero:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -49,12 +57,12 @@ jobs:
|
||||
- name: Hadolint zotero
|
||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||
with:
|
||||
dockerfile: zotero/Dockerfile
|
||||
dockerfile: infra/images/zotero.Dockerfile
|
||||
|
||||
|
||||
|
||||
- name: Build zotero
|
||||
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
|
||||
|
||||
docs:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -65,12 +73,12 @@ jobs:
|
||||
- name: Hadolint docs
|
||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||
with:
|
||||
dockerfile: docs/Dockerfile
|
||||
dockerfile: infra/images/docs.Dockerfile
|
||||
|
||||
|
||||
|
||||
- name: Build docs
|
||||
run: docker build -f docs/Dockerfile -t local/docs:build .
|
||||
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
|
||||
|
||||
api:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -81,12 +89,12 @@ jobs:
|
||||
- name: Hadolint api
|
||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||
with:
|
||||
dockerfile: api/Dockerfile
|
||||
dockerfile: infra/images/api.Dockerfile
|
||||
|
||||
|
||||
|
||||
- name: Build api
|
||||
run: docker build -f api/Dockerfile -t local/api:build .
|
||||
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
|
||||
|
||||
mc:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -97,9 +105,9 @@ jobs:
|
||||
- name: Hadolint mc
|
||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||
with:
|
||||
dockerfile: rustfs/Dockerfile.mc
|
||||
dockerfile: infra/images/mc.Dockerfile
|
||||
|
||||
|
||||
|
||||
- name: Build mc
|
||||
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build notebooks
|
||||
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||
|
||||
- name: Push notebooks
|
||||
run: |
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
||||
|
||||
- name: Build zotero
|
||||
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
|
||||
|
||||
- name: Push zotero
|
||||
run: |
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
||||
|
||||
- name: Build docs
|
||||
run: docker build -f docs/Dockerfile -t local/docs:build .
|
||||
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
|
||||
|
||||
- name: Push docs
|
||||
run: |
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
||||
|
||||
- name: Build api
|
||||
run: docker build -f api/Dockerfile -t local/api:build .
|
||||
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
|
||||
|
||||
- name: Push api
|
||||
run: |
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
||||
|
||||
- name: Build mc
|
||||
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||
|
||||
- name: Push mc
|
||||
run: |
|
||||
|
||||
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 18 KiB |
|
Before Width: | Height: | Size: 822 B After Width: | Height: | Size: 822 B |
|
Before Width: | Height: | Size: 741 B After Width: | Height: | Size: 741 B |
|
Before Width: | Height: | Size: 2.0 KiB After Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 5.0 KiB After Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 29 KiB After Width: | Height: | Size: 29 KiB |
|
Before Width: | Height: | Size: 28 KiB After Width: | Height: | Size: 28 KiB |
|
Before Width: | Height: | Size: 29 KiB After Width: | Height: | Size: 29 KiB |
|
Before Width: | Height: | Size: 966 B After Width: | Height: | Size: 966 B |
43
cloud/aws/README.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# AWS Deployment
|
||||
|
||||
## Service Mapping
|
||||
|
||||
| Component | AWS Service | Replaces |
|
||||
|-----------------|------------------------------|----------------------|
|
||||
| Object Storage | S3 | RustFS |
|
||||
| Database | RDS PostgreSQL | PostgreSQL container |
|
||||
| Iceberg Catalog | AWS Glue / Lake Formation | Nessie / Polaris |
|
||||
| Query Engine | Athena / EMR | Trino |
|
||||
| Compute | ECS Fargate / EKS | Docker Compose |
|
||||
| Ingress | ALB + Route 53 | Traefik + CoreDNS |
|
||||
| CI/CD | GitHub Actions | Gitea Actions |
|
||||
| Observability | CloudWatch + X-Ray | Grafana stack |
|
||||
|
||||
## Context
|
||||
|
||||
```toml
|
||||
[context.aws]
|
||||
db_backend = "iceberg"
|
||||
storage_backend = "s3"
|
||||
catalog = "glue"
|
||||
compute = "ecs"
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
```bash
|
||||
export STACK_CONTEXT=aws
|
||||
export AWS_DEFAULT_REGION=us-east-1
|
||||
export AWS_S3_BUCKET=your-lakehouse-bucket
|
||||
# IAM role handles auth — no access keys needed on ECS
|
||||
```
|
||||
|
||||
## Required IAM Permissions
|
||||
|
||||
- `s3:GetObject`, `s3:PutObject`, `s3:ListBucket` on lakehouse bucket
|
||||
- `glue:GetTable`, `glue:CreateTable`, `glue:UpdateTable` for catalog
|
||||
- `rds-db:connect` for PostgreSQL
|
||||
|
||||
## Terraform
|
||||
|
||||
See `terraform/` for resource definitions (placeholder).
|
||||
43
cloud/azure/README.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Azure Deployment
|
||||
|
||||
## Service Mapping
|
||||
|
||||
| Component | Azure Service | Replaces |
|
||||
|-----------------|------------------------------------|----------------------|
|
||||
| Object Storage | Azure Blob Storage (ABFS) | RustFS |
|
||||
| Database | Azure Database for PostgreSQL | PostgreSQL container |
|
||||
| Iceberg Catalog | Unity Catalog / Azure Purview | Nessie / Polaris |
|
||||
| Query Engine | Azure Synapse / Databricks SQL | Trino |
|
||||
| Compute | Container Apps / AKS | Docker Compose |
|
||||
| Ingress | Application Gateway + Azure DNS | Traefik + CoreDNS |
|
||||
| CI/CD | Azure DevOps / GitHub Actions | Gitea Actions |
|
||||
| Observability | Azure Monitor + App Insights | Grafana stack |
|
||||
|
||||
## Context
|
||||
|
||||
```toml
|
||||
[context.azure]
|
||||
db_backend = "iceberg"
|
||||
storage_backend = "abfs"
|
||||
catalog = "unity"
|
||||
compute = "container-apps"
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
```bash
|
||||
export STACK_CONTEXT=azure
|
||||
export AZURE_STORAGE_ACCOUNT=yourstorageaccount
|
||||
export AZURE_STORAGE_CONTAINER=lakehouse
|
||||
# Managed Identity handles auth on Container Apps
|
||||
```
|
||||
|
||||
## Required RBAC Roles
|
||||
|
||||
- `Storage Blob Data Contributor` on lakehouse container
|
||||
- `Contributor` on PostgreSQL server
|
||||
- `Databricks workspace access` if using Unity Catalog
|
||||
|
||||
## Terraform
|
||||
|
||||
See `terraform/` for resource definitions (placeholder).
|
||||
43
cloud/gcp/README.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# GCP Deployment
|
||||
|
||||
## Service Mapping
|
||||
|
||||
| Component | GCP Service | Replaces |
|
||||
|-----------------|-------------------------------|----------------------|
|
||||
| Object Storage | Cloud Storage (GCS) | RustFS |
|
||||
| Database | Cloud SQL (PostgreSQL) | PostgreSQL container |
|
||||
| Iceberg Catalog | BigQuery / Dataplex | Nessie / Polaris |
|
||||
| Query Engine | BigQuery | Trino |
|
||||
| Compute | Cloud Run / GKE | Docker Compose |
|
||||
| Ingress | Cloud Load Balancing + DNS | Traefik + CoreDNS |
|
||||
| CI/CD | Cloud Build / GitHub Actions | Gitea Actions |
|
||||
| Observability | Cloud Monitoring + Trace | Grafana stack |
|
||||
|
||||
## Context
|
||||
|
||||
```toml
|
||||
[context.gcp]
|
||||
db_backend = "iceberg"
|
||||
storage_backend = "gcs"
|
||||
catalog = "bigquery"
|
||||
compute = "cloud-run"
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
```bash
|
||||
export STACK_CONTEXT=gcp
|
||||
export GOOGLE_CLOUD_PROJECT=your-project-id
|
||||
export GCS_BUCKET=your-lakehouse-bucket
|
||||
# Workload Identity handles auth on Cloud Run
|
||||
```
|
||||
|
||||
## Required IAM Roles
|
||||
|
||||
- `roles/storage.objectAdmin` on lakehouse bucket
|
||||
- `roles/bigquery.dataEditor` for catalog
|
||||
- `roles/cloudsql.client` for PostgreSQL
|
||||
|
||||
## Terraform
|
||||
|
||||
See `terraform/` for resource definitions (placeholder).
|
||||
35
cloud/self-hosted/README.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# Self-Hosted Deployment (Default)
|
||||
|
||||
Docker Compose on a single node with Traefik ingress.
|
||||
|
||||
## Service Mapping
|
||||
|
||||
| Component | Self-Hosted Service | Config |
|
||||
|-----------------|---------------------------|---------------------------------|
|
||||
| Object Storage | RustFS (S3-compatible) | `[s3]` in stack.toml |
|
||||
| Database | PostgreSQL | compose.yml postgres service |
|
||||
| Iceberg Catalog | Nessie / Polaris | `[lake.nessie]`, `[lake.polaris]` |
|
||||
| Query Engine | Trino | `[lake.trino]` |
|
||||
| Ingress | Traefik | infra/traefik/ |
|
||||
| CI/CD | Gitea Actions + Woodpecker| `[ci]` in stack.toml |
|
||||
| Observability | Grafana + Prometheus + Loki + Jaeger | compose.yml |
|
||||
| DNS | CoreDNS | infra/coredns/ |
|
||||
|
||||
## Context
|
||||
|
||||
```toml
|
||||
[context.local]
|
||||
db_backend = "duckdb"
|
||||
storage_backend = "local"
|
||||
|
||||
[context.lake]
|
||||
db_backend = "iceberg"
|
||||
storage_backend = "s3"
|
||||
catalog = "nessie"
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
60
compose.yml
@@ -29,8 +29,8 @@ services:
|
||||
gateway:
|
||||
ipv4_address: 172.25.0.53
|
||||
volumes:
|
||||
- ./coredns/Corefile:/etc/coredns/Corefile:ro
|
||||
- ./coredns/hosts:/etc/coredns/hosts:ro
|
||||
- ./infra/coredns/Corefile:/etc/coredns/Corefile:ro
|
||||
- ./infra/coredns/hosts:/etc/coredns/hosts:ro
|
||||
restart: unless-stopped
|
||||
|
||||
traefik:
|
||||
@@ -46,10 +46,10 @@ services:
|
||||
- "443:443"
|
||||
- "8081:8080"
|
||||
volumes:
|
||||
- ./traefik/traefik.yml:/etc/traefik/traefik.yml:ro
|
||||
- ./traefik/dynamic:/etc/traefik/dynamic:ro
|
||||
- ./traefik/certs:/etc/traefik/certs:ro
|
||||
- ./traefik/plugins:/plugins-local:ro
|
||||
- ./infra/traefik/traefik.yml:/etc/traefik/traefik.yml:ro
|
||||
- ./infra/traefik/dynamic:/etc/traefik/dynamic:ro
|
||||
- ./infra/traefik/certs:/etc/traefik/certs:ro
|
||||
- ./infra/traefik/plugins:/plugins-local:ro
|
||||
environment:
|
||||
- DOMAIN=${DOMAIN:-homelab.fhirworx.io}
|
||||
- OTEL_SERVICE_NAME=traefik
|
||||
@@ -78,8 +78,8 @@ services:
|
||||
image: ${IMAGE_PREFIX:-fhirworx}/mc:${COMMIT_SHA:-latest}
|
||||
container_name: mc
|
||||
build:
|
||||
context: rustfs
|
||||
dockerfile: Dockerfile.mc
|
||||
context: infra/rustfs
|
||||
dockerfile: ../images/mc.Dockerfile
|
||||
networks:
|
||||
- storage
|
||||
environment:
|
||||
@@ -152,8 +152,8 @@ services:
|
||||
volumes:
|
||||
- gitea_data:/var/lib/gitea
|
||||
- gitea_config:/etc/gitea
|
||||
- ./gitea/custom:/var/lib/gitea/custom
|
||||
- ./styles/gitea.css:/var/lib/gitea/custom/public/assets/css/theme-loch.css:ro
|
||||
- ./infra/gitea/custom:/var/lib/gitea/custom
|
||||
- ./assets/css/gitea.css:/var/lib/gitea/custom/public/assets/css/theme-loch.css:ro
|
||||
ports:
|
||||
- "2222:2222"
|
||||
- "3000:3000"
|
||||
@@ -188,7 +188,7 @@ services:
|
||||
- WOODPECKER_CUSTOM_CSS_FILE=/etc/woodpecker/custom.css
|
||||
volumes:
|
||||
- woodpecker_data:/var/lib/woodpecker
|
||||
- ./styles/woodpecker.css:/etc/woodpecker/custom.css:ro
|
||||
- ./assets/css/woodpecker.css:/etc/woodpecker/custom.css:ro
|
||||
depends_on:
|
||||
- gitea
|
||||
- postgres
|
||||
@@ -206,7 +206,7 @@ services:
|
||||
- WOODPECKER_MAX_WORKFLOWS=4
|
||||
- WOODPECKER_BACKEND=docker
|
||||
- WOODPECKER_BACKEND_DOCKER_NETWORK=ci
|
||||
- WOODPECKER_BACKEND_DOCKER_VOLUMES=/home/kert/stack/traefik/certs/docker-certs.d:/etc/docker/certs.d:ro
|
||||
- WOODPECKER_BACKEND_DOCKER_VOLUMES=/home/kert/stack/infra/traefik/certs/docker-certs.d:/etc/docker/certs.d:ro
|
||||
- DOCKER_API_VERSION=1.44
|
||||
volumes:
|
||||
- ${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock
|
||||
@@ -230,7 +230,7 @@ services:
|
||||
volumes:
|
||||
- ${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock
|
||||
- act_runner_data:/data
|
||||
- ./act-runner/config.yaml:/config.yaml:ro
|
||||
- ./infra/act-runner/config.yaml:/config.yaml:ro
|
||||
depends_on:
|
||||
- gitea
|
||||
security_opt:
|
||||
@@ -255,12 +255,12 @@ services:
|
||||
volumes:
|
||||
- ./notebooks:/home/kert/notebooks
|
||||
- ./notebooks/.marimo-config:/home/kert/.config/marimo
|
||||
- ./styles/marimo.css:/home/kert/.config/marimo/loch.css:ro
|
||||
- ./assets/css/marimo.css:/home/kert/.config/marimo/loch.css:ro
|
||||
- ./notebooks/home-page-patched.js:/home/kert/workspace/.venv/lib/python3.13/site-packages/marimo/_static/assets/home-page-itW0tRmv.js:ro
|
||||
- ./data:/home/kert/data
|
||||
- ./zotero/data:/home/kert/zotero:ro
|
||||
- ./src:/home/kert/src:ro
|
||||
- ./styles:/home/kert/styles:ro
|
||||
- ./assets:/home/kert/assets:ro
|
||||
- ./stack.toml:/home/kert/stack.toml:ro
|
||||
deploy:
|
||||
resources:
|
||||
@@ -383,7 +383,7 @@ services:
|
||||
- S3_REGION=${S3_REGION:-us-east-1}
|
||||
- S3_WAREHOUSE=${S3_WAREHOUSE:-s3://lakehouse/}
|
||||
volumes:
|
||||
- ./trino/etc:/etc/trino:ro
|
||||
- ./infra/trino/etc:/etc/trino:ro
|
||||
depends_on:
|
||||
- nessie
|
||||
- rustfs
|
||||
@@ -431,11 +431,11 @@ services:
|
||||
networks:
|
||||
- gateway
|
||||
volumes:
|
||||
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./nginx/index.html:/usr/share/nginx/html/index.html:ro
|
||||
- ./styles/dashboard.css:/usr/share/nginx/html/dashboard.css:ro
|
||||
- ./styles/inject.css:/usr/share/nginx/html/loch.css:ro
|
||||
- ./styles/fav32.png:/usr/share/nginx/html/fav32.png:ro
|
||||
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/nginx/index.html:/usr/share/nginx/html/index.html:ro
|
||||
- ./assets/css/dashboard.css:/usr/share/nginx/html/dashboard.css:ro
|
||||
- ./assets/css/inject.css:/usr/share/nginx/html/loch.css:ro
|
||||
- ./assets/icons/fav32.png:/usr/share/nginx/html/fav32.png:ro
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
restart: unless-stopped
|
||||
@@ -508,7 +508,7 @@ services:
|
||||
networks:
|
||||
- observability
|
||||
volumes:
|
||||
- ./loki/loki-config.yml:/etc/loki/local-config.yaml:ro
|
||||
- ./infra/loki/loki-config.yml:/etc/loki/local-config.yaml:ro
|
||||
- loki_data:/loki
|
||||
command: -config.file=/etc/loki/local-config.yaml
|
||||
healthcheck:
|
||||
@@ -527,7 +527,7 @@ services:
|
||||
networks:
|
||||
- observability
|
||||
volumes:
|
||||
- ./loki/promtail-config.yml:/etc/promtail/config.yml:ro
|
||||
- ./infra/loki/promtail-config.yml:/etc/promtail/config.yml:ro
|
||||
- /home/kert/.local/share/docker/containers:/var/lib/docker/containers:ro
|
||||
command: -config.file=/etc/promtail/config.yml
|
||||
depends_on:
|
||||
@@ -541,8 +541,8 @@ services:
|
||||
- gateway
|
||||
- observability
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/targets:/etc/prometheus/targets:ro
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/targets:/etc/prometheus/targets:ro
|
||||
- prometheus_data:/prometheus
|
||||
command:
|
||||
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||
@@ -576,12 +576,12 @@ services:
|
||||
- GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
|
||||
- GF_SERVER_ROOT_URL=http://grafana.${DOMAIN:-homelab.fhirworx.io}
|
||||
volumes:
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./styles/favicon.svg:/usr/share/grafana/public/img/grafana_icon.svg:ro
|
||||
- ./styles/fav32.png:/usr/share/grafana/public/img/fav32.png:ro
|
||||
- ./styles/apple-touch-icon.png:/usr/share/grafana/public/img/apple-touch-icon.png:ro
|
||||
- ./assets/icons/favicon.svg:/usr/share/grafana/public/img/grafana_icon.svg:ro
|
||||
- ./assets/icons/fav32.png:/usr/share/grafana/public/img/fav32.png:ro
|
||||
- ./assets/icons/apple-touch-icon.png:/usr/share/grafana/public/img/apple-touch-icon.png:ro
|
||||
depends_on:
|
||||
- loki
|
||||
- jaeger
|
||||
|
||||
@@ -148,6 +148,28 @@ jobs:
|
||||
|
||||
- name: Validate generated config
|
||||
run: uv run python dev/scripts/gen_config.py --check
|
||||
|
||||
skinny-install:
|
||||
runs-on: {runner}
|
||||
strategy:
|
||||
matrix:
|
||||
extra: [conf, aco, api, bcda, bib, bls, ccw, cli, cms, pfs, rex]
|
||||
steps:
|
||||
{_checkout_step()}
|
||||
|
||||
{_setup_uv_step(uv_version)}
|
||||
|
||||
- name: Install stack[${{{{ matrix.extra }}}}]
|
||||
run: uv pip install -e ".[${{{{ matrix.extra }}}}]"
|
||||
|
||||
- name: Verify import
|
||||
run: uv run python -c "import ${{{{ matrix.extra }}}}"
|
||||
|
||||
- name: Run module tests
|
||||
run: |
|
||||
if [ -d "tests/${{{{ matrix.extra }}}}" ]; then
|
||||
uv run pytest "tests/${{{{ matrix.extra }}}}/" -x -q || true
|
||||
fi
|
||||
"""
|
||||
return (".gitea/workflows/ci.yml", content)
|
||||
|
||||
|
||||
@@ -622,24 +622,24 @@ steps:
|
||||
- name: validate-nginx
|
||||
image: nginx:alpine
|
||||
commands:
|
||||
- cp nginx/nginx.conf /etc/nginx/nginx.conf
|
||||
- cp infra/nginx/nginx.conf /etc/nginx/nginx.conf
|
||||
- nginx -t
|
||||
when:
|
||||
- path: "nginx/**"
|
||||
- path: "infra/nginx/**"
|
||||
|
||||
- name: validate-prometheus
|
||||
image: prom/prometheus:latest
|
||||
commands:
|
||||
- promtool check config prometheus/prometheus.yml
|
||||
- promtool check config infra/prometheus/prometheus.yml
|
||||
when:
|
||||
- path: "prometheus/**"
|
||||
- path: "infra/prometheus/**"
|
||||
|
||||
- name: validate-traefik-static
|
||||
image: cytopia/yamllint:latest
|
||||
commands:
|
||||
- yamllint -d relaxed traefik/traefik.yml
|
||||
- yamllint -d relaxed infra/traefik/traefik.yml
|
||||
when:
|
||||
- path: "traefik/traefik.yml"
|
||||
- path: "infra/traefik/traefik.yml"
|
||||
|
||||
- name: validate-traefik-template
|
||||
image: traefik:v3.3
|
||||
@@ -648,30 +648,30 @@ steps:
|
||||
commands:
|
||||
- |
|
||||
timeout 5 traefik \\
|
||||
--providers.file.directory=traefik/dynamic \\
|
||||
--providers.file.directory=infra/traefik/dynamic \\
|
||||
--api.dashboard=false \\
|
||||
--log.level=DEBUG 2>&1 | head -80 || true
|
||||
- echo "Traefik template syntax OK"
|
||||
when:
|
||||
- path: "traefik/dynamic/**"
|
||||
- path: "infra/traefik/dynamic/**"
|
||||
|
||||
- name: validate-loki
|
||||
image: cytopia/yamllint:latest
|
||||
commands:
|
||||
- yamllint -d relaxed loki/
|
||||
- yamllint -d relaxed infra/loki/
|
||||
when:
|
||||
- path: "loki/**"
|
||||
- path: "infra/loki/**"
|
||||
|
||||
- name: validate-trino
|
||||
image: alpine:3
|
||||
commands:
|
||||
- |
|
||||
OK=true
|
||||
for f in trino/etc/config.properties trino/etc/node.properties trino/etc/jvm.config; do
|
||||
for f in infra/trino/etc/config.properties infra/trino/etc/node.properties infra/trino/etc/jvm.config; do
|
||||
if [ ! -f "$f" ]; then echo "MISSING: $f"; OK=false; fi
|
||||
done
|
||||
ls trino/etc/catalog/*.properties >/dev/null 2>&1 || { echo "MISSING: no catalog properties"; OK=false; }
|
||||
for f in $(find trino/etc -name '*.properties'); do
|
||||
ls infra/trino/etc/catalog/*.properties >/dev/null 2>&1 || { echo "MISSING: no catalog properties"; OK=false; }
|
||||
for f in $(find infra/trino/etc -name '*.properties'); do
|
||||
while IFS= read -r line; do
|
||||
case "$line" in ''|'#'*|'!'*) continue ;; esac
|
||||
echo "$line" | grep -q '=' || { echo "BAD LINE in $f: $line"; OK=false; }
|
||||
@@ -680,7 +680,7 @@ steps:
|
||||
$OK && echo "Trino config validation passed"
|
||||
$OK
|
||||
when:
|
||||
- path: "trino/etc/**"
|
||||
- path: "infra/trino/etc/**"
|
||||
""")
|
||||
|
||||
parts.append(self._failure_reporter)
|
||||
|
||||
@@ -20,8 +20,8 @@ from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
CERT_DIR = REPO_ROOT / "traefik" / "certs"
|
||||
COREDNS_DIR = REPO_ROOT / "coredns"
|
||||
CERT_DIR = REPO_ROOT / "infra" / "traefik" / "certs"
|
||||
COREDNS_DIR = REPO_ROOT / "infra" / "coredns"
|
||||
DOCKER_CERTS_DIR = CERT_DIR / "docker-certs.d"
|
||||
|
||||
CA_KEY = CERT_DIR / "ca.key"
|
||||
|
||||
@@ -75,7 +75,7 @@ def gen_coredns() -> dict[str, str]:
|
||||
cache 300
|
||||
}}
|
||||
"""
|
||||
return {"coredns/hosts": hosts, "coredns/Corefile": corefile}
|
||||
return {"infra/coredns/hosts": hosts, "infra/coredns/Corefile": corefile}
|
||||
|
||||
|
||||
# ── Backend dispatch ─────────────────────────────────────────────
|
||||
@@ -177,6 +177,37 @@ def check_all(files: dict[str, str]) -> list[str]:
|
||||
return diffs
|
||||
|
||||
|
||||
def check_modules() -> list[str]:
|
||||
"""Verify module-name and optional-dependencies stay in sync."""
|
||||
import tomllib
|
||||
|
||||
pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text())
|
||||
module_names = set(
|
||||
pyproject.get("tool", {})
|
||||
.get("uv", {})
|
||||
.get("build-backend", {})
|
||||
.get("module-name", [])
|
||||
)
|
||||
opt_deps = set(pyproject.get("project", {}).get("optional-dependencies", {}).keys())
|
||||
# Groups that are aggregates or cloud providers, not Python modules
|
||||
meta_groups = {"all", "lake", "dev", "aws", "gcp", "azure"}
|
||||
|
||||
errors = []
|
||||
for mod in sorted(module_names):
|
||||
if mod not in opt_deps and mod not in meta_groups:
|
||||
errors.append(
|
||||
f"module '{mod}' in [tool.uv.build-backend].module-name "
|
||||
f"has no matching [project.optional-dependencies] group"
|
||||
)
|
||||
for group in sorted(opt_deps - meta_groups):
|
||||
if group not in module_names:
|
||||
errors.append(
|
||||
f"optional-dep group '{group}' has no matching "
|
||||
f"module in [tool.uv.build-backend].module-name"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate derived config from stack.toml [ci] backend"
|
||||
@@ -200,13 +231,21 @@ def main() -> int:
|
||||
|
||||
if args.check:
|
||||
diffs = check_all(files)
|
||||
mod_errors = check_modules()
|
||||
all_errors = diffs + mod_errors
|
||||
if all_errors:
|
||||
if diffs:
|
||||
print("Files out of date:")
|
||||
for d in diffs:
|
||||
print(f" {d}")
|
||||
if mod_errors:
|
||||
print("Module/dependency sync errors:")
|
||||
for e in mod_errors:
|
||||
print(f" {e}")
|
||||
print("\nRun: uv run python dev/scripts/gen_config.py")
|
||||
return 1
|
||||
print("All generated files are up-to-date.")
|
||||
print("Module names and optional-dependencies are in sync.")
|
||||
return 0
|
||||
|
||||
write_all(files)
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
# Usage: ./dev/scripts/install_certs.sh
|
||||
|
||||
set -uo pipefail
|
||||
CERT="$(cd "$(dirname "$0")/../.." && pwd)/traefik/certs/ca.crt"
|
||||
CERT="$(cd "$(dirname "$0")/../.." && pwd)/infra/traefik/certs/ca.crt"
|
||||
CERT_NAME="Homelab CA"
|
||||
|
||||
if [ ! -f "$CERT" ]; then
|
||||
|
||||
112
pyproject.toml
@@ -1,32 +1,109 @@
|
||||
[project]
|
||||
name = "stack"
|
||||
version = "0.1.0"
|
||||
description = "Add your description here"
|
||||
description = "Healthcare data platform — ACO analytics, CMS data, BCDA FHIR, PFS equations"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"databricks-cli>=0.18.0",
|
||||
"databricks-sdk>=0.85.0",
|
||||
"duckdb>=1.0.0",
|
||||
"fastapi>=0.135.1",
|
||||
"fastexcel>=0.19.0",
|
||||
"fsspec>=2024.1.0",
|
||||
"httpx>=0.28.1",
|
||||
"narwhals>=2.17.0",
|
||||
"pyarrow>=23.0.0",
|
||||
"cryptography>=46.0.5",
|
||||
"pyasn1>=0.6.3",
|
||||
"pyjwt>=2.12.0",
|
||||
"sqlglot>=26.0.0",
|
||||
"typer>=0.24.1",
|
||||
"uvicorn>=0.41.0",
|
||||
"pydantic>=2.0.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
stack = "cli:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
conf = [
|
||||
"pydantic>=2.0.0",
|
||||
"httpx>=0.28.1",
|
||||
"duckdb>=1.0.0",
|
||||
]
|
||||
aco = [
|
||||
"stack[conf]",
|
||||
"narwhals>=2.17.0",
|
||||
"sqlglot>=26.0.0",
|
||||
"pyarrow>=23.0.0",
|
||||
"duckdb>=1.0.0",
|
||||
]
|
||||
api = [
|
||||
"stack[conf]",
|
||||
"fastapi>=0.135.1",
|
||||
"uvicorn>=0.41.0",
|
||||
"httpx>=0.28.1",
|
||||
"pyjwt>=2.12.0",
|
||||
"cryptography>=46.0.5",
|
||||
]
|
||||
bcda = [
|
||||
"stack[conf]",
|
||||
"httpx>=0.28.1",
|
||||
"fsspec>=2024.1.0",
|
||||
"narwhals>=2.17.0",
|
||||
"pyarrow>=23.0.0",
|
||||
]
|
||||
bib = [
|
||||
"stack[conf]",
|
||||
"pydantic>=2.0.0",
|
||||
]
|
||||
bls = []
|
||||
ccw = [
|
||||
"pydantic>=2.0.0",
|
||||
]
|
||||
cli = [
|
||||
"stack[aco]",
|
||||
"stack[api]",
|
||||
"stack[bib]",
|
||||
"typer>=0.24.1",
|
||||
"uvicorn>=0.41.0",
|
||||
]
|
||||
cms = [
|
||||
"narwhals>=2.17.0",
|
||||
"pydantic>=2.0.0",
|
||||
]
|
||||
pfs = [
|
||||
"stack[conf]",
|
||||
"stack[rex]",
|
||||
"duckdb>=1.0.0",
|
||||
"narwhals>=2.17.0",
|
||||
]
|
||||
rex = [
|
||||
"stack[conf]",
|
||||
"narwhals>=2.17.0",
|
||||
"pyarrow>=23.0.0",
|
||||
"fsspec>=2024.1.0",
|
||||
]
|
||||
lake = [
|
||||
"stack[aco]",
|
||||
"databricks-cli>=0.18.0",
|
||||
"databricks-sdk>=0.85.0",
|
||||
]
|
||||
aws = [
|
||||
"boto3>=1.35.0",
|
||||
"s3fs>=2026.2.0",
|
||||
]
|
||||
gcp = [
|
||||
"google-cloud-storage>=2.18.0",
|
||||
"gcsfs>=2024.1.0",
|
||||
]
|
||||
azure = [
|
||||
"azure-storage-blob>=12.23.0",
|
||||
"adlfs>=2024.1.0",
|
||||
]
|
||||
all = [
|
||||
"stack[aco]",
|
||||
"stack[api]",
|
||||
"stack[bcda]",
|
||||
"stack[bib]",
|
||||
"stack[bls]",
|
||||
"stack[ccw]",
|
||||
"stack[cli]",
|
||||
"stack[cms]",
|
||||
"stack[pfs]",
|
||||
"stack[rex]",
|
||||
"stack[lake]",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"stack[all]",
|
||||
"coverage>=7.13.4",
|
||||
"dbt-core==1.10.15",
|
||||
"dbt-duckdb>=1.10,<1.11",
|
||||
@@ -42,6 +119,7 @@ dev = [
|
||||
"obstore>=0.9.2",
|
||||
"s3fs>=2026.2.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["uv_build>=0.7,<1"]
|
||||
build-backend = "uv_build"
|
||||
@@ -59,4 +137,4 @@ testpaths = ["tests"]
|
||||
[tool.uv.build-backend]
|
||||
module-name = ["aco", "api", "bcda", "bib", "bls", "ccw", "cli", "cms", "conf", "pfs", "rex"]
|
||||
namespace = true
|
||||
source-exclude = ["compose.yml","grafana/**","nginx/**","prometheus/**","rustfs/**","data/**", "loki/**", "notebooks/**", "traefik/**", "woodpecker/**", "gitea/**", "polaris/**", "trino/**", "zotero/**", "tuva/**"]
|
||||
source-exclude = ["compose.yml","infra/**","data/**","notebooks/**","zotero/**","tuva/**","assets/**","docs/**","dev/**","bundle/**"]
|
||||
|
||||