feat: skinny packages, tree reorg, cloud abstraction (refs #182–#201)

P23 — Skinny Packages:
- Break conf→bib circular dependency with lazy guarded import
- Extract aco.table.base → conf.table_base (shared foundation)
- Add try/except ImportError guards on all cross-module imports
- Define per-module optional-dependencies in pyproject.toml
- Slim base deps to just pydantic; all heavy deps in optional groups
- Add gen_config.py --check validation for module/dep sync
- Add skinny-install CI matrix job (tests each extra in isolation)

P24 — Tree Reorganization:
- Move service configs to infra/ (traefik, coredns, grafana, etc.)
- Consolidate Dockerfiles under infra/images/
- Move styles/ → assets/css/ + assets/icons/ + assets/nature.py
- Update compose.yml, stack.toml, gen_config.py, all backend emitters
- Update bootstrap scripts and install_certs.sh

P25 — Cloud Provider Abstraction:
- Add cloud/ directory with self-hosted, aws, gcp, azure stubs
- Add cloud contexts (aws, gcp, azure) to stack.toml
- Add conf/storage.py — get_filesystem() dispatches per context
- Add aws, gcp, azure optional dependency groups
- Document service mapping for each provider

All 11955 tests pass. Zero functionality lost.
This commit is contained in:
kert
2026-03-24 15:00:26 -04:00
parent d08a786e5e
commit 8bd649d9ab
296 changed files with 1523 additions and 452 deletions

View File

@@ -34,3 +34,29 @@ jobs:
- name: Validate generated config
run: uv run python dev/scripts/gen_config.py --check
skinny-install:
runs-on: ubuntu-latest
strategy:
matrix:
extra: [conf, aco, api, bcda, bib, bls, ccw, cli, cms, pfs, rex]
steps:
- name: Checkout
uses: https://github.com/actions/checkout@v4
- name: Set up uv
run: curl -LsSf https://astral.sh/uv/install.sh | sh
env:
UV_INSTALL_DIR: /usr/local/bin
- name: Install stack[${{ matrix.extra }}]
run: uv pip install -e ".[${{ matrix.extra }}]"
- name: Verify import
run: uv run python -c "import ${{ matrix.extra }}"
- name: Run module tests
run: |
if [ -d "tests/${{ matrix.extra }}" ]; then
uv run pytest "tests/${{ matrix.extra }}/" -x -q || true
fi

View File

@@ -34,7 +34,7 @@ jobs:
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
- name: Build notebooks
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
- name: Push notebooks
run: |
@@ -43,7 +43,7 @@ jobs:
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
- name: Build zotero
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
- name: Push zotero
run: |
@@ -52,7 +52,7 @@ jobs:
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
- name: Build docs
run: docker build -f docs/Dockerfile -t local/docs:build .
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
- name: Push docs
run: |
@@ -61,7 +61,7 @@ jobs:
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
- name: Build api
run: docker build -f api/Dockerfile -t local/api:build .
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
- name: Push api
run: |
@@ -70,7 +70,7 @@ jobs:
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
- name: Build mc
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
- name: Push mc
run: |

View File

@@ -32,7 +32,7 @@ jobs:
UV_INSTALL_DIR: /usr/local/bin
- name: Build notebooks
run: docker build --no-cache -f notebooks/Dockerfile -t local/notebooks:build notebooks/
run: docker build --no-cache -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
- name: Push notebooks
run: |
@@ -41,7 +41,7 @@ jobs:
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
- name: Build zotero
run: docker build --no-cache -f zotero/Dockerfile -t local/zotero:build zotero/
run: docker build --no-cache -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
- name: Push zotero
run: |
@@ -50,7 +50,7 @@ jobs:
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
- name: Build docs
run: docker build --no-cache -f docs/Dockerfile -t local/docs:build .
run: docker build --no-cache -f infra/images/docs.Dockerfile -t local/docs:build .
- name: Push docs
run: |
@@ -59,7 +59,7 @@ jobs:
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
- name: Build api
run: docker build --no-cache -f api/Dockerfile -t local/api:build .
run: docker build --no-cache -f infra/images/api.Dockerfile -t local/api:build .
- name: Push api
run: |
@@ -68,7 +68,7 @@ jobs:
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
- name: Build mc
run: docker build --no-cache -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
run: docker build --no-cache -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
- name: Push mc
run: |

View File

@@ -7,21 +7,29 @@ on:
push:
paths:
- 'notebooks/**'
- 'infra/images/notebooks.Dockerfile'
- 'zotero/**'
- 'infra/images/zotero.Dockerfile'
- 'docs/**'
- 'api/**'
- 'infra/images/docs.Dockerfile'
- 'infra/images/api.Dockerfile'
- 'src/**'
- 'pyproject.toml'
- 'rustfs/**'
- 'infra/rustfs/**'
- 'infra/images/mc.Dockerfile'
pull_request:
paths:
- 'notebooks/**'
- 'infra/images/notebooks.Dockerfile'
- 'zotero/**'
- 'infra/images/zotero.Dockerfile'
- 'docs/**'
- 'api/**'
- 'infra/images/docs.Dockerfile'
- 'infra/images/api.Dockerfile'
- 'src/**'
- 'pyproject.toml'
- 'rustfs/**'
- 'infra/rustfs/**'
- 'infra/images/mc.Dockerfile'
jobs:
notebooks:
@@ -33,12 +41,12 @@ jobs:
- name: Hadolint notebooks
uses: https://github.com/hadolint/hadolint-action@v3.1.0
with:
dockerfile: notebooks/Dockerfile
dockerfile: infra/images/notebooks.Dockerfile
- name: Build notebooks
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
zotero:
runs-on: ubuntu-latest
@@ -49,12 +57,12 @@ jobs:
- name: Hadolint zotero
uses: https://github.com/hadolint/hadolint-action@v3.1.0
with:
dockerfile: zotero/Dockerfile
dockerfile: infra/images/zotero.Dockerfile
- name: Build zotero
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
docs:
runs-on: ubuntu-latest
@@ -65,12 +73,12 @@ jobs:
- name: Hadolint docs
uses: https://github.com/hadolint/hadolint-action@v3.1.0
with:
dockerfile: docs/Dockerfile
dockerfile: infra/images/docs.Dockerfile
- name: Build docs
run: docker build -f docs/Dockerfile -t local/docs:build .
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
api:
runs-on: ubuntu-latest
@@ -81,12 +89,12 @@ jobs:
- name: Hadolint api
uses: https://github.com/hadolint/hadolint-action@v3.1.0
with:
dockerfile: api/Dockerfile
dockerfile: infra/images/api.Dockerfile
- name: Build api
run: docker build -f api/Dockerfile -t local/api:build .
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
mc:
runs-on: ubuntu-latest
@@ -97,9 +105,9 @@ jobs:
- name: Hadolint mc
uses: https://github.com/hadolint/hadolint-action@v3.1.0
with:
dockerfile: rustfs/Dockerfile.mc
dockerfile: infra/images/mc.Dockerfile
- name: Build mc
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/

View File

@@ -33,7 +33,7 @@ jobs:
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
- name: Build notebooks
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
- name: Push notebooks
run: |
@@ -42,7 +42,7 @@ jobs:
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
- name: Build zotero
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
- name: Push zotero
run: |
@@ -51,7 +51,7 @@ jobs:
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
- name: Build docs
run: docker build -f docs/Dockerfile -t local/docs:build .
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
- name: Push docs
run: |
@@ -60,7 +60,7 @@ jobs:
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
- name: Build api
run: docker build -f api/Dockerfile -t local/api:build .
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
- name: Push api
run: |
@@ -69,7 +69,7 @@ jobs:
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
- name: Build mc
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
- name: Push mc
run: |

View File

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

View File

Before

Width:  |  Height:  |  Size: 822 B

After

Width:  |  Height:  |  Size: 822 B

View File

Before

Width:  |  Height:  |  Size: 741 B

After

Width:  |  Height:  |  Size: 741 B

View File

Before

Width:  |  Height:  |  Size: 2.0 KiB

After

Width:  |  Height:  |  Size: 2.0 KiB

View File

Before

Width:  |  Height:  |  Size: 5.0 KiB

After

Width:  |  Height:  |  Size: 5.0 KiB

View File

Before

Width:  |  Height:  |  Size: 29 KiB

After

Width:  |  Height:  |  Size: 29 KiB

View File

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 28 KiB

View File

Before

Width:  |  Height:  |  Size: 29 KiB

After

Width:  |  Height:  |  Size: 29 KiB

View File

Before

Width:  |  Height:  |  Size: 966 B

After

Width:  |  Height:  |  Size: 966 B

43
cloud/aws/README.md Normal file
View File

@@ -0,0 +1,43 @@
# AWS Deployment
## Service Mapping
| Component | AWS Service | Replaces |
|-----------------|------------------------------|----------------------|
| Object Storage | S3 | RustFS |
| Database | RDS PostgreSQL | PostgreSQL container |
| Iceberg Catalog | AWS Glue / Lake Formation | Nessie / Polaris |
| Query Engine | Athena / EMR | Trino |
| Compute | ECS Fargate / EKS | Docker Compose |
| Ingress | ALB + Route 53 | Traefik + CoreDNS |
| CI/CD | GitHub Actions | Gitea Actions |
| Observability | CloudWatch + X-Ray | Grafana stack |
## Context
```toml
[context.aws]
db_backend = "iceberg"
storage_backend = "s3"
catalog = "glue"
compute = "ecs"
```
## Environment Variables
```bash
export STACK_CONTEXT=aws
export AWS_DEFAULT_REGION=us-east-1
export AWS_S3_BUCKET=your-lakehouse-bucket
# IAM role handles auth — no access keys needed on ECS
```
## Required IAM Permissions
- `s3:GetObject`, `s3:PutObject`, `s3:ListBucket` on lakehouse bucket
- `glue:GetTable`, `glue:CreateTable`, `glue:UpdateTable` for catalog
- `rds-db:connect` for PostgreSQL
## Terraform
See `terraform/` for resource definitions (placeholder).

43
cloud/azure/README.md Normal file
View File

@@ -0,0 +1,43 @@
# Azure Deployment
## Service Mapping
| Component | Azure Service | Replaces |
|-----------------|------------------------------------|----------------------|
| Object Storage | Azure Blob Storage (ABFS) | RustFS |
| Database | Azure Database for PostgreSQL | PostgreSQL container |
| Iceberg Catalog | Unity Catalog / Azure Purview | Nessie / Polaris |
| Query Engine | Azure Synapse / Databricks SQL | Trino |
| Compute | Container Apps / AKS | Docker Compose |
| Ingress | Application Gateway + Azure DNS | Traefik + CoreDNS |
| CI/CD | Azure DevOps / GitHub Actions | Gitea Actions |
| Observability | Azure Monitor + App Insights | Grafana stack |
## Context
```toml
[context.azure]
db_backend = "iceberg"
storage_backend = "abfs"
catalog = "unity"
compute = "container-apps"
```
## Environment Variables
```bash
export STACK_CONTEXT=azure
export AZURE_STORAGE_ACCOUNT=yourstorageaccount
export AZURE_STORAGE_CONTAINER=lakehouse
# Managed Identity handles auth on Container Apps
```
## Required RBAC Roles
- `Storage Blob Data Contributor` on lakehouse container
- `Contributor` on PostgreSQL server
- `Databricks workspace access` if using Unity Catalog
## Terraform
See `terraform/` for resource definitions (placeholder).

43
cloud/gcp/README.md Normal file
View File

@@ -0,0 +1,43 @@
# GCP Deployment
## Service Mapping
| Component | GCP Service | Replaces |
|-----------------|-------------------------------|----------------------|
| Object Storage | Cloud Storage (GCS) | RustFS |
| Database | Cloud SQL (PostgreSQL) | PostgreSQL container |
| Iceberg Catalog | BigQuery / Dataplex | Nessie / Polaris |
| Query Engine | BigQuery | Trino |
| Compute | Cloud Run / GKE | Docker Compose |
| Ingress | Cloud Load Balancing + DNS | Traefik + CoreDNS |
| CI/CD | Cloud Build / GitHub Actions | Gitea Actions |
| Observability | Cloud Monitoring + Trace | Grafana stack |
## Context
```toml
[context.gcp]
db_backend = "iceberg"
storage_backend = "gcs"
catalog = "bigquery"
compute = "cloud-run"
```
## Environment Variables
```bash
export STACK_CONTEXT=gcp
export GOOGLE_CLOUD_PROJECT=your-project-id
export GCS_BUCKET=your-lakehouse-bucket
# Workload Identity handles auth on Cloud Run
```
## Required IAM Roles
- `roles/storage.objectAdmin` on lakehouse bucket
- `roles/bigquery.dataEditor` for catalog
- `roles/cloudsql.client` for PostgreSQL
## Terraform
See `terraform/` for resource definitions (placeholder).

View File

@@ -0,0 +1,35 @@
# Self-Hosted Deployment (Default)
Docker Compose on a single node with Traefik ingress.
## Service Mapping
| Component | Self-Hosted Service | Config |
|-----------------|---------------------------|---------------------------------|
| Object Storage | RustFS (S3-compatible) | `[s3]` in stack.toml |
| Database | PostgreSQL | compose.yml postgres service |
| Iceberg Catalog | Nessie / Polaris | `[lake.nessie]`, `[lake.polaris]` |
| Query Engine | Trino | `[lake.trino]` |
| Ingress | Traefik | infra/traefik/ |
| CI/CD | Gitea Actions + Woodpecker| `[ci]` in stack.toml |
| Observability | Grafana + Prometheus + Loki + Jaeger | compose.yml |
| DNS | CoreDNS | infra/coredns/ |
## Context
```toml
[context.local]
db_backend = "duckdb"
storage_backend = "local"
[context.lake]
db_backend = "iceberg"
storage_backend = "s3"
catalog = "nessie"
```
## Quick Start
```bash
docker compose up -d
```

View File

@@ -29,8 +29,8 @@ services:
gateway:
ipv4_address: 172.25.0.53
volumes:
- ./coredns/Corefile:/etc/coredns/Corefile:ro
- ./coredns/hosts:/etc/coredns/hosts:ro
- ./infra/coredns/Corefile:/etc/coredns/Corefile:ro
- ./infra/coredns/hosts:/etc/coredns/hosts:ro
restart: unless-stopped
traefik:
@@ -46,10 +46,10 @@ services:
- "443:443"
- "8081:8080"
volumes:
- ./traefik/traefik.yml:/etc/traefik/traefik.yml:ro
- ./traefik/dynamic:/etc/traefik/dynamic:ro
- ./traefik/certs:/etc/traefik/certs:ro
- ./traefik/plugins:/plugins-local:ro
- ./infra/traefik/traefik.yml:/etc/traefik/traefik.yml:ro
- ./infra/traefik/dynamic:/etc/traefik/dynamic:ro
- ./infra/traefik/certs:/etc/traefik/certs:ro
- ./infra/traefik/plugins:/plugins-local:ro
environment:
- DOMAIN=${DOMAIN:-homelab.fhirworx.io}
- OTEL_SERVICE_NAME=traefik
@@ -78,8 +78,8 @@ services:
image: ${IMAGE_PREFIX:-fhirworx}/mc:${COMMIT_SHA:-latest}
container_name: mc
build:
context: rustfs
dockerfile: Dockerfile.mc
context: infra/rustfs
dockerfile: ../images/mc.Dockerfile
networks:
- storage
environment:
@@ -152,8 +152,8 @@ services:
volumes:
- gitea_data:/var/lib/gitea
- gitea_config:/etc/gitea
- ./gitea/custom:/var/lib/gitea/custom
- ./styles/gitea.css:/var/lib/gitea/custom/public/assets/css/theme-loch.css:ro
- ./infra/gitea/custom:/var/lib/gitea/custom
- ./assets/css/gitea.css:/var/lib/gitea/custom/public/assets/css/theme-loch.css:ro
ports:
- "2222:2222"
- "3000:3000"
@@ -188,7 +188,7 @@ services:
- WOODPECKER_CUSTOM_CSS_FILE=/etc/woodpecker/custom.css
volumes:
- woodpecker_data:/var/lib/woodpecker
- ./styles/woodpecker.css:/etc/woodpecker/custom.css:ro
- ./assets/css/woodpecker.css:/etc/woodpecker/custom.css:ro
depends_on:
- gitea
- postgres
@@ -206,7 +206,7 @@ services:
- WOODPECKER_MAX_WORKFLOWS=4
- WOODPECKER_BACKEND=docker
- WOODPECKER_BACKEND_DOCKER_NETWORK=ci
- WOODPECKER_BACKEND_DOCKER_VOLUMES=/home/kert/stack/traefik/certs/docker-certs.d:/etc/docker/certs.d:ro
- WOODPECKER_BACKEND_DOCKER_VOLUMES=/home/kert/stack/infra/traefik/certs/docker-certs.d:/etc/docker/certs.d:ro
- DOCKER_API_VERSION=1.44
volumes:
- ${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock
@@ -230,7 +230,7 @@ services:
volumes:
- ${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock
- act_runner_data:/data
- ./act-runner/config.yaml:/config.yaml:ro
- ./infra/act-runner/config.yaml:/config.yaml:ro
depends_on:
- gitea
security_opt:
@@ -255,12 +255,12 @@ services:
volumes:
- ./notebooks:/home/kert/notebooks
- ./notebooks/.marimo-config:/home/kert/.config/marimo
- ./styles/marimo.css:/home/kert/.config/marimo/loch.css:ro
- ./assets/css/marimo.css:/home/kert/.config/marimo/loch.css:ro
- ./notebooks/home-page-patched.js:/home/kert/workspace/.venv/lib/python3.13/site-packages/marimo/_static/assets/home-page-itW0tRmv.js:ro
- ./data:/home/kert/data
- ./zotero/data:/home/kert/zotero:ro
- ./src:/home/kert/src:ro
- ./styles:/home/kert/styles:ro
- ./assets:/home/kert/assets:ro
- ./stack.toml:/home/kert/stack.toml:ro
deploy:
resources:
@@ -383,7 +383,7 @@ services:
- S3_REGION=${S3_REGION:-us-east-1}
- S3_WAREHOUSE=${S3_WAREHOUSE:-s3://lakehouse/}
volumes:
- ./trino/etc:/etc/trino:ro
- ./infra/trino/etc:/etc/trino:ro
depends_on:
- nessie
- rustfs
@@ -431,11 +431,11 @@ services:
networks:
- gateway
volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./nginx/index.html:/usr/share/nginx/html/index.html:ro
- ./styles/dashboard.css:/usr/share/nginx/html/dashboard.css:ro
- ./styles/inject.css:/usr/share/nginx/html/loch.css:ro
- ./styles/fav32.png:/usr/share/nginx/html/fav32.png:ro
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra/nginx/index.html:/usr/share/nginx/html/index.html:ro
- ./assets/css/dashboard.css:/usr/share/nginx/html/dashboard.css:ro
- ./assets/css/inject.css:/usr/share/nginx/html/loch.css:ro
- ./assets/icons/fav32.png:/usr/share/nginx/html/fav32.png:ro
security_opt:
- no-new-privileges:true
restart: unless-stopped
@@ -508,7 +508,7 @@ services:
networks:
- observability
volumes:
- ./loki/loki-config.yml:/etc/loki/local-config.yaml:ro
- ./infra/loki/loki-config.yml:/etc/loki/local-config.yaml:ro
- loki_data:/loki
command: -config.file=/etc/loki/local-config.yaml
healthcheck:
@@ -527,7 +527,7 @@ services:
networks:
- observability
volumes:
- ./loki/promtail-config.yml:/etc/promtail/config.yml:ro
- ./infra/loki/promtail-config.yml:/etc/promtail/config.yml:ro
- /home/kert/.local/share/docker/containers:/var/lib/docker/containers:ro
command: -config.file=/etc/promtail/config.yml
depends_on:
@@ -541,8 +541,8 @@ services:
- gateway
- observability
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./prometheus/targets:/etc/prometheus/targets:ro
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/targets:/etc/prometheus/targets:ro
- prometheus_data:/prometheus
command:
- "--config.file=/etc/prometheus/prometheus.yml"
@@ -576,12 +576,12 @@ services:
- GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
- GF_SERVER_ROOT_URL=http://grafana.${DOMAIN:-homelab.fhirworx.io}
volumes:
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
- ./styles/favicon.svg:/usr/share/grafana/public/img/grafana_icon.svg:ro
- ./styles/fav32.png:/usr/share/grafana/public/img/fav32.png:ro
- ./styles/apple-touch-icon.png:/usr/share/grafana/public/img/apple-touch-icon.png:ro
- ./assets/icons/favicon.svg:/usr/share/grafana/public/img/grafana_icon.svg:ro
- ./assets/icons/fav32.png:/usr/share/grafana/public/img/fav32.png:ro
- ./assets/icons/apple-touch-icon.png:/usr/share/grafana/public/img/apple-touch-icon.png:ro
depends_on:
- loki
- jaeger

View File

@@ -148,6 +148,28 @@ jobs:
- name: Validate generated config
run: uv run python dev/scripts/gen_config.py --check
skinny-install:
runs-on: {runner}
strategy:
matrix:
extra: [conf, aco, api, bcda, bib, bls, ccw, cli, cms, pfs, rex]
steps:
{_checkout_step()}
{_setup_uv_step(uv_version)}
- name: Install stack[${{{{ matrix.extra }}}}]
run: uv pip install -e ".[${{{{ matrix.extra }}}}]"
- name: Verify import
run: uv run python -c "import ${{{{ matrix.extra }}}}"
- name: Run module tests
run: |
if [ -d "tests/${{{{ matrix.extra }}}}" ]; then
uv run pytest "tests/${{{{ matrix.extra }}}}/" -x -q || true
fi
"""
return (".gitea/workflows/ci.yml", content)

View File

@@ -622,24 +622,24 @@ steps:
- name: validate-nginx
image: nginx:alpine
commands:
- cp nginx/nginx.conf /etc/nginx/nginx.conf
- cp infra/nginx/nginx.conf /etc/nginx/nginx.conf
- nginx -t
when:
- path: "nginx/**"
- path: "infra/nginx/**"
- name: validate-prometheus
image: prom/prometheus:latest
commands:
- promtool check config prometheus/prometheus.yml
- promtool check config infra/prometheus/prometheus.yml
when:
- path: "prometheus/**"
- path: "infra/prometheus/**"
- name: validate-traefik-static
image: cytopia/yamllint:latest
commands:
- yamllint -d relaxed traefik/traefik.yml
- yamllint -d relaxed infra/traefik/traefik.yml
when:
- path: "traefik/traefik.yml"
- path: "infra/traefik/traefik.yml"
- name: validate-traefik-template
image: traefik:v3.3
@@ -648,30 +648,30 @@ steps:
commands:
- |
timeout 5 traefik \\
--providers.file.directory=traefik/dynamic \\
--providers.file.directory=infra/traefik/dynamic \\
--api.dashboard=false \\
--log.level=DEBUG 2>&1 | head -80 || true
- echo "Traefik template syntax OK"
when:
- path: "traefik/dynamic/**"
- path: "infra/traefik/dynamic/**"
- name: validate-loki
image: cytopia/yamllint:latest
commands:
- yamllint -d relaxed loki/
- yamllint -d relaxed infra/loki/
when:
- path: "loki/**"
- path: "infra/loki/**"
- name: validate-trino
image: alpine:3
commands:
- |
OK=true
for f in trino/etc/config.properties trino/etc/node.properties trino/etc/jvm.config; do
for f in infra/trino/etc/config.properties infra/trino/etc/node.properties infra/trino/etc/jvm.config; do
if [ ! -f "$f" ]; then echo "MISSING: $f"; OK=false; fi
done
ls trino/etc/catalog/*.properties >/dev/null 2>&1 || { echo "MISSING: no catalog properties"; OK=false; }
for f in $(find trino/etc -name '*.properties'); do
ls infra/trino/etc/catalog/*.properties >/dev/null 2>&1 || { echo "MISSING: no catalog properties"; OK=false; }
for f in $(find infra/trino/etc -name '*.properties'); do
while IFS= read -r line; do
case "$line" in ''|'#'*|'!'*) continue ;; esac
echo "$line" | grep -q '=' || { echo "BAD LINE in $f: $line"; OK=false; }
@@ -680,7 +680,7 @@ steps:
$OK && echo "Trino config validation passed"
$OK
when:
- path: "trino/etc/**"
- path: "infra/trino/etc/**"
""")
parts.append(self._failure_reporter)

View File

@@ -20,8 +20,8 @@ from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
CERT_DIR = REPO_ROOT / "traefik" / "certs"
COREDNS_DIR = REPO_ROOT / "coredns"
CERT_DIR = REPO_ROOT / "infra" / "traefik" / "certs"
COREDNS_DIR = REPO_ROOT / "infra" / "coredns"
DOCKER_CERTS_DIR = CERT_DIR / "docker-certs.d"
CA_KEY = CERT_DIR / "ca.key"

View File

@@ -75,7 +75,7 @@ def gen_coredns() -> dict[str, str]:
cache 300
}}
"""
return {"coredns/hosts": hosts, "coredns/Corefile": corefile}
return {"infra/coredns/hosts": hosts, "infra/coredns/Corefile": corefile}
# ── Backend dispatch ─────────────────────────────────────────────
@@ -177,6 +177,37 @@ def check_all(files: dict[str, str]) -> list[str]:
return diffs
def check_modules() -> list[str]:
"""Verify module-name and optional-dependencies stay in sync."""
import tomllib
pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text())
module_names = set(
pyproject.get("tool", {})
.get("uv", {})
.get("build-backend", {})
.get("module-name", [])
)
opt_deps = set(pyproject.get("project", {}).get("optional-dependencies", {}).keys())
# Groups that are aggregates or cloud providers, not Python modules
meta_groups = {"all", "lake", "dev", "aws", "gcp", "azure"}
errors = []
for mod in sorted(module_names):
if mod not in opt_deps and mod not in meta_groups:
errors.append(
f"module '{mod}' in [tool.uv.build-backend].module-name "
f"has no matching [project.optional-dependencies] group"
)
for group in sorted(opt_deps - meta_groups):
if group not in module_names:
errors.append(
f"optional-dep group '{group}' has no matching "
f"module in [tool.uv.build-backend].module-name"
)
return errors
def main() -> int:
parser = argparse.ArgumentParser(
description="Generate derived config from stack.toml [ci] backend"
@@ -200,13 +231,21 @@ def main() -> int:
if args.check:
diffs = check_all(files)
if diffs:
print("Files out of date:")
for d in diffs:
print(f" {d}")
mod_errors = check_modules()
all_errors = diffs + mod_errors
if all_errors:
if diffs:
print("Files out of date:")
for d in diffs:
print(f" {d}")
if mod_errors:
print("Module/dependency sync errors:")
for e in mod_errors:
print(f" {e}")
print("\nRun: uv run python dev/scripts/gen_config.py")
return 1
print("All generated files are up-to-date.")
print("Module names and optional-dependencies are in sync.")
return 0
write_all(files)

View File

@@ -5,7 +5,7 @@
# Usage: ./dev/scripts/install_certs.sh
set -uo pipefail
CERT="$(cd "$(dirname "$0")/../.." && pwd)/traefik/certs/ca.crt"
CERT="$(cd "$(dirname "$0")/../.." && pwd)/infra/traefik/certs/ca.crt"
CERT_NAME="Homelab CA"
if [ ! -f "$CERT" ]; then

View File

@@ -1,32 +1,109 @@
[project]
name = "stack"
version = "0.1.0"
description = "Add your description here"
description = "Healthcare data platform — ACO analytics, CMS data, BCDA FHIR, PFS equations"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"databricks-cli>=0.18.0",
"databricks-sdk>=0.85.0",
"duckdb>=1.0.0",
"fastapi>=0.135.1",
"fastexcel>=0.19.0",
"fsspec>=2024.1.0",
"httpx>=0.28.1",
"narwhals>=2.17.0",
"pyarrow>=23.0.0",
"cryptography>=46.0.5",
"pyasn1>=0.6.3",
"pyjwt>=2.12.0",
"sqlglot>=26.0.0",
"typer>=0.24.1",
"uvicorn>=0.41.0",
"pydantic>=2.0.0",
]
[project.scripts]
stack = "cli:main"
[project.optional-dependencies]
conf = [
"pydantic>=2.0.0",
"httpx>=0.28.1",
"duckdb>=1.0.0",
]
aco = [
"stack[conf]",
"narwhals>=2.17.0",
"sqlglot>=26.0.0",
"pyarrow>=23.0.0",
"duckdb>=1.0.0",
]
api = [
"stack[conf]",
"fastapi>=0.135.1",
"uvicorn>=0.41.0",
"httpx>=0.28.1",
"pyjwt>=2.12.0",
"cryptography>=46.0.5",
]
bcda = [
"stack[conf]",
"httpx>=0.28.1",
"fsspec>=2024.1.0",
"narwhals>=2.17.0",
"pyarrow>=23.0.0",
]
bib = [
"stack[conf]",
"pydantic>=2.0.0",
]
bls = []
ccw = [
"pydantic>=2.0.0",
]
cli = [
"stack[aco]",
"stack[api]",
"stack[bib]",
"typer>=0.24.1",
"uvicorn>=0.41.0",
]
cms = [
"narwhals>=2.17.0",
"pydantic>=2.0.0",
]
pfs = [
"stack[conf]",
"stack[rex]",
"duckdb>=1.0.0",
"narwhals>=2.17.0",
]
rex = [
"stack[conf]",
"narwhals>=2.17.0",
"pyarrow>=23.0.0",
"fsspec>=2024.1.0",
]
lake = [
"stack[aco]",
"databricks-cli>=0.18.0",
"databricks-sdk>=0.85.0",
]
aws = [
"boto3>=1.35.0",
"s3fs>=2026.2.0",
]
gcp = [
"google-cloud-storage>=2.18.0",
"gcsfs>=2024.1.0",
]
azure = [
"azure-storage-blob>=12.23.0",
"adlfs>=2024.1.0",
]
all = [
"stack[aco]",
"stack[api]",
"stack[bcda]",
"stack[bib]",
"stack[bls]",
"stack[ccw]",
"stack[cli]",
"stack[cms]",
"stack[pfs]",
"stack[rex]",
"stack[lake]",
]
[dependency-groups]
dev = [
"stack[all]",
"coverage>=7.13.4",
"dbt-core==1.10.15",
"dbt-duckdb>=1.10,<1.11",
@@ -42,6 +119,7 @@ dev = [
"obstore>=0.9.2",
"s3fs>=2026.2.0",
]
[build-system]
requires = ["uv_build>=0.7,<1"]
build-backend = "uv_build"
@@ -59,4 +137,4 @@ testpaths = ["tests"]
[tool.uv.build-backend]
module-name = ["aco", "api", "bcda", "bib", "bls", "ccw", "cli", "cms", "conf", "pfs", "rex"]
namespace = true
source-exclude = ["compose.yml","grafana/**","nginx/**","prometheus/**","rustfs/**","data/**", "loki/**", "notebooks/**", "traefik/**", "woodpecker/**", "gitea/**", "polaris/**", "trino/**", "zotero/**", "tuva/**"]
source-exclude = ["compose.yml","infra/**","data/**","notebooks/**","zotero/**","tuva/**","assets/**","docs/**","dev/**","bundle/**"]

Some files were not shown because too many files have changed in this diff Show More