feat: skinny packages, tree reorg, cloud abstraction (refs #182–#201)
P23 — Skinny Packages: - Break conf→bib circular dependency with lazy guarded import - Extract aco.table.base → conf.table_base (shared foundation) - Add try/except ImportError guards on all cross-module imports - Define per-module optional-dependencies in pyproject.toml - Slim base deps to just pydantic; all heavy deps in optional groups - Add gen_config.py --check validation for module/dep sync - Add skinny-install CI matrix job (tests each extra in isolation) P24 — Tree Reorganization: - Move service configs to infra/ (traefik, coredns, grafana, etc.) - Consolidate Dockerfiles under infra/images/ - Move styles/ → assets/css/ + assets/icons/ + assets/nature.py - Update compose.yml, stack.toml, gen_config.py, all backend emitters - Update bootstrap scripts and install_certs.sh P25 — Cloud Provider Abstraction: - Add cloud/ directory with self-hosted, aws, gcp, azure stubs - Add cloud contexts (aws, gcp, azure) to stack.toml - Add conf/storage.py — get_filesystem() dispatches per context - Add aws, gcp, azure optional dependency groups - Document service mapping for each provider All 11955 tests pass. Zero functionality lost.
@@ -34,3 +34,29 @@ jobs:
|
|||||||
|
|
||||||
- name: Validate generated config
|
- name: Validate generated config
|
||||||
run: uv run python dev/scripts/gen_config.py --check
|
run: uv run python dev/scripts/gen_config.py --check
|
||||||
|
|
||||||
|
skinny-install:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
extra: [conf, aco, api, bcda, bib, bls, ccw, cli, cms, pfs, rex]
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: https://github.com/actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up uv
|
||||||
|
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||||
|
env:
|
||||||
|
UV_INSTALL_DIR: /usr/local/bin
|
||||||
|
|
||||||
|
- name: Install stack[${{ matrix.extra }}]
|
||||||
|
run: uv pip install -e ".[${{ matrix.extra }}]"
|
||||||
|
|
||||||
|
- name: Verify import
|
||||||
|
run: uv run python -c "import ${{ matrix.extra }}"
|
||||||
|
|
||||||
|
- name: Run module tests
|
||||||
|
run: |
|
||||||
|
if [ -d "tests/${{ matrix.extra }}" ]; then
|
||||||
|
uv run pytest "tests/${{ matrix.extra }}/" -x -q || true
|
||||||
|
fi
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ jobs:
|
|||||||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
- name: Build notebooks
|
- name: Build notebooks
|
||||||
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||||
|
|
||||||
- name: Push notebooks
|
- name: Push notebooks
|
||||||
run: |
|
run: |
|
||||||
@@ -43,7 +43,7 @@ jobs:
|
|||||||
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
||||||
|
|
||||||
- name: Build zotero
|
- name: Build zotero
|
||||||
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
|
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
|
||||||
|
|
||||||
- name: Push zotero
|
- name: Push zotero
|
||||||
run: |
|
run: |
|
||||||
@@ -52,7 +52,7 @@ jobs:
|
|||||||
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
||||||
|
|
||||||
- name: Build docs
|
- name: Build docs
|
||||||
run: docker build -f docs/Dockerfile -t local/docs:build .
|
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
|
||||||
|
|
||||||
- name: Push docs
|
- name: Push docs
|
||||||
run: |
|
run: |
|
||||||
@@ -61,7 +61,7 @@ jobs:
|
|||||||
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
||||||
|
|
||||||
- name: Build api
|
- name: Build api
|
||||||
run: docker build -f api/Dockerfile -t local/api:build .
|
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
|
||||||
|
|
||||||
- name: Push api
|
- name: Push api
|
||||||
run: |
|
run: |
|
||||||
@@ -70,7 +70,7 @@ jobs:
|
|||||||
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
||||||
|
|
||||||
- name: Build mc
|
- name: Build mc
|
||||||
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||||
|
|
||||||
- name: Push mc
|
- name: Push mc
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ jobs:
|
|||||||
UV_INSTALL_DIR: /usr/local/bin
|
UV_INSTALL_DIR: /usr/local/bin
|
||||||
|
|
||||||
- name: Build notebooks
|
- name: Build notebooks
|
||||||
run: docker build --no-cache -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
run: docker build --no-cache -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||||
|
|
||||||
- name: Push notebooks
|
- name: Push notebooks
|
||||||
run: |
|
run: |
|
||||||
@@ -41,7 +41,7 @@ jobs:
|
|||||||
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
||||||
|
|
||||||
- name: Build zotero
|
- name: Build zotero
|
||||||
run: docker build --no-cache -f zotero/Dockerfile -t local/zotero:build zotero/
|
run: docker build --no-cache -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
|
||||||
|
|
||||||
- name: Push zotero
|
- name: Push zotero
|
||||||
run: |
|
run: |
|
||||||
@@ -50,7 +50,7 @@ jobs:
|
|||||||
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
||||||
|
|
||||||
- name: Build docs
|
- name: Build docs
|
||||||
run: docker build --no-cache -f docs/Dockerfile -t local/docs:build .
|
run: docker build --no-cache -f infra/images/docs.Dockerfile -t local/docs:build .
|
||||||
|
|
||||||
- name: Push docs
|
- name: Push docs
|
||||||
run: |
|
run: |
|
||||||
@@ -59,7 +59,7 @@ jobs:
|
|||||||
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
||||||
|
|
||||||
- name: Build api
|
- name: Build api
|
||||||
run: docker build --no-cache -f api/Dockerfile -t local/api:build .
|
run: docker build --no-cache -f infra/images/api.Dockerfile -t local/api:build .
|
||||||
|
|
||||||
- name: Push api
|
- name: Push api
|
||||||
run: |
|
run: |
|
||||||
@@ -68,7 +68,7 @@ jobs:
|
|||||||
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
||||||
|
|
||||||
- name: Build mc
|
- name: Build mc
|
||||||
run: docker build --no-cache -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
run: docker build --no-cache -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||||
|
|
||||||
- name: Push mc
|
- name: Push mc
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -7,21 +7,29 @@ on:
|
|||||||
push:
|
push:
|
||||||
paths:
|
paths:
|
||||||
- 'notebooks/**'
|
- 'notebooks/**'
|
||||||
|
- 'infra/images/notebooks.Dockerfile'
|
||||||
- 'zotero/**'
|
- 'zotero/**'
|
||||||
|
- 'infra/images/zotero.Dockerfile'
|
||||||
- 'docs/**'
|
- 'docs/**'
|
||||||
- 'api/**'
|
- 'infra/images/docs.Dockerfile'
|
||||||
|
- 'infra/images/api.Dockerfile'
|
||||||
- 'src/**'
|
- 'src/**'
|
||||||
- 'pyproject.toml'
|
- 'pyproject.toml'
|
||||||
- 'rustfs/**'
|
- 'infra/rustfs/**'
|
||||||
|
- 'infra/images/mc.Dockerfile'
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- 'notebooks/**'
|
- 'notebooks/**'
|
||||||
|
- 'infra/images/notebooks.Dockerfile'
|
||||||
- 'zotero/**'
|
- 'zotero/**'
|
||||||
|
- 'infra/images/zotero.Dockerfile'
|
||||||
- 'docs/**'
|
- 'docs/**'
|
||||||
- 'api/**'
|
- 'infra/images/docs.Dockerfile'
|
||||||
|
- 'infra/images/api.Dockerfile'
|
||||||
- 'src/**'
|
- 'src/**'
|
||||||
- 'pyproject.toml'
|
- 'pyproject.toml'
|
||||||
- 'rustfs/**'
|
- 'infra/rustfs/**'
|
||||||
|
- 'infra/images/mc.Dockerfile'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
notebooks:
|
notebooks:
|
||||||
@@ -33,12 +41,12 @@ jobs:
|
|||||||
- name: Hadolint notebooks
|
- name: Hadolint notebooks
|
||||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||||
with:
|
with:
|
||||||
dockerfile: notebooks/Dockerfile
|
dockerfile: infra/images/notebooks.Dockerfile
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build notebooks
|
- name: Build notebooks
|
||||||
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||||
|
|
||||||
zotero:
|
zotero:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -49,12 +57,12 @@ jobs:
|
|||||||
- name: Hadolint zotero
|
- name: Hadolint zotero
|
||||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||||
with:
|
with:
|
||||||
dockerfile: zotero/Dockerfile
|
dockerfile: infra/images/zotero.Dockerfile
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build zotero
|
- name: Build zotero
|
||||||
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
|
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
|
||||||
|
|
||||||
docs:
|
docs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -65,12 +73,12 @@ jobs:
|
|||||||
- name: Hadolint docs
|
- name: Hadolint docs
|
||||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||||
with:
|
with:
|
||||||
dockerfile: docs/Dockerfile
|
dockerfile: infra/images/docs.Dockerfile
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build docs
|
- name: Build docs
|
||||||
run: docker build -f docs/Dockerfile -t local/docs:build .
|
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
|
||||||
|
|
||||||
api:
|
api:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -81,12 +89,12 @@ jobs:
|
|||||||
- name: Hadolint api
|
- name: Hadolint api
|
||||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||||
with:
|
with:
|
||||||
dockerfile: api/Dockerfile
|
dockerfile: infra/images/api.Dockerfile
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build api
|
- name: Build api
|
||||||
run: docker build -f api/Dockerfile -t local/api:build .
|
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
|
||||||
|
|
||||||
mc:
|
mc:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -97,9 +105,9 @@ jobs:
|
|||||||
- name: Hadolint mc
|
- name: Hadolint mc
|
||||||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||||||
with:
|
with:
|
||||||
dockerfile: rustfs/Dockerfile.mc
|
dockerfile: infra/images/mc.Dockerfile
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build mc
|
- name: Build mc
|
||||||
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ jobs:
|
|||||||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
- name: Build notebooks
|
- name: Build notebooks
|
||||||
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||||
|
|
||||||
- name: Push notebooks
|
- name: Push notebooks
|
||||||
run: |
|
run: |
|
||||||
@@ -42,7 +42,7 @@ jobs:
|
|||||||
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
||||||
|
|
||||||
- name: Build zotero
|
- name: Build zotero
|
||||||
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
|
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build zotero/
|
||||||
|
|
||||||
- name: Push zotero
|
- name: Push zotero
|
||||||
run: |
|
run: |
|
||||||
@@ -51,7 +51,7 @@ jobs:
|
|||||||
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
||||||
|
|
||||||
- name: Build docs
|
- name: Build docs
|
||||||
run: docker build -f docs/Dockerfile -t local/docs:build .
|
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
|
||||||
|
|
||||||
- name: Push docs
|
- name: Push docs
|
||||||
run: |
|
run: |
|
||||||
@@ -60,7 +60,7 @@ jobs:
|
|||||||
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
||||||
|
|
||||||
- name: Build api
|
- name: Build api
|
||||||
run: docker build -f api/Dockerfile -t local/api:build .
|
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
|
||||||
|
|
||||||
- name: Push api
|
- name: Push api
|
||||||
run: |
|
run: |
|
||||||
@@ -69,7 +69,7 @@ jobs:
|
|||||||
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
||||||
|
|
||||||
- name: Build mc
|
- name: Build mc
|
||||||
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||||
|
|
||||||
- name: Push mc
|
- name: Push mc
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 18 KiB |
|
Before Width: | Height: | Size: 822 B After Width: | Height: | Size: 822 B |
|
Before Width: | Height: | Size: 741 B After Width: | Height: | Size: 741 B |
|
Before Width: | Height: | Size: 2.0 KiB After Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 5.0 KiB After Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 29 KiB After Width: | Height: | Size: 29 KiB |
|
Before Width: | Height: | Size: 28 KiB After Width: | Height: | Size: 28 KiB |
|
Before Width: | Height: | Size: 29 KiB After Width: | Height: | Size: 29 KiB |
|
Before Width: | Height: | Size: 966 B After Width: | Height: | Size: 966 B |
43
cloud/aws/README.md
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
# AWS Deployment
|
||||||
|
|
||||||
|
## Service Mapping
|
||||||
|
|
||||||
|
| Component | AWS Service | Replaces |
|
||||||
|
|-----------------|------------------------------|----------------------|
|
||||||
|
| Object Storage | S3 | RustFS |
|
||||||
|
| Database | RDS PostgreSQL | PostgreSQL container |
|
||||||
|
| Iceberg Catalog | AWS Glue / Lake Formation | Nessie / Polaris |
|
||||||
|
| Query Engine | Athena / EMR | Trino |
|
||||||
|
| Compute | ECS Fargate / EKS | Docker Compose |
|
||||||
|
| Ingress | ALB + Route 53 | Traefik + CoreDNS |
|
||||||
|
| CI/CD | GitHub Actions | Gitea Actions |
|
||||||
|
| Observability | CloudWatch + X-Ray | Grafana stack |
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[context.aws]
|
||||||
|
db_backend = "iceberg"
|
||||||
|
storage_backend = "s3"
|
||||||
|
catalog = "glue"
|
||||||
|
compute = "ecs"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Environment Variables
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export STACK_CONTEXT=aws
|
||||||
|
export AWS_DEFAULT_REGION=us-east-1
|
||||||
|
export AWS_S3_BUCKET=your-lakehouse-bucket
|
||||||
|
# IAM role handles auth — no access keys needed on ECS
|
||||||
|
```
|
||||||
|
|
||||||
|
## Required IAM Permissions
|
||||||
|
|
||||||
|
- `s3:GetObject`, `s3:PutObject`, `s3:ListBucket` on lakehouse bucket
|
||||||
|
- `glue:GetTable`, `glue:CreateTable`, `glue:UpdateTable` for catalog
|
||||||
|
- `rds-db:connect` for PostgreSQL
|
||||||
|
|
||||||
|
## Terraform
|
||||||
|
|
||||||
|
See `terraform/` for resource definitions (placeholder).
|
||||||
43
cloud/azure/README.md
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
# Azure Deployment
|
||||||
|
|
||||||
|
## Service Mapping
|
||||||
|
|
||||||
|
| Component | Azure Service | Replaces |
|
||||||
|
|-----------------|------------------------------------|----------------------|
|
||||||
|
| Object Storage | Azure Blob Storage (ABFS) | RustFS |
|
||||||
|
| Database | Azure Database for PostgreSQL | PostgreSQL container |
|
||||||
|
| Iceberg Catalog | Unity Catalog / Azure Purview | Nessie / Polaris |
|
||||||
|
| Query Engine | Azure Synapse / Databricks SQL | Trino |
|
||||||
|
| Compute | Container Apps / AKS | Docker Compose |
|
||||||
|
| Ingress | Application Gateway + Azure DNS | Traefik + CoreDNS |
|
||||||
|
| CI/CD | Azure DevOps / GitHub Actions | Gitea Actions |
|
||||||
|
| Observability | Azure Monitor + App Insights | Grafana stack |
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[context.azure]
|
||||||
|
db_backend = "iceberg"
|
||||||
|
storage_backend = "abfs"
|
||||||
|
catalog = "unity"
|
||||||
|
compute = "container-apps"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Environment Variables
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export STACK_CONTEXT=azure
|
||||||
|
export AZURE_STORAGE_ACCOUNT=yourstorageaccount
|
||||||
|
export AZURE_STORAGE_CONTAINER=lakehouse
|
||||||
|
# Managed Identity handles auth on Container Apps
|
||||||
|
```
|
||||||
|
|
||||||
|
## Required RBAC Roles
|
||||||
|
|
||||||
|
- `Storage Blob Data Contributor` on lakehouse container
|
||||||
|
- `Contributor` on PostgreSQL server
|
||||||
|
- `Databricks workspace access` if using Unity Catalog
|
||||||
|
|
||||||
|
## Terraform
|
||||||
|
|
||||||
|
See `terraform/` for resource definitions (placeholder).
|
||||||
43
cloud/gcp/README.md
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
# GCP Deployment
|
||||||
|
|
||||||
|
## Service Mapping
|
||||||
|
|
||||||
|
| Component | GCP Service | Replaces |
|
||||||
|
|-----------------|-------------------------------|----------------------|
|
||||||
|
| Object Storage | Cloud Storage (GCS) | RustFS |
|
||||||
|
| Database | Cloud SQL (PostgreSQL) | PostgreSQL container |
|
||||||
|
| Iceberg Catalog | BigQuery / Dataplex | Nessie / Polaris |
|
||||||
|
| Query Engine | BigQuery | Trino |
|
||||||
|
| Compute | Cloud Run / GKE | Docker Compose |
|
||||||
|
| Ingress | Cloud Load Balancing + DNS | Traefik + CoreDNS |
|
||||||
|
| CI/CD | Cloud Build / GitHub Actions | Gitea Actions |
|
||||||
|
| Observability | Cloud Monitoring + Trace | Grafana stack |
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[context.gcp]
|
||||||
|
db_backend = "iceberg"
|
||||||
|
storage_backend = "gcs"
|
||||||
|
catalog = "bigquery"
|
||||||
|
compute = "cloud-run"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Environment Variables
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export STACK_CONTEXT=gcp
|
||||||
|
export GOOGLE_CLOUD_PROJECT=your-project-id
|
||||||
|
export GCS_BUCKET=your-lakehouse-bucket
|
||||||
|
# Workload Identity handles auth on Cloud Run
|
||||||
|
```
|
||||||
|
|
||||||
|
## Required IAM Roles
|
||||||
|
|
||||||
|
- `roles/storage.objectAdmin` on lakehouse bucket
|
||||||
|
- `roles/bigquery.dataEditor` for catalog
|
||||||
|
- `roles/cloudsql.client` for PostgreSQL
|
||||||
|
|
||||||
|
## Terraform
|
||||||
|
|
||||||
|
See `terraform/` for resource definitions (placeholder).
|
||||||
35
cloud/self-hosted/README.md
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
# Self-Hosted Deployment (Default)
|
||||||
|
|
||||||
|
Docker Compose on a single node with Traefik ingress.
|
||||||
|
|
||||||
|
## Service Mapping
|
||||||
|
|
||||||
|
| Component | Self-Hosted Service | Config |
|
||||||
|
|-----------------|---------------------------|---------------------------------|
|
||||||
|
| Object Storage | RustFS (S3-compatible) | `[s3]` in stack.toml |
|
||||||
|
| Database | PostgreSQL | compose.yml postgres service |
|
||||||
|
| Iceberg Catalog | Nessie / Polaris | `[lake.nessie]`, `[lake.polaris]` |
|
||||||
|
| Query Engine | Trino | `[lake.trino]` |
|
||||||
|
| Ingress | Traefik | infra/traefik/ |
|
||||||
|
| CI/CD | Gitea Actions + Woodpecker| `[ci]` in stack.toml |
|
||||||
|
| Observability | Grafana + Prometheus + Loki + Jaeger | compose.yml |
|
||||||
|
| DNS | CoreDNS | infra/coredns/ |
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[context.local]
|
||||||
|
db_backend = "duckdb"
|
||||||
|
storage_backend = "local"
|
||||||
|
|
||||||
|
[context.lake]
|
||||||
|
db_backend = "iceberg"
|
||||||
|
storage_backend = "s3"
|
||||||
|
catalog = "nessie"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
60
compose.yml
@@ -29,8 +29,8 @@ services:
|
|||||||
gateway:
|
gateway:
|
||||||
ipv4_address: 172.25.0.53
|
ipv4_address: 172.25.0.53
|
||||||
volumes:
|
volumes:
|
||||||
- ./coredns/Corefile:/etc/coredns/Corefile:ro
|
- ./infra/coredns/Corefile:/etc/coredns/Corefile:ro
|
||||||
- ./coredns/hosts:/etc/coredns/hosts:ro
|
- ./infra/coredns/hosts:/etc/coredns/hosts:ro
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
traefik:
|
traefik:
|
||||||
@@ -46,10 +46,10 @@ services:
|
|||||||
- "443:443"
|
- "443:443"
|
||||||
- "8081:8080"
|
- "8081:8080"
|
||||||
volumes:
|
volumes:
|
||||||
- ./traefik/traefik.yml:/etc/traefik/traefik.yml:ro
|
- ./infra/traefik/traefik.yml:/etc/traefik/traefik.yml:ro
|
||||||
- ./traefik/dynamic:/etc/traefik/dynamic:ro
|
- ./infra/traefik/dynamic:/etc/traefik/dynamic:ro
|
||||||
- ./traefik/certs:/etc/traefik/certs:ro
|
- ./infra/traefik/certs:/etc/traefik/certs:ro
|
||||||
- ./traefik/plugins:/plugins-local:ro
|
- ./infra/traefik/plugins:/plugins-local:ro
|
||||||
environment:
|
environment:
|
||||||
- DOMAIN=${DOMAIN:-homelab.fhirworx.io}
|
- DOMAIN=${DOMAIN:-homelab.fhirworx.io}
|
||||||
- OTEL_SERVICE_NAME=traefik
|
- OTEL_SERVICE_NAME=traefik
|
||||||
@@ -78,8 +78,8 @@ services:
|
|||||||
image: ${IMAGE_PREFIX:-fhirworx}/mc:${COMMIT_SHA:-latest}
|
image: ${IMAGE_PREFIX:-fhirworx}/mc:${COMMIT_SHA:-latest}
|
||||||
container_name: mc
|
container_name: mc
|
||||||
build:
|
build:
|
||||||
context: rustfs
|
context: infra/rustfs
|
||||||
dockerfile: Dockerfile.mc
|
dockerfile: ../images/mc.Dockerfile
|
||||||
networks:
|
networks:
|
||||||
- storage
|
- storage
|
||||||
environment:
|
environment:
|
||||||
@@ -152,8 +152,8 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- gitea_data:/var/lib/gitea
|
- gitea_data:/var/lib/gitea
|
||||||
- gitea_config:/etc/gitea
|
- gitea_config:/etc/gitea
|
||||||
- ./gitea/custom:/var/lib/gitea/custom
|
- ./infra/gitea/custom:/var/lib/gitea/custom
|
||||||
- ./styles/gitea.css:/var/lib/gitea/custom/public/assets/css/theme-loch.css:ro
|
- ./assets/css/gitea.css:/var/lib/gitea/custom/public/assets/css/theme-loch.css:ro
|
||||||
ports:
|
ports:
|
||||||
- "2222:2222"
|
- "2222:2222"
|
||||||
- "3000:3000"
|
- "3000:3000"
|
||||||
@@ -188,7 +188,7 @@ services:
|
|||||||
- WOODPECKER_CUSTOM_CSS_FILE=/etc/woodpecker/custom.css
|
- WOODPECKER_CUSTOM_CSS_FILE=/etc/woodpecker/custom.css
|
||||||
volumes:
|
volumes:
|
||||||
- woodpecker_data:/var/lib/woodpecker
|
- woodpecker_data:/var/lib/woodpecker
|
||||||
- ./styles/woodpecker.css:/etc/woodpecker/custom.css:ro
|
- ./assets/css/woodpecker.css:/etc/woodpecker/custom.css:ro
|
||||||
depends_on:
|
depends_on:
|
||||||
- gitea
|
- gitea
|
||||||
- postgres
|
- postgres
|
||||||
@@ -206,7 +206,7 @@ services:
|
|||||||
- WOODPECKER_MAX_WORKFLOWS=4
|
- WOODPECKER_MAX_WORKFLOWS=4
|
||||||
- WOODPECKER_BACKEND=docker
|
- WOODPECKER_BACKEND=docker
|
||||||
- WOODPECKER_BACKEND_DOCKER_NETWORK=ci
|
- WOODPECKER_BACKEND_DOCKER_NETWORK=ci
|
||||||
- WOODPECKER_BACKEND_DOCKER_VOLUMES=/home/kert/stack/traefik/certs/docker-certs.d:/etc/docker/certs.d:ro
|
- WOODPECKER_BACKEND_DOCKER_VOLUMES=/home/kert/stack/infra/traefik/certs/docker-certs.d:/etc/docker/certs.d:ro
|
||||||
- DOCKER_API_VERSION=1.44
|
- DOCKER_API_VERSION=1.44
|
||||||
volumes:
|
volumes:
|
||||||
- ${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock
|
- ${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock
|
||||||
@@ -230,7 +230,7 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock
|
- ${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock
|
||||||
- act_runner_data:/data
|
- act_runner_data:/data
|
||||||
- ./act-runner/config.yaml:/config.yaml:ro
|
- ./infra/act-runner/config.yaml:/config.yaml:ro
|
||||||
depends_on:
|
depends_on:
|
||||||
- gitea
|
- gitea
|
||||||
security_opt:
|
security_opt:
|
||||||
@@ -255,12 +255,12 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./notebooks:/home/kert/notebooks
|
- ./notebooks:/home/kert/notebooks
|
||||||
- ./notebooks/.marimo-config:/home/kert/.config/marimo
|
- ./notebooks/.marimo-config:/home/kert/.config/marimo
|
||||||
- ./styles/marimo.css:/home/kert/.config/marimo/loch.css:ro
|
- ./assets/css/marimo.css:/home/kert/.config/marimo/loch.css:ro
|
||||||
- ./notebooks/home-page-patched.js:/home/kert/workspace/.venv/lib/python3.13/site-packages/marimo/_static/assets/home-page-itW0tRmv.js:ro
|
- ./notebooks/home-page-patched.js:/home/kert/workspace/.venv/lib/python3.13/site-packages/marimo/_static/assets/home-page-itW0tRmv.js:ro
|
||||||
- ./data:/home/kert/data
|
- ./data:/home/kert/data
|
||||||
- ./zotero/data:/home/kert/zotero:ro
|
- ./zotero/data:/home/kert/zotero:ro
|
||||||
- ./src:/home/kert/src:ro
|
- ./src:/home/kert/src:ro
|
||||||
- ./styles:/home/kert/styles:ro
|
- ./assets:/home/kert/assets:ro
|
||||||
- ./stack.toml:/home/kert/stack.toml:ro
|
- ./stack.toml:/home/kert/stack.toml:ro
|
||||||
deploy:
|
deploy:
|
||||||
resources:
|
resources:
|
||||||
@@ -383,7 +383,7 @@ services:
|
|||||||
- S3_REGION=${S3_REGION:-us-east-1}
|
- S3_REGION=${S3_REGION:-us-east-1}
|
||||||
- S3_WAREHOUSE=${S3_WAREHOUSE:-s3://lakehouse/}
|
- S3_WAREHOUSE=${S3_WAREHOUSE:-s3://lakehouse/}
|
||||||
volumes:
|
volumes:
|
||||||
- ./trino/etc:/etc/trino:ro
|
- ./infra/trino/etc:/etc/trino:ro
|
||||||
depends_on:
|
depends_on:
|
||||||
- nessie
|
- nessie
|
||||||
- rustfs
|
- rustfs
|
||||||
@@ -431,11 +431,11 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- gateway
|
- gateway
|
||||||
volumes:
|
volumes:
|
||||||
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||||
- ./nginx/index.html:/usr/share/nginx/html/index.html:ro
|
- ./infra/nginx/index.html:/usr/share/nginx/html/index.html:ro
|
||||||
- ./styles/dashboard.css:/usr/share/nginx/html/dashboard.css:ro
|
- ./assets/css/dashboard.css:/usr/share/nginx/html/dashboard.css:ro
|
||||||
- ./styles/inject.css:/usr/share/nginx/html/loch.css:ro
|
- ./assets/css/inject.css:/usr/share/nginx/html/loch.css:ro
|
||||||
- ./styles/fav32.png:/usr/share/nginx/html/fav32.png:ro
|
- ./assets/icons/fav32.png:/usr/share/nginx/html/fav32.png:ro
|
||||||
security_opt:
|
security_opt:
|
||||||
- no-new-privileges:true
|
- no-new-privileges:true
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
@@ -508,7 +508,7 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- observability
|
- observability
|
||||||
volumes:
|
volumes:
|
||||||
- ./loki/loki-config.yml:/etc/loki/local-config.yaml:ro
|
- ./infra/loki/loki-config.yml:/etc/loki/local-config.yaml:ro
|
||||||
- loki_data:/loki
|
- loki_data:/loki
|
||||||
command: -config.file=/etc/loki/local-config.yaml
|
command: -config.file=/etc/loki/local-config.yaml
|
||||||
healthcheck:
|
healthcheck:
|
||||||
@@ -527,7 +527,7 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- observability
|
- observability
|
||||||
volumes:
|
volumes:
|
||||||
- ./loki/promtail-config.yml:/etc/promtail/config.yml:ro
|
- ./infra/loki/promtail-config.yml:/etc/promtail/config.yml:ro
|
||||||
- /home/kert/.local/share/docker/containers:/var/lib/docker/containers:ro
|
- /home/kert/.local/share/docker/containers:/var/lib/docker/containers:ro
|
||||||
command: -config.file=/etc/promtail/config.yml
|
command: -config.file=/etc/promtail/config.yml
|
||||||
depends_on:
|
depends_on:
|
||||||
@@ -541,8 +541,8 @@ services:
|
|||||||
- gateway
|
- gateway
|
||||||
- observability
|
- observability
|
||||||
volumes:
|
volumes:
|
||||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||||
- ./prometheus/targets:/etc/prometheus/targets:ro
|
- ./infra/prometheus/targets:/etc/prometheus/targets:ro
|
||||||
- prometheus_data:/prometheus
|
- prometheus_data:/prometheus
|
||||||
command:
|
command:
|
||||||
- "--config.file=/etc/prometheus/prometheus.yml"
|
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||||
@@ -576,12 +576,12 @@ services:
|
|||||||
- GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
|
- GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
|
||||||
- GF_SERVER_ROOT_URL=http://grafana.${DOMAIN:-homelab.fhirworx.io}
|
- GF_SERVER_ROOT_URL=http://grafana.${DOMAIN:-homelab.fhirworx.io}
|
||||||
volumes:
|
volumes:
|
||||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||||
- grafana_data:/var/lib/grafana
|
- grafana_data:/var/lib/grafana
|
||||||
- ./styles/favicon.svg:/usr/share/grafana/public/img/grafana_icon.svg:ro
|
- ./assets/icons/favicon.svg:/usr/share/grafana/public/img/grafana_icon.svg:ro
|
||||||
- ./styles/fav32.png:/usr/share/grafana/public/img/fav32.png:ro
|
- ./assets/icons/fav32.png:/usr/share/grafana/public/img/fav32.png:ro
|
||||||
- ./styles/apple-touch-icon.png:/usr/share/grafana/public/img/apple-touch-icon.png:ro
|
- ./assets/icons/apple-touch-icon.png:/usr/share/grafana/public/img/apple-touch-icon.png:ro
|
||||||
depends_on:
|
depends_on:
|
||||||
- loki
|
- loki
|
||||||
- jaeger
|
- jaeger
|
||||||
|
|||||||
@@ -148,6 +148,28 @@ jobs:
|
|||||||
|
|
||||||
- name: Validate generated config
|
- name: Validate generated config
|
||||||
run: uv run python dev/scripts/gen_config.py --check
|
run: uv run python dev/scripts/gen_config.py --check
|
||||||
|
|
||||||
|
skinny-install:
|
||||||
|
runs-on: {runner}
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
extra: [conf, aco, api, bcda, bib, bls, ccw, cli, cms, pfs, rex]
|
||||||
|
steps:
|
||||||
|
{_checkout_step()}
|
||||||
|
|
||||||
|
{_setup_uv_step(uv_version)}
|
||||||
|
|
||||||
|
- name: Install stack[${{{{ matrix.extra }}}}]
|
||||||
|
run: uv pip install -e ".[${{{{ matrix.extra }}}}]"
|
||||||
|
|
||||||
|
- name: Verify import
|
||||||
|
run: uv run python -c "import ${{{{ matrix.extra }}}}"
|
||||||
|
|
||||||
|
- name: Run module tests
|
||||||
|
run: |
|
||||||
|
if [ -d "tests/${{{{ matrix.extra }}}}" ]; then
|
||||||
|
uv run pytest "tests/${{{{ matrix.extra }}}}/" -x -q || true
|
||||||
|
fi
|
||||||
"""
|
"""
|
||||||
return (".gitea/workflows/ci.yml", content)
|
return (".gitea/workflows/ci.yml", content)
|
||||||
|
|
||||||
|
|||||||
@@ -622,24 +622,24 @@ steps:
|
|||||||
- name: validate-nginx
|
- name: validate-nginx
|
||||||
image: nginx:alpine
|
image: nginx:alpine
|
||||||
commands:
|
commands:
|
||||||
- cp nginx/nginx.conf /etc/nginx/nginx.conf
|
- cp infra/nginx/nginx.conf /etc/nginx/nginx.conf
|
||||||
- nginx -t
|
- nginx -t
|
||||||
when:
|
when:
|
||||||
- path: "nginx/**"
|
- path: "infra/nginx/**"
|
||||||
|
|
||||||
- name: validate-prometheus
|
- name: validate-prometheus
|
||||||
image: prom/prometheus:latest
|
image: prom/prometheus:latest
|
||||||
commands:
|
commands:
|
||||||
- promtool check config prometheus/prometheus.yml
|
- promtool check config infra/prometheus/prometheus.yml
|
||||||
when:
|
when:
|
||||||
- path: "prometheus/**"
|
- path: "infra/prometheus/**"
|
||||||
|
|
||||||
- name: validate-traefik-static
|
- name: validate-traefik-static
|
||||||
image: cytopia/yamllint:latest
|
image: cytopia/yamllint:latest
|
||||||
commands:
|
commands:
|
||||||
- yamllint -d relaxed traefik/traefik.yml
|
- yamllint -d relaxed infra/traefik/traefik.yml
|
||||||
when:
|
when:
|
||||||
- path: "traefik/traefik.yml"
|
- path: "infra/traefik/traefik.yml"
|
||||||
|
|
||||||
- name: validate-traefik-template
|
- name: validate-traefik-template
|
||||||
image: traefik:v3.3
|
image: traefik:v3.3
|
||||||
@@ -648,30 +648,30 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
timeout 5 traefik \\
|
timeout 5 traefik \\
|
||||||
--providers.file.directory=traefik/dynamic \\
|
--providers.file.directory=infra/traefik/dynamic \\
|
||||||
--api.dashboard=false \\
|
--api.dashboard=false \\
|
||||||
--log.level=DEBUG 2>&1 | head -80 || true
|
--log.level=DEBUG 2>&1 | head -80 || true
|
||||||
- echo "Traefik template syntax OK"
|
- echo "Traefik template syntax OK"
|
||||||
when:
|
when:
|
||||||
- path: "traefik/dynamic/**"
|
- path: "infra/traefik/dynamic/**"
|
||||||
|
|
||||||
- name: validate-loki
|
- name: validate-loki
|
||||||
image: cytopia/yamllint:latest
|
image: cytopia/yamllint:latest
|
||||||
commands:
|
commands:
|
||||||
- yamllint -d relaxed loki/
|
- yamllint -d relaxed infra/loki/
|
||||||
when:
|
when:
|
||||||
- path: "loki/**"
|
- path: "infra/loki/**"
|
||||||
|
|
||||||
- name: validate-trino
|
- name: validate-trino
|
||||||
image: alpine:3
|
image: alpine:3
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
OK=true
|
OK=true
|
||||||
for f in trino/etc/config.properties trino/etc/node.properties trino/etc/jvm.config; do
|
for f in infra/trino/etc/config.properties infra/trino/etc/node.properties infra/trino/etc/jvm.config; do
|
||||||
if [ ! -f "$f" ]; then echo "MISSING: $f"; OK=false; fi
|
if [ ! -f "$f" ]; then echo "MISSING: $f"; OK=false; fi
|
||||||
done
|
done
|
||||||
ls trino/etc/catalog/*.properties >/dev/null 2>&1 || { echo "MISSING: no catalog properties"; OK=false; }
|
ls infra/trino/etc/catalog/*.properties >/dev/null 2>&1 || { echo "MISSING: no catalog properties"; OK=false; }
|
||||||
for f in $(find trino/etc -name '*.properties'); do
|
for f in $(find infra/trino/etc -name '*.properties'); do
|
||||||
while IFS= read -r line; do
|
while IFS= read -r line; do
|
||||||
case "$line" in ''|'#'*|'!'*) continue ;; esac
|
case "$line" in ''|'#'*|'!'*) continue ;; esac
|
||||||
echo "$line" | grep -q '=' || { echo "BAD LINE in $f: $line"; OK=false; }
|
echo "$line" | grep -q '=' || { echo "BAD LINE in $f: $line"; OK=false; }
|
||||||
@@ -680,7 +680,7 @@ steps:
|
|||||||
$OK && echo "Trino config validation passed"
|
$OK && echo "Trino config validation passed"
|
||||||
$OK
|
$OK
|
||||||
when:
|
when:
|
||||||
- path: "trino/etc/**"
|
- path: "infra/trino/etc/**"
|
||||||
""")
|
""")
|
||||||
|
|
||||||
parts.append(self._failure_reporter)
|
parts.append(self._failure_reporter)
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ from pathlib import Path
|
|||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
CERT_DIR = REPO_ROOT / "traefik" / "certs"
|
CERT_DIR = REPO_ROOT / "infra" / "traefik" / "certs"
|
||||||
COREDNS_DIR = REPO_ROOT / "coredns"
|
COREDNS_DIR = REPO_ROOT / "infra" / "coredns"
|
||||||
DOCKER_CERTS_DIR = CERT_DIR / "docker-certs.d"
|
DOCKER_CERTS_DIR = CERT_DIR / "docker-certs.d"
|
||||||
|
|
||||||
CA_KEY = CERT_DIR / "ca.key"
|
CA_KEY = CERT_DIR / "ca.key"
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ def gen_coredns() -> dict[str, str]:
|
|||||||
cache 300
|
cache 300
|
||||||
}}
|
}}
|
||||||
"""
|
"""
|
||||||
return {"coredns/hosts": hosts, "coredns/Corefile": corefile}
|
return {"infra/coredns/hosts": hosts, "infra/coredns/Corefile": corefile}
|
||||||
|
|
||||||
|
|
||||||
# ── Backend dispatch ─────────────────────────────────────────────
|
# ── Backend dispatch ─────────────────────────────────────────────
|
||||||
@@ -177,6 +177,37 @@ def check_all(files: dict[str, str]) -> list[str]:
|
|||||||
return diffs
|
return diffs
|
||||||
|
|
||||||
|
|
||||||
|
def check_modules() -> list[str]:
|
||||||
|
"""Verify module-name and optional-dependencies stay in sync."""
|
||||||
|
import tomllib
|
||||||
|
|
||||||
|
pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text())
|
||||||
|
module_names = set(
|
||||||
|
pyproject.get("tool", {})
|
||||||
|
.get("uv", {})
|
||||||
|
.get("build-backend", {})
|
||||||
|
.get("module-name", [])
|
||||||
|
)
|
||||||
|
opt_deps = set(pyproject.get("project", {}).get("optional-dependencies", {}).keys())
|
||||||
|
# Groups that are aggregates or cloud providers, not Python modules
|
||||||
|
meta_groups = {"all", "lake", "dev", "aws", "gcp", "azure"}
|
||||||
|
|
||||||
|
errors = []
|
||||||
|
for mod in sorted(module_names):
|
||||||
|
if mod not in opt_deps and mod not in meta_groups:
|
||||||
|
errors.append(
|
||||||
|
f"module '{mod}' in [tool.uv.build-backend].module-name "
|
||||||
|
f"has no matching [project.optional-dependencies] group"
|
||||||
|
)
|
||||||
|
for group in sorted(opt_deps - meta_groups):
|
||||||
|
if group not in module_names:
|
||||||
|
errors.append(
|
||||||
|
f"optional-dep group '{group}' has no matching "
|
||||||
|
f"module in [tool.uv.build-backend].module-name"
|
||||||
|
)
|
||||||
|
return errors
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Generate derived config from stack.toml [ci] backend"
|
description="Generate derived config from stack.toml [ci] backend"
|
||||||
@@ -200,13 +231,21 @@ def main() -> int:
|
|||||||
|
|
||||||
if args.check:
|
if args.check:
|
||||||
diffs = check_all(files)
|
diffs = check_all(files)
|
||||||
if diffs:
|
mod_errors = check_modules()
|
||||||
print("Files out of date:")
|
all_errors = diffs + mod_errors
|
||||||
for d in diffs:
|
if all_errors:
|
||||||
print(f" {d}")
|
if diffs:
|
||||||
|
print("Files out of date:")
|
||||||
|
for d in diffs:
|
||||||
|
print(f" {d}")
|
||||||
|
if mod_errors:
|
||||||
|
print("Module/dependency sync errors:")
|
||||||
|
for e in mod_errors:
|
||||||
|
print(f" {e}")
|
||||||
print("\nRun: uv run python dev/scripts/gen_config.py")
|
print("\nRun: uv run python dev/scripts/gen_config.py")
|
||||||
return 1
|
return 1
|
||||||
print("All generated files are up-to-date.")
|
print("All generated files are up-to-date.")
|
||||||
|
print("Module names and optional-dependencies are in sync.")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
write_all(files)
|
write_all(files)
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
# Usage: ./dev/scripts/install_certs.sh
|
# Usage: ./dev/scripts/install_certs.sh
|
||||||
|
|
||||||
set -uo pipefail
|
set -uo pipefail
|
||||||
CERT="$(cd "$(dirname "$0")/../.." && pwd)/traefik/certs/ca.crt"
|
CERT="$(cd "$(dirname "$0")/../.." && pwd)/infra/traefik/certs/ca.crt"
|
||||||
CERT_NAME="Homelab CA"
|
CERT_NAME="Homelab CA"
|
||||||
|
|
||||||
if [ ! -f "$CERT" ]; then
|
if [ ! -f "$CERT" ]; then
|
||||||
|
|||||||
112
pyproject.toml
@@ -1,32 +1,109 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "stack"
|
name = "stack"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
description = "Add your description here"
|
description = "Healthcare data platform — ACO analytics, CMS data, BCDA FHIR, PFS equations"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"databricks-cli>=0.18.0",
|
"pydantic>=2.0.0",
|
||||||
"databricks-sdk>=0.85.0",
|
|
||||||
"duckdb>=1.0.0",
|
|
||||||
"fastapi>=0.135.1",
|
|
||||||
"fastexcel>=0.19.0",
|
|
||||||
"fsspec>=2024.1.0",
|
|
||||||
"httpx>=0.28.1",
|
|
||||||
"narwhals>=2.17.0",
|
|
||||||
"pyarrow>=23.0.0",
|
|
||||||
"cryptography>=46.0.5",
|
|
||||||
"pyasn1>=0.6.3",
|
|
||||||
"pyjwt>=2.12.0",
|
|
||||||
"sqlglot>=26.0.0",
|
|
||||||
"typer>=0.24.1",
|
|
||||||
"uvicorn>=0.41.0",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
stack = "cli:main"
|
stack = "cli:main"
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
conf = [
|
||||||
|
"pydantic>=2.0.0",
|
||||||
|
"httpx>=0.28.1",
|
||||||
|
"duckdb>=1.0.0",
|
||||||
|
]
|
||||||
|
aco = [
|
||||||
|
"stack[conf]",
|
||||||
|
"narwhals>=2.17.0",
|
||||||
|
"sqlglot>=26.0.0",
|
||||||
|
"pyarrow>=23.0.0",
|
||||||
|
"duckdb>=1.0.0",
|
||||||
|
]
|
||||||
|
api = [
|
||||||
|
"stack[conf]",
|
||||||
|
"fastapi>=0.135.1",
|
||||||
|
"uvicorn>=0.41.0",
|
||||||
|
"httpx>=0.28.1",
|
||||||
|
"pyjwt>=2.12.0",
|
||||||
|
"cryptography>=46.0.5",
|
||||||
|
]
|
||||||
|
bcda = [
|
||||||
|
"stack[conf]",
|
||||||
|
"httpx>=0.28.1",
|
||||||
|
"fsspec>=2024.1.0",
|
||||||
|
"narwhals>=2.17.0",
|
||||||
|
"pyarrow>=23.0.0",
|
||||||
|
]
|
||||||
|
bib = [
|
||||||
|
"stack[conf]",
|
||||||
|
"pydantic>=2.0.0",
|
||||||
|
]
|
||||||
|
bls = []
|
||||||
|
ccw = [
|
||||||
|
"pydantic>=2.0.0",
|
||||||
|
]
|
||||||
|
cli = [
|
||||||
|
"stack[aco]",
|
||||||
|
"stack[api]",
|
||||||
|
"stack[bib]",
|
||||||
|
"typer>=0.24.1",
|
||||||
|
"uvicorn>=0.41.0",
|
||||||
|
]
|
||||||
|
cms = [
|
||||||
|
"narwhals>=2.17.0",
|
||||||
|
"pydantic>=2.0.0",
|
||||||
|
]
|
||||||
|
pfs = [
|
||||||
|
"stack[conf]",
|
||||||
|
"stack[rex]",
|
||||||
|
"duckdb>=1.0.0",
|
||||||
|
"narwhals>=2.17.0",
|
||||||
|
]
|
||||||
|
rex = [
|
||||||
|
"stack[conf]",
|
||||||
|
"narwhals>=2.17.0",
|
||||||
|
"pyarrow>=23.0.0",
|
||||||
|
"fsspec>=2024.1.0",
|
||||||
|
]
|
||||||
|
lake = [
|
||||||
|
"stack[aco]",
|
||||||
|
"databricks-cli>=0.18.0",
|
||||||
|
"databricks-sdk>=0.85.0",
|
||||||
|
]
|
||||||
|
aws = [
|
||||||
|
"boto3>=1.35.0",
|
||||||
|
"s3fs>=2026.2.0",
|
||||||
|
]
|
||||||
|
gcp = [
|
||||||
|
"google-cloud-storage>=2.18.0",
|
||||||
|
"gcsfs>=2024.1.0",
|
||||||
|
]
|
||||||
|
azure = [
|
||||||
|
"azure-storage-blob>=12.23.0",
|
||||||
|
"adlfs>=2024.1.0",
|
||||||
|
]
|
||||||
|
all = [
|
||||||
|
"stack[aco]",
|
||||||
|
"stack[api]",
|
||||||
|
"stack[bcda]",
|
||||||
|
"stack[bib]",
|
||||||
|
"stack[bls]",
|
||||||
|
"stack[ccw]",
|
||||||
|
"stack[cli]",
|
||||||
|
"stack[cms]",
|
||||||
|
"stack[pfs]",
|
||||||
|
"stack[rex]",
|
||||||
|
"stack[lake]",
|
||||||
|
]
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
|
"stack[all]",
|
||||||
"coverage>=7.13.4",
|
"coverage>=7.13.4",
|
||||||
"dbt-core==1.10.15",
|
"dbt-core==1.10.15",
|
||||||
"dbt-duckdb>=1.10,<1.11",
|
"dbt-duckdb>=1.10,<1.11",
|
||||||
@@ -42,6 +119,7 @@ dev = [
|
|||||||
"obstore>=0.9.2",
|
"obstore>=0.9.2",
|
||||||
"s3fs>=2026.2.0",
|
"s3fs>=2026.2.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["uv_build>=0.7,<1"]
|
requires = ["uv_build>=0.7,<1"]
|
||||||
build-backend = "uv_build"
|
build-backend = "uv_build"
|
||||||
@@ -59,4 +137,4 @@ testpaths = ["tests"]
|
|||||||
[tool.uv.build-backend]
|
[tool.uv.build-backend]
|
||||||
module-name = ["aco", "api", "bcda", "bib", "bls", "ccw", "cli", "cms", "conf", "pfs", "rex"]
|
module-name = ["aco", "api", "bcda", "bib", "bls", "ccw", "cli", "cms", "conf", "pfs", "rex"]
|
||||||
namespace = true
|
namespace = true
|
||||||
source-exclude = ["compose.yml","grafana/**","nginx/**","prometheus/**","rustfs/**","data/**", "loki/**", "notebooks/**", "traefik/**", "woodpecker/**", "gitea/**", "polaris/**", "trino/**", "zotero/**", "tuva/**"]
|
source-exclude = ["compose.yml","infra/**","data/**","notebooks/**","zotero/**","tuva/**","assets/**","docs/**","dev/**","bundle/**"]
|
||||||
|
|||||||