P18: eliminate remaining hardcoded values across the stack
- compose.yml: IMAGE_PREFIX env var for all fhirworx/ images,
NESSIE_API_URI env var for trino catalog config
- src/aco/lake/*: all catalog_uri defaults from cfg.lake.*
instead of hardcoded nessie/polaris URLs
- trino iceberg.properties: use ${ENV:NESSIE_API_URI}
- docs: docusaurus url from DOCS_URL env var
- dev/scripts: bootstrap_certs reads from cfg.platform,
bootstrap_secrets reads gitea_url from cfg.services,
install_certs uses $DOMAIN
- deploy.yml: publish uses $CI_REPO_OWNER for pypi URL,
prep-docs uses $CI_WORKSPACE
- rebuild-all.yml: deploy uses $STACK_ROOT for host bind mount
This commit is contained in:
@@ -57,7 +57,7 @@ steps:
|
|||||||
REGISTRY_PASS:
|
REGISTRY_PASS:
|
||||||
from_secret: registry_pass
|
from_secret: registry_pass
|
||||||
commands:
|
commands:
|
||||||
- uv publish --publish-url http://gitea:3000/api/packages/homelab/pypi --username "$REGISTRY_USER" --password "$REGISTRY_PASS" dist/*
|
- uv publish --publish-url http://gitea:3000/api/packages/${CI_REPO_OWNER}/pypi --username "$REGISTRY_USER" --password "$REGISTRY_PASS" dist/*
|
||||||
depends_on:
|
depends_on:
|
||||||
- build-package
|
- build-package
|
||||||
|
|
||||||
|
|||||||
@@ -159,7 +159,7 @@ steps:
|
|||||||
- docker compose up -d --remove-orphans
|
- docker compose up -d --remove-orphans
|
||||||
volumes:
|
volumes:
|
||||||
- /run/user/1000/docker.sock:/var/run/docker.sock
|
- /run/user/1000/docker.sock:/var/run/docker.sock
|
||||||
- /home/kert/stack:/home/kert/stack
|
- ${STACK_ROOT:-/home/kert/stack}:${STACK_ROOT:-/home/kert/stack}
|
||||||
depends_on:
|
depends_on:
|
||||||
- scan-notebooks
|
- scan-notebooks
|
||||||
- scan-zotero
|
- scan-zotero
|
||||||
|
|||||||
15
compose.yml
15
compose.yml
@@ -58,7 +58,7 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
rustfs:
|
rustfs:
|
||||||
image: fhirworx/rustfs:latest
|
image: ${IMAGE_PREFIX:-fhirworx}/rustfs:latest
|
||||||
container_name: rustfs
|
container_name: rustfs
|
||||||
user: "10001:10001"
|
user: "10001:10001"
|
||||||
networks:
|
networks:
|
||||||
@@ -75,7 +75,7 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
mc:
|
mc:
|
||||||
image: fhirworx/mc:${COMMIT_SHA:-latest}
|
image: ${IMAGE_PREFIX:-fhirworx}/mc:${COMMIT_SHA:-latest}
|
||||||
container_name: mc
|
container_name: mc
|
||||||
build:
|
build:
|
||||||
context: rustfs
|
context: rustfs
|
||||||
@@ -98,7 +98,7 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
image: fhirworx/postgresql:latest
|
image: ${IMAGE_PREFIX:-fhirworx}/postgresql:latest
|
||||||
container_name: postgres
|
container_name: postgres
|
||||||
networks:
|
networks:
|
||||||
- storage
|
- storage
|
||||||
@@ -216,7 +216,7 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
notebooks:
|
notebooks:
|
||||||
image: fhirworx/notebooks:${COMMIT_SHA:-latest}
|
image: ${IMAGE_PREFIX:-fhirworx}/notebooks:${COMMIT_SHA:-latest}
|
||||||
container_name: notebooks
|
container_name: notebooks
|
||||||
networks:
|
networks:
|
||||||
- gateway
|
- gateway
|
||||||
@@ -251,7 +251,7 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
zotero:
|
zotero:
|
||||||
image: fhirworx/zotero:${COMMIT_SHA:-latest}
|
image: ${IMAGE_PREFIX:-fhirworx}/zotero:${COMMIT_SHA:-latest}
|
||||||
container_name: zotero
|
container_name: zotero
|
||||||
networks:
|
networks:
|
||||||
- gateway
|
- gateway
|
||||||
@@ -356,6 +356,7 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
- NESSIE_S3_ACCESS_KEY=${NESSIE_S3_ACCESS_KEY}
|
- NESSIE_S3_ACCESS_KEY=${NESSIE_S3_ACCESS_KEY}
|
||||||
- NESSIE_S3_SECRET_KEY=${NESSIE_S3_SECRET_KEY}
|
- NESSIE_S3_SECRET_KEY=${NESSIE_S3_SECRET_KEY}
|
||||||
|
- NESSIE_API_URI=${NESSIE_API_URI:-http://nessie:19120/api/v2}
|
||||||
- S3_ENDPOINT=${S3_ENDPOINT:-http://rustfs:9000}
|
- S3_ENDPOINT=${S3_ENDPOINT:-http://rustfs:9000}
|
||||||
- S3_REGION=${S3_REGION:-us-east-1}
|
- S3_REGION=${S3_REGION:-us-east-1}
|
||||||
- S3_WAREHOUSE=${S3_WAREHOUSE:-s3://lakehouse/}
|
- S3_WAREHOUSE=${S3_WAREHOUSE:-s3://lakehouse/}
|
||||||
@@ -418,14 +419,14 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
docs:
|
docs:
|
||||||
image: fhirworx/docs:${COMMIT_SHA:-latest}
|
image: ${IMAGE_PREFIX:-fhirworx}/docs:${COMMIT_SHA:-latest}
|
||||||
container_name: docs
|
container_name: docs
|
||||||
networks:
|
networks:
|
||||||
- gateway
|
- gateway
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
api:
|
api:
|
||||||
image: fhirworx/api:${COMMIT_SHA:-latest}
|
image: ${IMAGE_PREFIX:-fhirworx}/api:${COMMIT_SHA:-latest}
|
||||||
container_name: api
|
container_name: api
|
||||||
networks:
|
networks:
|
||||||
- gateway
|
- gateway
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
"""Bootstrap TLS certificates for the homelab stack.
|
"""Bootstrap TLS certificates for the homelab stack.
|
||||||
|
|
||||||
Generates a self-signed CA and wildcard certificate for *.homelab.fhirworx.io,
|
Generates a self-signed CA and wildcard certificate for *.$DOMAIN,
|
||||||
then distributes the CA cert to Docker's certs.d for registry trust and
|
then distributes the CA cert to Docker's certs.d for registry trust and
|
||||||
regenerates the CoreDNS hosts file from HOST_IP.
|
regenerates the CoreDNS hosts file from HOST_IP.
|
||||||
|
|
||||||
@@ -33,8 +33,10 @@ CA_SUBJECT = "/CN=Homelab CA/O=fhirworx"
|
|||||||
CA_DAYS = 3650
|
CA_DAYS = 3650
|
||||||
TLS_DAYS = 3650
|
TLS_DAYS = 3650
|
||||||
|
|
||||||
DOMAIN = os.environ.get("DOMAIN", "homelab.fhirworx.io")
|
from conf import cfg
|
||||||
HOST_IP = os.environ.get("HOST_IP", "192.168.1.192")
|
|
||||||
|
DOMAIN = os.environ.get("DOMAIN", cfg.platform.domain)
|
||||||
|
HOST_IP = os.environ.get("HOST_IP", cfg.platform.host_ip)
|
||||||
|
|
||||||
HOSTS_SUBDOMAINS = [
|
HOSTS_SUBDOMAINS = [
|
||||||
"",
|
"",
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ WP_SECRETS = {
|
|||||||
"registry_user": ("kert", None), # static value, not from .env
|
"registry_user": ("kert", None), # static value, not from .env
|
||||||
"registry_pass": (None, "GITEA_ADMIN_PASSWORD"),
|
"registry_pass": (None, "GITEA_ADMIN_PASSWORD"),
|
||||||
"gitea_token": (None, "GITEA_TOKEN"),
|
"gitea_token": (None, "GITEA_TOKEN"),
|
||||||
"gitea_url": ("http://gitea:3000", None),
|
"gitea_url": (None, "_GITEA_SERVICE_URL"), # resolved from cfg below
|
||||||
"s3_access_key": (None, "RUSTFS_ACCESS_KEY"),
|
"s3_access_key": (None, "RUSTFS_ACCESS_KEY"),
|
||||||
"s3_secret_key": (None, "RUSTFS_SECRET_KEY"),
|
"s3_secret_key": (None, "RUSTFS_SECRET_KEY"),
|
||||||
"root_key": (None, "ROOT_KEY"),
|
"root_key": (None, "ROOT_KEY"),
|
||||||
@@ -226,6 +226,9 @@ def bootstrap_gitea(env: dict[str, str]) -> list[str]:
|
|||||||
|
|
||||||
def bootstrap_woodpecker(env: dict[str, str]) -> list[str]:
|
def bootstrap_woodpecker(env: dict[str, str]) -> list[str]:
|
||||||
"""Set Woodpecker repo secrets via database."""
|
"""Set Woodpecker repo secrets via database."""
|
||||||
|
from conf import cfg
|
||||||
|
|
||||||
|
env.setdefault("_GITEA_SERVICE_URL", cfg.services.gitea)
|
||||||
errors = []
|
errors = []
|
||||||
wp_pw = env.get("WOODPECKER_DB_PASSWORD", "")
|
wp_pw = env.get("WOODPECKER_DB_PASSWORD", "")
|
||||||
for name, (static_val, env_var) in WP_SECRETS.items():
|
for name, (static_val, env_var) in WP_SECRETS.items():
|
||||||
|
|||||||
@@ -49,4 +49,4 @@ echo " export NODE_EXTRA_CA_CERTS=$CERT"
|
|||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "Done. Restart Firefox/Chrome to pick up the new CA."
|
echo "Done. Restart Firefox/Chrome to pick up the new CA."
|
||||||
echo "HTTPS: https://docs.homelab.fhirworx.io should now load without warnings."
|
echo "HTTPS: https://docs.${DOMAIN:-homelab.fhirworx.io} should now load without warnings."
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ const config = {
|
|||||||
title: "Stack",
|
title: "Stack",
|
||||||
tagline: "Healthcare data platform documentation",
|
tagline: "Healthcare data platform documentation",
|
||||||
favicon: "img/favicon.png",
|
favicon: "img/favicon.png",
|
||||||
url: "http://docs.homelab.fhirworx.io",
|
url: process.env.DOCS_URL || "http://docs.homelab.fhirworx.io",
|
||||||
baseUrl: "/",
|
baseUrl: "/",
|
||||||
onBrokenLinks: "warn",
|
onBrokenLinks: "warn",
|
||||||
|
|
||||||
|
|||||||
@@ -16,14 +16,15 @@ Four contexts, one interface::
|
|||||||
ctx = DuckDBContext(database=path("db.aco")) # path from conf
|
ctx = DuckDBContext(database=path("db.aco")) # path from conf
|
||||||
|
|
||||||
# Iceberg on Nessie — versioned catalog, S3 storage
|
# Iceberg on Nessie — versioned catalog, S3 storage
|
||||||
|
from conf import cfg
|
||||||
ctx = IcebergContext(
|
ctx = IcebergContext(
|
||||||
catalog_uri="http://nessie:19120/iceberg/",
|
catalog_uri=cfg.lake.nessie.catalog_uri,
|
||||||
warehouse="s3://lakehouse/",
|
warehouse="s3://lakehouse/",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Iceberg on Polaris — governed catalog, same REST spec
|
# Iceberg on Polaris — governed catalog, same REST spec
|
||||||
ctx = IcebergContext(
|
ctx = IcebergContext(
|
||||||
catalog_uri="http://polaris:8181/api/catalog",
|
catalog_uri=cfg.lake.polaris.catalog_uri,
|
||||||
warehouse="s3://lakehouse/",
|
warehouse="s3://lakehouse/",
|
||||||
properties={"credential": "root:<secret>"},
|
properties={"credential": "root:<secret>"},
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -33,8 +33,9 @@ Usage::
|
|||||||
cat.model("core.encounter") # CoreEncounter (SQLTable)
|
cat.model("core.encounter") # CoreEncounter (SQLTable)
|
||||||
|
|
||||||
# With Iceberg catalog — validates against a live warehouse
|
# With Iceberg catalog — validates against a live warehouse
|
||||||
|
from conf import cfg
|
||||||
cat = Catalog(
|
cat = Catalog(
|
||||||
catalog_uri="http://nessie:19120/iceberg/",
|
catalog_uri=cfg.lake.nessie.catalog_uri,
|
||||||
warehouse="s3://lakehouse/",
|
warehouse="s3://lakehouse/",
|
||||||
)
|
)
|
||||||
cat.iceberg_namespaces() # ['core', 'readmissions', ...]
|
cat.iceberg_namespaces() # ['core', 'readmissions', ...]
|
||||||
|
|||||||
@@ -35,8 +35,9 @@ Usage::
|
|||||||
cache = readmissions.pipeline.run(ctx.load)
|
cache = readmissions.pipeline.run(ctx.load)
|
||||||
|
|
||||||
# Iceberg on Nessie (versioned, S3-backed)
|
# Iceberg on Nessie (versioned, S3-backed)
|
||||||
|
from conf import cfg
|
||||||
ctx = IcebergContext(
|
ctx = IcebergContext(
|
||||||
catalog_uri="http://nessie:19120/iceberg/",
|
catalog_uri=cfg.lake.nessie.catalog_uri,
|
||||||
catalog_type="rest",
|
catalog_type="rest",
|
||||||
warehouse="s3://lakehouse/",
|
warehouse="s3://lakehouse/",
|
||||||
)
|
)
|
||||||
@@ -44,11 +45,11 @@ Usage::
|
|||||||
|
|
||||||
# Iceberg on Polaris (governed, same REST spec)
|
# Iceberg on Polaris (governed, same REST spec)
|
||||||
ctx = IcebergContext(
|
ctx = IcebergContext(
|
||||||
catalog_uri="http://polaris:8181/api/catalog",
|
catalog_uri=cfg.lake.polaris.catalog_uri,
|
||||||
catalog_type="rest",
|
catalog_type="rest",
|
||||||
warehouse="s3://lakehouse/",
|
warehouse="s3://lakehouse/",
|
||||||
properties={
|
properties={
|
||||||
"oauth2-server-uri": "http://polaris:8181/api/catalog/v1/oauth/tokens",
|
"oauth2-server-uri": f"{cfg.services.polaris}/api/catalog/v1/oauth/tokens",
|
||||||
"credential": "root:<secret>",
|
"credential": "root:<secret>",
|
||||||
"scope": "PRINCIPAL_ROLE:ALL",
|
"scope": "PRINCIPAL_ROLE:ALL",
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ Usage::
|
|||||||
|
|
||||||
from aco.lake.deploy import deploy_schemas
|
from aco.lake.deploy import deploy_schemas
|
||||||
|
|
||||||
stats = deploy_schemas(catalog_uri="http://nessie:19120/iceberg/")
|
from conf import cfg
|
||||||
|
stats = deploy_schemas(catalog_uri=cfg.lake.nessie.catalog_uri)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|||||||
@@ -50,7 +50,8 @@ Usage::
|
|||||||
results = execute(readmissions.pipeline, ctx)
|
results = execute(readmissions.pipeline, ctx)
|
||||||
|
|
||||||
# Iceberg on Nessie — direct execution via PyIceberg
|
# Iceberg on Nessie — direct execution via PyIceberg
|
||||||
ctx = IcebergContext(catalog_uri="http://nessie:19120/iceberg/")
|
from conf import cfg
|
||||||
|
ctx = IcebergContext(catalog_uri=cfg.lake.nessie.catalog_uri)
|
||||||
results = execute(readmissions.pipeline, ctx)
|
results = execute(readmissions.pipeline, ctx)
|
||||||
|
|
||||||
# Trino — transpiled execution
|
# Trino — transpiled execution
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
connector.name=iceberg
|
connector.name=iceberg
|
||||||
iceberg.catalog.type=nessie
|
iceberg.catalog.type=nessie
|
||||||
iceberg.nessie-catalog.uri=http://nessie:19120/api/v2
|
iceberg.nessie-catalog.uri=${ENV:NESSIE_API_URI}
|
||||||
iceberg.nessie-catalog.ref=main
|
iceberg.nessie-catalog.ref=main
|
||||||
iceberg.nessie-catalog.default-warehouse-dir=${ENV:S3_WAREHOUSE}
|
iceberg.nessie-catalog.default-warehouse-dir=${ENV:S3_WAREHOUSE}
|
||||||
fs.native-s3.enabled=true
|
fs.native-s3.enabled=true
|
||||||
|
|||||||
Reference in New Issue
Block a user