homelab/stack
1
0
Fork 0
Code Issues 20 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

P18: config consolidation — fix #132 #133 #134 #137 #138 #139 #141

Browse Source 
- stack.toml: add [platform] (domain, registry, repo, ci_email,
  host_ip, image_prefix) and [services] (gitea, woodpecker,
  rustfs, nessie, polaris, trino, grafana) sections
- Python clients: default URLs from cfg.services.* instead of
  hardcoded strings (gitea, woodpecker, rustfs, admin clients)
- diag reporter: resolve labels by name via Gitea API instead
  of hardcoded IDs; repo owner/name from cfg.platform.repo
- CI pipelines: YAML anchors &registry/&org for single source
  of truth; $REGISTRY/$ORG env vars in scan commands
- ci.yml: use $CI_REPO for git push URL and S3 badge target
- rebuild-all.yml: $IMAGE_PREFIX, $STACK_ROOT env vars
- deploy.yml: $CI_WORKSPACE for host volume mounts
This commit is contained in:
kert
2026-03-23 19:49:47 -04:00
parent 9f6683ab10
commit e801b987b6
12 changed files with 154 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.woodpecker/ci.yml
View File

@@ -55,11 +55,11 @@ steps:
from_secret: registry_pass
commands:
- git config user.name woodpecker-ci
- git config user.email ci@fhirworx.io
- git config user.email ci@${CI_REPO_OWNER}.io
- git add coverage.svg
- git diff --cached --quiet && echo "no change" && exit 0
- git commit -m "update coverage badge [skip ci]"
- git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/homelab/stack.git HEAD:main
- git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/${CI_REPO}.git HEAD:main
when:
- branch: main
event: push
@@ -77,7 +77,7 @@ steps:
secret_key:
from_secret: s3_secret_key
source: "coverage.svg"
target: /badges/homelab/stack/
target: /badges/${CI_REPO}/
path_style: true
overwrite: true
depends_on:

17
.woodpecker/deploy.yml
View File

@@ -16,17 +16,22 @@ when:
variables:
- &buildx_image woodpeckerci/plugin-docker-buildx:5-insecure
- &registry gitea.homelab.fhirworx.io
- &org homelab
- &buildx_base
buildkit_config: |
[registry."gitea.homelab.fhirworx.io"]
http = true
registry: gitea.homelab.fhirworx.io
registry: *registry
username:
from_secret: registry_user
password:
from_secret: registry_pass
- &trivy_scan
image: aquasec/trivy:latest
environment:
REGISTRY: *registry
ORG: *org
- &diag_env
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
@@ -73,7 +78,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
-o notebooks-scan.json
"gitea.homelab.fhirworx.io/homelab/notebooks:sha-${CI_COMMIT_SHA:0:8}"
"$REGISTRY/$ORG/notebooks:${CI_COMMIT_SHA:0:8}"
depends_on:
- build-push-notebooks
@@ -94,7 +99,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
-o zotero-scan.json
"gitea.homelab.fhirworx.io/homelab/zotero:sha-${CI_COMMIT_SHA:0:8}"
"$REGISTRY/$ORG/zotero:${CI_COMMIT_SHA:0:8}"
depends_on:
- build-push-zotero
@@ -102,7 +107,7 @@ steps:
- name: prep-docs-context
image: alpine:3
volumes:
- /home/kert/stack/data:/host-data:ro
- ${CI_WORKSPACE}/data:/host-data:ro
commands:
- mkdir -p data
- cp /host-data/bib.sqlite data/ 2>/dev/null || true
@@ -125,7 +130,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
-o docs-scan.json
"gitea.homelab.fhirworx.io/homelab/docs:sha-${CI_COMMIT_SHA:0:8}"
"$REGISTRY/$ORG/docs:${CI_COMMIT_SHA:0:8}"
depends_on:
- build-push-docs
@@ -146,7 +151,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
-o api-scan.json
"gitea.homelab.fhirworx.io/homelab/api:sha-${CI_COMMIT_SHA:0:8}"
"$REGISTRY/$ORG/api:${CI_COMMIT_SHA:0:8}"
depends_on:
- build-push-api

18
.woodpecker/harden.yml
View File

@@ -2,23 +2,31 @@
# Rebuilds all images with apt-get/apk upgrade, scans for vulns,
# and auto-closes Gitea vuln issues when scans are clean.
# Trigger: manual or weekly cron.
#
# Registry host is defined ONCE as &registry — change it here to
# retarget all builds and scans.
when:
- event: [manual, cron]
variables:
- &buildx_image woodpeckerci/plugin-docker-buildx:5-insecure
- &registry gitea.homelab.fhirworx.io
- &org homelab
- &buildx_base
buildkit_config: |
[registry."gitea.homelab.fhirworx.io"]
http = true
registry: gitea.homelab.fhirworx.io
registry: *registry
username:
from_secret: registry_user
password:
from_secret: registry_pass
- &trivy_scan
image: aquasec/trivy:latest
environment:
REGISTRY: *registry
ORG: *org
- &diag_env
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
@@ -84,7 +92,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL
--exit-code 0 --format json -o notebooks-scan.json
"gitea.homelab.fhirworx.io/homelab/notebooks:hardened"
"$REGISTRY/$ORG/notebooks:hardened"
depends_on: [build-notebooks]
- name: scan-zotero
@@ -92,7 +100,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL
--exit-code 0 --format json -o zotero-scan.json
"gitea.homelab.fhirworx.io/homelab/zotero:hardened"
"$REGISTRY/$ORG/zotero:hardened"
depends_on: [build-zotero]
- name: scan-docs
@@ -100,7 +108,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL
--exit-code 0 --format json -o docs-scan.json
"gitea.homelab.fhirworx.io/homelab/docs:hardened"
"$REGISTRY/$ORG/docs:hardened"
depends_on: [build-docs]
- name: scan-api
@@ -108,7 +116,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL
--exit-code 0 --format json -o api-scan.json
"gitea.homelab.fhirworx.io/homelab/api:hardened"
"$REGISTRY/$ORG/api:hardened"
depends_on: [build-api]
# ── Close resolved vuln issues OR file new ones ────────────

39
.woodpecker/rebuild-all.yml
View File

@@ -3,17 +3,23 @@ when:
variables:
- &buildx_image woodpeckerci/plugin-docker-buildx:5-insecure
- &registry gitea.homelab.fhirworx.io
- &org homelab
- &image_prefix fhirworx
- &buildx_base
buildkit_config: |
[registry."gitea.homelab.fhirworx.io"]
http = true
registry: gitea.homelab.fhirworx.io
registry: *registry
username:
from_secret: registry_user
password:
from_secret: registry_pass
- &trivy_scan
image: aquasec/trivy:latest
environment:
REGISTRY: *registry
ORG: *org
- &diag_env
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
@@ -46,11 +52,12 @@ steps:
- name: prep-docs-context
image: alpine:3
volumes:
- /home/kert/stack/data:/host-data:ro
environment:
STACK_ROOT:
from_secret: stack_root
commands:
- mkdir -p data
- cp /host-data/bib.sqlite data/ 2>/dev/null || true
- cp ${STACK_ROOT:-/home/kert/stack}/data/bib.sqlite data/ 2>/dev/null || true
- name: build-push-docs
image: *buildx_image
@@ -92,7 +99,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
-o notebooks-scan.json
"gitea.homelab.fhirworx.io/homelab/notebooks:sha-${CI_COMMIT_SHA:0:8}"
"$REGISTRY/$ORG/notebooks:${CI_COMMIT_SHA:0:8}"
depends_on:
- build-push-notebooks
@@ -101,7 +108,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
-o zotero-scan.json
"gitea.homelab.fhirworx.io/homelab/zotero:sha-${CI_COMMIT_SHA:0:8}"
"$REGISTRY/$ORG/zotero:${CI_COMMIT_SHA:0:8}"
depends_on:
- build-push-zotero
@@ -110,7 +117,7 @@ steps:
commands:
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
-o api-scan.json
"gitea.homelab.fhirworx.io/homelab/api:sha-${CI_COMMIT_SHA:0:8}"
"$REGISTRY/$ORG/api:${CI_COMMIT_SHA:0:8}"
depends_on:
- build-push-api
@@ -132,19 +139,27 @@ steps:
- name: deploy
image: docker:cli
environment:
REGISTRY: *registry
ORG: *org
IMAGE_PREFIX: *image_prefix
STACK_ROOT:
from_secret: stack_root
volumes:
- /run/user/1000/docker.sock:/var/run/docker.sock
- /home/kert/stack:/home/kert/stack
commands:
- cd /home/kert/stack
- STACK_ROOT=${STACK_ROOT:-/home/kert/stack}
- cd $STACK_ROOT
- TAG=${CI_COMMIT_SHA:0:8}
- FQDN=gitea.homelab.fhirworx.io
- for SVC in notebooks zotero docs api mc; do
docker pull $FQDN/homelab/$SVC:$TAG &&
docker tag $FQDN/homelab/$SVC:$TAG fhirworx/$SVC:$TAG;
docker pull $REGISTRY/$ORG/$SVC:$TAG &&
docker tag $REGISTRY/$ORG/$SVC:$TAG $IMAGE_PREFIX/$SVC:$TAG;
done
- sed -i "s/^COMMIT_SHA=.*/COMMIT_SHA=$TAG/" .env 2>/dev/null || echo "COMMIT_SHA=$TAG" >> .env
- docker compose up -d --remove-orphans
volumes:
- /run/user/1000/docker.sock:/var/run/docker.sock
- /home/kert/stack:/home/kert/stack
depends_on:
- scan-notebooks
- scan-zotero

2
src/aco/load/stage.py
View File

@@ -106,7 +106,7 @@ def _promote_to_iceberg(
catalog_uri=lake.nessie.catalog_uri,
warehouse=lake.warehouse,
properties={
"s3.endpoint": cfg.lake.get("s3_endpoint", "http://rustfs:9000"),
"s3.endpoint": cfg.lake.get("s3_endpoint", cfg.services.rustfs),
"s3.path-style-access": "true",
},
)

14
src/api/auth/deploy.py
View File

@@ -69,11 +69,13 @@ def verify_postgres(
return errors
def verify_gitea(
values: dict[str, str], *, base_url: str = "http://gitea:3000/api/v1"
) -> list[str]:
def verify_gitea(values: dict[str, str], *, base_url: str = "") -> list[str]:
"""Verify Gitea API is reachable with the current token."""
errors = []
if not base_url:
from conf import cfg
base_url = f"{cfg.services.gitea}/api/v1"
token = values.get("GITEA_TOKEN", "")
if not token:
errors.append("gitea: no GITEA_TOKEN")
@@ -92,8 +94,12 @@ def verify_gitea(
return errors
def verify_rustfs(*, endpoint: str = "http://rustfs:9000") -> list[str]:
def verify_rustfs(*, endpoint: str = "") -> list[str]:
"""Verify RustFS is reachable (returns 403 = alive, needs auth)."""
if not endpoint:
from conf import cfg
endpoint = cfg.services.rustfs
errors = []
try:
resp = httpx.get(f"{endpoint}/minio/health/live", timeout=10)

12
src/api/auth/provision.py
View File

@@ -209,12 +209,16 @@ def provision_gitea(
values: dict[str, str],
*,
admin_user: str = "kert",
base_url: str = "http://gitea:3000/api/v1",
base_url: str = "",
) -> str:
"""Rotate Gitea admin password and create a fresh API token.
Uses the Gitea HTTP API exclusively — no docker exec needed.
"""
if not base_url:
from conf import cfg
base_url = f"{cfg.services.gitea}/api/v1"
pw = values["GITEA_ADMIN_PASSWORD"]
current_token = values.get("GITEA_TOKEN", "")
@@ -297,9 +301,13 @@ def provision_woodpecker(
values: dict[str, str],
*,
repo_id: int = 1,
base_url: str = "http://woodpecker-server:8000/api",
base_url: str = "",
) -> None:
"""Sync rotated credentials to Woodpecker repo secrets."""
if not base_url:
from conf import cfg
base_url = f"{cfg.services.woodpecker}/api"
token = _get_woodpecker_token(values)
wp = WoodpeckerClient(token, base_url=base_url)

16
src/api/clients/gitea/client.py
View File

@@ -20,9 +20,13 @@ class GiteaClient(Client):
self,
token: str,
*,
base_url: str = "http://gitea:3000/api/v1",
base_url: str = "",
**kw,
) -> None:
if not base_url:
from conf import cfg
base_url = f"{cfg.services.gitea}/api/v1"
super().__init__(base_url, **kw)
self._token = token
@@ -94,3 +98,13 @@ class GiteaClient(Client):
def create_issue(self, owner: str, repo: str, body: dict) -> dict:
return self.post(f"/repos/{owner}/{repo}/issues", json=body).json()
def list_labels(self, owner: str, repo: str) -> list[dict]:
return self.get(f"/repos/{owner}/{repo}/labels", params={"limit": 50}).json()
def resolve_labels(self, owner: str, repo: str, names: list[str]) -> list[int]:
"""Resolve label names to IDs. Unknown names are silently skipped."""
if not hasattr(self, "_label_cache"):
labels = self.list_labels(owner, repo)
self._label_cache = {l["name"]: l["id"] for l in labels}
return [self._label_cache[n] for n in names if n in self._label_cache]

12
src/api/clients/rustfs/client.py
View File

@@ -30,9 +30,13 @@ class RustFSClient(Client):
access_key: str,
secret_key: str,
*,
base_url: str = "http://rustfs:9000",
base_url: str = "",
**kw,
) -> None:
if not base_url:
from conf import cfg
base_url = cfg.services.rustfs
super().__init__(base_url, **kw)
self._access_key = access_key
self._secret_key = secret_key
@@ -103,9 +107,13 @@ class RustFSAdmin(Client):
access_key: str,
secret_key: str,
*,
base_url: str = "http://rustfs:9000/minio/v2",
base_url: str = "",
**kw,
) -> None:
if not base_url:
from conf import cfg
base_url = f"{cfg.services.rustfs}/minio/v2"
super().__init__(base_url, **kw)
self._access_key = access_key
self._secret_key = secret_key

6
src/api/clients/woodpecker/client.py
View File

@@ -20,9 +20,13 @@ class WoodpeckerClient(Client):
self,
token: str,
*,
base_url: str = "http://woodpecker-server:8000/api",
base_url: str = "",
**kw,
) -> None:
if not base_url:
from conf import cfg
base_url = f"{cfg.services.woodpecker}/api"
super().__init__(base_url, **kw)
self._token = token

81
src/api/diag/__main__.py
View File

@@ -16,40 +16,23 @@ import logging
import os
import sys
from conf import cfg
log = logging.getLogger(__name__)
# ── Label ID mapping ──────────────────────────────────────────────
# These IDs correspond to labels created in the Gitea repo.
PIPELINE_LABELS: dict[str, int] = {
"ci": 14,
"deploy": 15,
"harden": 16,
"infra-ci": 17,
"rebuild-all": 18,
"release": 19,
}
STEP_LABELS: dict[str, int] = {
"lint": 20,
"test": 21,
"build": 22,
"scan": 23,
"deploy": 24,
"publish": 25,
"hadolint": 26,
"validate": 27,
}
IMAGE_LABELS: dict[str, int] = {
"api": 28,
"notebooks": 29,
"zotero": 30,
"docs": 31,
"mc": 32,
}
INFRA_LABEL = 33
CI_LABEL = 13 # existing "ci" label
# ── Label name patterns ───────────────────────────────────────────
PIPELINE_NAMES = ["ci", "deploy", "harden", "infra-ci", "rebuild-all", "release"]
STEP_KEYWORDS = [
"lint",
"test",
"build",
"scan",
"deploy",
"publish",
"hadolint",
"validate",
]
IMAGE_KEYWORDS = ["api", "notebooks", "zotero", "docs", "mc"]
def _get_env(name: str) -> str:
@@ -59,30 +42,30 @@ def _get_env(name: str) -> str:
return val
def _resolve_labels(pipeline_name: str, step_name: str) -> list[int]:
"""Resolve label IDs from pipeline and step names."""
labels: list[int] = [CI_LABEL]
def _resolve_label_names(pipeline_name: str, step_name: str) -> list[str]:
"""Resolve label names from pipeline and step names."""
names: list[str] = ["ci"]
# Pipeline label
for key, lid in PIPELINE_LABELS.items():
for key in PIPELINE_NAMES:
if key in pipeline_name:
labels.append(lid)
names.append(f"pipeline:{key}")
break
# Step category label — match the broadest category
# Step category label
step_lower = step_name.lower()
for key, lid in STEP_LABELS.items():
for key in STEP_KEYWORDS:
if key in step_lower:
labels.append(lid)
names.append(f"step:{key}")
break
# Image label — detect which image the step relates to
for key, lid in IMAGE_LABELS.items():
# Image label
for key in IMAGE_KEYWORDS:
if key in step_lower:
labels.append(lid)
names.append(f"image:{key}")
break
return labels
return names
def _pipeline_name_from_env() -> str:
@@ -126,7 +109,8 @@ def main() -> int:
if "/" in repo_name:
owner, repo = repo_name.split("/", 1)
else:
owner, repo = "homelab", "stack"
parts = cfg.platform.repo.split("/", 1)
owner, repo = parts[0], parts[1]
pipeline_name = _pipeline_name_from_env()
@@ -135,9 +119,7 @@ def main() -> int:
wp = WoodpeckerClient(
gitea_token,
base_url=os.environ.get(
"CI_WOODPECKER_URL", "http://woodpecker-server:8000/api"
),
base_url=os.environ.get("CI_WOODPECKER_URL", f"{cfg.services.woodpecker}/api"),
)
rid = int(repo_id)
@@ -167,7 +149,8 @@ def main() -> int:
log.info("Processing failed step: %s (id=%d)", step_name, step_id)
labels = _resolve_labels(pipeline_name, step_name)
label_names = _resolve_label_names(pipeline_name, step_name)
labels = gitea.resolve_labels(owner, repo, label_names)
# Fetch logs — Woodpecker returns a list of log line dicts
try:

17
stack.toml
View File

@@ -13,6 +13,23 @@
# the [db], [storage], and [lake] sections supply the concrete
# connection details for the selected backend.
[platform]
domain = "homelab.fhirworx.io"
registry = "gitea.homelab.fhirworx.io"
image_prefix = "fhirworx"
repo = "homelab/stack"
ci_email = "ci@fhirworx.io"
host_ip = "192.168.1.192"
[services]
gitea = "http://gitea:3000"
woodpecker = "http://woodpecker-server:8000"
rustfs = "http://rustfs:9000"
nessie = "http://nessie:19120"
polaris = "http://polaris:8181"
trino = "http://trino:8080"
grafana = "http://grafana:3000"
[context]
active = "local"