P18: stack.toml drives ALL generated config via gen_config.py
Templates in .woodpecker/templates/ use __REGISTRY__, __ORG__, __IMAGE_PREFIX__ placeholders. gen_config.py reads stack.toml [platform] section and renders: - .woodpecker/deploy.yml, harden.yml, rebuild-all.yml - coredns/hosts, coredns/Corefile Change the domain in stack.toml, run gen_config.py, done. `--check` mode verifies files are up-to-date (CI-friendly). Replaces gen_coredns.py with unified gen_config.py.
This commit is contained in:
kert
291
.woodpecker/templates/deploy.yml
Normal file
291
.woodpecker/templates/deploy.yml
Normal file
@@ -0,0 +1,291 @@
|
||||
# ── Deploy ───────────────────────────────────────────────────────
|
||||
# Builds the Python package, pushes it to Gitea's PyPI registry,
|
||||
# builds, scans, and pushes container images, then updates the
|
||||
# host working copy and restarts all changed services.
|
||||
# Only runs on pushes to main (i.e. after PR merge).
|
||||
#
|
||||
# Image naming:
|
||||
# __REGISTRY__/__ORG__/<service>:<8chars>
|
||||
#
|
||||
# Buildkit pushes via HTTP through Traefik (TLS terminated at edge).
|
||||
# DNS resolved by CoreDNS via Traefik UDP on the CI network.
|
||||
|
||||
when:
|
||||
- event: push
|
||||
branch: main
|
||||
|
||||
variables:
|
||||
- &buildx_image woodpeckerci/plugin-docker-buildx:5-insecure
|
||||
- ®istry __REGISTRY__
|
||||
- &org __ORG__
|
||||
- &buildx_base
|
||||
buildkit_config: |
|
||||
[registry."__REGISTRY__"]
|
||||
http = true
|
||||
registry: *registry
|
||||
username:
|
||||
from_secret: registry_user
|
||||
password:
|
||||
from_secret: registry_pass
|
||||
- &trivy_scan
|
||||
image: aquasec/trivy:latest
|
||||
environment:
|
||||
REGISTRY: *registry
|
||||
ORG: *org
|
||||
- &diag_env
|
||||
UV_PYTHON_PREFERENCE: only-system
|
||||
UV_LINK_MODE: copy
|
||||
UV_PROJECT_ENVIRONMENT: .venv
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
|
||||
steps:
|
||||
# ── Python package ──────────────────────────────────────────
|
||||
- name: build-package
|
||||
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
commands:
|
||||
- BASE=$(grep '^version' pyproject.toml | head -1 | sed 's/.*"\(.*\)"/\1/')
|
||||
- uv version "$BASE.dev${CI_PIPELINE_NUMBER}" --no-sync
|
||||
- uv build --out-dir dist/
|
||||
- ls -lh dist/
|
||||
|
||||
- name: publish-package
|
||||
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
environment:
|
||||
REGISTRY_USER:
|
||||
from_secret: registry_user
|
||||
REGISTRY_PASS:
|
||||
from_secret: registry_pass
|
||||
commands:
|
||||
- uv publish --publish-url http://gitea:3000/api/packages/${CI_REPO_OWNER}/pypi --username "$REGISTRY_USER" --password "$REGISTRY_PASS" dist/*
|
||||
depends_on:
|
||||
- build-package
|
||||
|
||||
# ── Notebooks image ─────────────────────────────────────────
|
||||
- name: build-push-notebooks
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/notebooks
|
||||
dockerfile: notebooks/Dockerfile
|
||||
context: notebooks/
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: scan-notebooks
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
|
||||
-o notebooks-scan.json
|
||||
"$REGISTRY/$ORG/notebooks:${CI_COMMIT_SHA:0:8}"
|
||||
depends_on:
|
||||
- build-push-notebooks
|
||||
|
||||
# ── Zotero image ────────────────────────────────────────────
|
||||
- name: build-push-zotero
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/zotero
|
||||
dockerfile: zotero/Dockerfile
|
||||
context: zotero/
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: scan-zotero
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
|
||||
-o zotero-scan.json
|
||||
"$REGISTRY/$ORG/zotero:${CI_COMMIT_SHA:0:8}"
|
||||
depends_on:
|
||||
- build-push-zotero
|
||||
|
||||
# ── Docs image ──────────────────────────────────────────────
|
||||
- name: prep-docs-context
|
||||
image: alpine:3
|
||||
volumes:
|
||||
- ${CI_WORKSPACE}/data:/host-data:ro
|
||||
commands:
|
||||
- mkdir -p data
|
||||
- cp /host-data/bib.sqlite data/ 2>/dev/null || true
|
||||
|
||||
- name: build-push-docs
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/docs
|
||||
dockerfile: docs/Dockerfile
|
||||
context: .
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
depends_on:
|
||||
- prep-docs-context
|
||||
|
||||
- name: scan-docs
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
|
||||
-o docs-scan.json
|
||||
"$REGISTRY/$ORG/docs:${CI_COMMIT_SHA:0:8}"
|
||||
depends_on:
|
||||
- build-push-docs
|
||||
|
||||
# ── API image ───────────────────────────────────────────────
|
||||
- name: build-push-api
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/api
|
||||
dockerfile: api/Dockerfile
|
||||
context: .
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: scan-api
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
|
||||
-o api-scan.json
|
||||
"$REGISTRY/$ORG/api:${CI_COMMIT_SHA:0:8}"
|
||||
depends_on:
|
||||
- build-push-api
|
||||
|
||||
# ── MC (MinIO Client) sidecar ────────────────────────────────
|
||||
- name: build-push-mc
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/mc
|
||||
dockerfile: rustfs/Dockerfile.mc
|
||||
context: rustfs/
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
# ── Upload scan results ─────────────────────────────────────
|
||||
- name: upload-notebooks-scan
|
||||
image: woodpeckerci/plugin-s3
|
||||
failure: ignore
|
||||
settings:
|
||||
endpoint: http://rustfs:9000
|
||||
bucket: gitea
|
||||
access_key:
|
||||
from_secret: s3_access_key
|
||||
secret_key:
|
||||
from_secret: s3_secret_key
|
||||
source: "notebooks-scan.json"
|
||||
target: /ci/${CI_REPO}/${CI_COMMIT_SHA:0:8}/
|
||||
path_style: true
|
||||
depends_on:
|
||||
- scan-notebooks
|
||||
|
||||
- name: upload-zotero-scan
|
||||
image: woodpeckerci/plugin-s3
|
||||
failure: ignore
|
||||
settings:
|
||||
endpoint: http://rustfs:9000
|
||||
bucket: gitea
|
||||
access_key:
|
||||
from_secret: s3_access_key
|
||||
secret_key:
|
||||
from_secret: s3_secret_key
|
||||
source: "zotero-scan.json"
|
||||
target: /ci/${CI_REPO}/${CI_COMMIT_SHA:0:8}/
|
||||
path_style: true
|
||||
depends_on:
|
||||
- scan-zotero
|
||||
|
||||
- name: upload-docs-scan
|
||||
image: woodpeckerci/plugin-s3
|
||||
failure: ignore
|
||||
settings:
|
||||
endpoint: http://rustfs:9000
|
||||
bucket: gitea
|
||||
access_key:
|
||||
from_secret: s3_access_key
|
||||
secret_key:
|
||||
from_secret: s3_secret_key
|
||||
source: "docs-scan.json"
|
||||
target: /ci/${CI_REPO}/${CI_COMMIT_SHA:0:8}/
|
||||
path_style: true
|
||||
depends_on:
|
||||
- scan-docs
|
||||
|
||||
- name: upload-api-scan
|
||||
image: woodpeckerci/plugin-s3
|
||||
failure: ignore
|
||||
settings:
|
||||
endpoint: http://rustfs:9000
|
||||
bucket: gitea
|
||||
access_key:
|
||||
from_secret: s3_access_key
|
||||
secret_key:
|
||||
from_secret: s3_secret_key
|
||||
source: "api-scan.json"
|
||||
target: /ci/${CI_REPO}/${CI_COMMIT_SHA:0:8}/
|
||||
path_style: true
|
||||
depends_on:
|
||||
- scan-api
|
||||
|
||||
# ── Report vulnerabilities ──────────────────────────────────
|
||||
- name: report-vulns
|
||||
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
failure: ignore
|
||||
environment:
|
||||
<<: *diag_env
|
||||
commands:
|
||||
- uv sync --no-dev
|
||||
- |
|
||||
for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json; do
|
||||
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
|
||||
done
|
||||
depends_on:
|
||||
- scan-notebooks
|
||||
- scan-zotero
|
||||
- scan-docs
|
||||
- scan-api
|
||||
|
||||
# ── Deploy (disabled — manual only) ────────────────────────────
|
||||
# Deploy is disabled until rootless Docker compose-from-CI issues
|
||||
# are resolved. Run manually: docker compose up -d
|
||||
# See Gitea issue #118 (RCA) for details.
|
||||
#
|
||||
# To deploy manually after a green pipeline:
|
||||
# cd /home/kert/stack
|
||||
# TAG=sha-$(git rev-parse --short HEAD)
|
||||
# FQDN=__REGISTRY__
|
||||
# for SVC in notebooks zotero docs api mc; do
|
||||
# docker pull $FQDN/__ORG__/$SVC:$TAG &&
|
||||
# docker tag $FQDN/__ORG__/$SVC:$TAG __IMAGE_PREFIX__/$SVC:$TAG
|
||||
# done
|
||||
# sed -i "s/^COMMIT_SHA=.*/COMMIT_SHA=$TAG/" .env
|
||||
# docker compose up -d notebooks zotero docs api mc
|
||||
- name: deploy
|
||||
image: alpine:3
|
||||
commands:
|
||||
- echo "Deploy disabled — run manually after green pipeline (see comments above)"
|
||||
depends_on:
|
||||
- publish-package
|
||||
- scan-notebooks
|
||||
- scan-zotero
|
||||
- scan-docs
|
||||
- scan-api
|
||||
- build-push-mc
|
||||
|
||||
# ── Failure reporter ─────────────────────────────────────────
|
||||
- name: report-failure
|
||||
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
failure: ignore
|
||||
environment:
|
||||
<<: *diag_env
|
||||
commands:
|
||||
- uv sync --no-dev
|
||||
- uv run python -m api.diag
|
||||
when:
|
||||
- status: [failure]
|
||||
|
||||
152
.woodpecker/templates/harden.yml
Normal file
152
.woodpecker/templates/harden.yml
Normal file
@@ -0,0 +1,152 @@
|
||||
# ── Hardening pipeline ────────────────────────────────────────────
|
||||
# Rebuilds all images with apt-get/apk upgrade, scans for vulns,
|
||||
# and auto-closes Gitea vuln issues when scans are clean.
|
||||
# Trigger: manual or weekly cron.
|
||||
#
|
||||
# Registry host is defined ONCE as ®istry — change it here to
|
||||
# retarget all builds and scans.
|
||||
|
||||
when:
|
||||
- event: [manual, cron]
|
||||
|
||||
variables:
|
||||
- &buildx_image woodpeckerci/plugin-docker-buildx:5-insecure
|
||||
- ®istry __REGISTRY__
|
||||
- &org __ORG__
|
||||
- &buildx_base
|
||||
buildkit_config: |
|
||||
[registry."__REGISTRY__"]
|
||||
http = true
|
||||
registry: *registry
|
||||
username:
|
||||
from_secret: registry_user
|
||||
password:
|
||||
from_secret: registry_pass
|
||||
- &trivy_scan
|
||||
image: aquasec/trivy:latest
|
||||
environment:
|
||||
REGISTRY: *registry
|
||||
ORG: *org
|
||||
- &diag_env
|
||||
UV_PYTHON_PREFERENCE: only-system
|
||||
UV_LINK_MODE: copy
|
||||
UV_PROJECT_ENVIRONMENT: .venv
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
|
||||
steps:
|
||||
# ── Build hardened images ──────────────────────────────────
|
||||
- name: build-notebooks
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/notebooks
|
||||
dockerfile: notebooks/Dockerfile
|
||||
context: notebooks/
|
||||
tags: [hardened, latest]
|
||||
no_cache: true
|
||||
|
||||
- name: build-zotero
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/zotero
|
||||
dockerfile: zotero/Dockerfile
|
||||
context: zotero/
|
||||
tags: [hardened, latest]
|
||||
no_cache: true
|
||||
|
||||
- name: build-docs
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/docs
|
||||
dockerfile: docs/Dockerfile
|
||||
context: .
|
||||
tags: [hardened, latest]
|
||||
no_cache: true
|
||||
|
||||
- name: build-api
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/api
|
||||
dockerfile: api/Dockerfile
|
||||
context: .
|
||||
tags: [hardened, latest]
|
||||
no_cache: true
|
||||
|
||||
- name: build-mc
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/mc
|
||||
dockerfile: rustfs/Dockerfile.mc
|
||||
context: rustfs/
|
||||
tags: [hardened, latest]
|
||||
no_cache: true
|
||||
|
||||
# ── Scan hardened images ───────────────────────────────────
|
||||
- name: scan-notebooks
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL
|
||||
--exit-code 0 --format json -o notebooks-scan.json
|
||||
"$REGISTRY/$ORG/notebooks:hardened"
|
||||
depends_on: [build-notebooks]
|
||||
|
||||
- name: scan-zotero
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL
|
||||
--exit-code 0 --format json -o zotero-scan.json
|
||||
"$REGISTRY/$ORG/zotero:hardened"
|
||||
depends_on: [build-zotero]
|
||||
|
||||
- name: scan-docs
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL
|
||||
--exit-code 0 --format json -o docs-scan.json
|
||||
"$REGISTRY/$ORG/docs:hardened"
|
||||
depends_on: [build-docs]
|
||||
|
||||
- name: scan-api
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL
|
||||
--exit-code 0 --format json -o api-scan.json
|
||||
"$REGISTRY/$ORG/api:hardened"
|
||||
depends_on: [build-api]
|
||||
|
||||
# ── Close resolved vuln issues OR file new ones ────────────
|
||||
- name: close-or-report-vulns
|
||||
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
environment:
|
||||
<<: *diag_env
|
||||
commands:
|
||||
- uv sync --no-dev
|
||||
- |
|
||||
for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json; do
|
||||
if [ -f "$f" ]; then
|
||||
uv run python -m api.diag.vuln --close "$f" || \
|
||||
uv run python -m api.diag.vuln "$f" || true
|
||||
fi
|
||||
done
|
||||
depends_on:
|
||||
- scan-notebooks
|
||||
- scan-zotero
|
||||
- scan-docs
|
||||
- scan-api
|
||||
|
||||
# ── Failure reporter ─────────────────────────────────────────
|
||||
- name: report-failure
|
||||
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
failure: ignore
|
||||
environment:
|
||||
<<: *diag_env
|
||||
commands:
|
||||
- uv sync --no-dev
|
||||
- uv run python -m api.diag
|
||||
when:
|
||||
- status: [failure]
|
||||
180
.woodpecker/templates/rebuild-all.yml
Normal file
180
.woodpecker/templates/rebuild-all.yml
Normal file
@@ -0,0 +1,180 @@
|
||||
when:
|
||||
- event: manual
|
||||
|
||||
variables:
|
||||
- &buildx_image woodpeckerci/plugin-docker-buildx:5-insecure
|
||||
- ®istry __REGISTRY__
|
||||
- &org __ORG__
|
||||
- &image_prefix __IMAGE_PREFIX__
|
||||
- &buildx_base
|
||||
buildkit_config: |
|
||||
[registry."__REGISTRY__"]
|
||||
http = true
|
||||
registry: *registry
|
||||
username:
|
||||
from_secret: registry_user
|
||||
password:
|
||||
from_secret: registry_pass
|
||||
- &trivy_scan
|
||||
image: aquasec/trivy:latest
|
||||
environment:
|
||||
REGISTRY: *registry
|
||||
ORG: *org
|
||||
- &diag_env
|
||||
UV_PYTHON_PREFERENCE: only-system
|
||||
UV_LINK_MODE: copy
|
||||
UV_PROJECT_ENVIRONMENT: .venv
|
||||
GITEA_TOKEN:
|
||||
from_secret: gitea_token
|
||||
|
||||
steps:
|
||||
- name: build-push-notebooks
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/notebooks
|
||||
dockerfile: notebooks/Dockerfile
|
||||
context: notebooks/
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: build-push-zotero
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/zotero
|
||||
dockerfile: zotero/Dockerfile
|
||||
context: zotero/
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: prep-docs-context
|
||||
image: alpine:3
|
||||
environment:
|
||||
STACK_ROOT:
|
||||
from_secret: stack_root
|
||||
commands:
|
||||
- mkdir -p data
|
||||
- cp ${STACK_ROOT:-/home/kert/stack}/data/bib.sqlite data/ 2>/dev/null || true
|
||||
|
||||
- name: build-push-docs
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/docs
|
||||
dockerfile: docs/Dockerfile
|
||||
context: .
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
depends_on:
|
||||
- prep-docs-context
|
||||
|
||||
- name: build-push-api
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/api
|
||||
dockerfile: api/Dockerfile
|
||||
context: .
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: build-push-mc
|
||||
image: *buildx_image
|
||||
settings:
|
||||
<<: *buildx_base
|
||||
repo: __REGISTRY__/__ORG__/mc
|
||||
dockerfile: rustfs/Dockerfile.mc
|
||||
context: rustfs/
|
||||
tags:
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: scan-notebooks
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
|
||||
-o notebooks-scan.json
|
||||
"$REGISTRY/$ORG/notebooks:${CI_COMMIT_SHA:0:8}"
|
||||
depends_on:
|
||||
- build-push-notebooks
|
||||
|
||||
- name: scan-zotero
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
|
||||
-o zotero-scan.json
|
||||
"$REGISTRY/$ORG/zotero:${CI_COMMIT_SHA:0:8}"
|
||||
depends_on:
|
||||
- build-push-zotero
|
||||
|
||||
- name: scan-api
|
||||
<<: *trivy_scan
|
||||
commands:
|
||||
- trivy image --image-src remote --insecure --severity HIGH,CRITICAL --exit-code 0 --format json
|
||||
-o api-scan.json
|
||||
"$REGISTRY/$ORG/api:${CI_COMMIT_SHA:0:8}"
|
||||
depends_on:
|
||||
- build-push-api
|
||||
|
||||
- name: report-vulns
|
||||
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
failure: ignore
|
||||
environment:
|
||||
<<: *diag_env
|
||||
commands:
|
||||
- uv sync --no-dev
|
||||
- |
|
||||
for f in notebooks-scan.json zotero-scan.json api-scan.json; do
|
||||
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
|
||||
done
|
||||
depends_on:
|
||||
- scan-notebooks
|
||||
- scan-zotero
|
||||
- scan-api
|
||||
|
||||
- name: deploy
|
||||
image: docker:cli
|
||||
environment:
|
||||
REGISTRY: *registry
|
||||
ORG: *org
|
||||
IMAGE_PREFIX: *image_prefix
|
||||
STACK_ROOT:
|
||||
from_secret: stack_root
|
||||
volumes:
|
||||
- /run/user/1000/docker.sock:/var/run/docker.sock
|
||||
commands:
|
||||
- STACK_ROOT=${STACK_ROOT:-/home/kert/stack}
|
||||
- cd $STACK_ROOT
|
||||
- TAG=${CI_COMMIT_SHA:0:8}
|
||||
- for SVC in notebooks zotero docs api mc; do
|
||||
docker pull $REGISTRY/$ORG/$SVC:$TAG &&
|
||||
docker tag $REGISTRY/$ORG/$SVC:$TAG $IMAGE_PREFIX/$SVC:$TAG;
|
||||
done
|
||||
- sed -i "s/^COMMIT_SHA=.*/COMMIT_SHA=$TAG/" .env 2>/dev/null || echo "COMMIT_SHA=$TAG" >> .env
|
||||
- docker compose up -d --remove-orphans
|
||||
volumes:
|
||||
- /run/user/1000/docker.sock:/var/run/docker.sock
|
||||
- ${STACK_ROOT:-/home/kert/stack}:${STACK_ROOT:-/home/kert/stack}
|
||||
depends_on:
|
||||
- scan-notebooks
|
||||
- scan-zotero
|
||||
- scan-api
|
||||
- build-push-docs
|
||||
- build-push-mc
|
||||
|
||||
# ── Failure reporter ─────────────────────────────────────────
|
||||
- name: report-failure
|
||||
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
failure: ignore
|
||||
environment:
|
||||
<<: *diag_env
|
||||
commands:
|
||||
- uv sync --no-dev
|
||||
- uv run python -m api.diag
|
||||
when:
|
||||
- status: [failure]
|
||||
151
dev/scripts/gen_config.py
Normal file
151
dev/scripts/gen_config.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""Generate all derived config files from stack.toml.
|
||||
|
||||
Reads [platform] and [services] from stack.toml and renders:
|
||||
- coredns/hosts + coredns/Corefile
|
||||
- .woodpecker/deploy.yml, harden.yml, rebuild-all.yml
|
||||
|
||||
Usage::
|
||||
|
||||
uv run python dev/scripts/gen_config.py # generate all
|
||||
uv run python dev/scripts/gen_config.py --check # verify files are up-to-date
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from conf import ROOT, cfg
|
||||
|
||||
# ── Platform values from stack.toml ──────────────────────────────
|
||||
|
||||
REGISTRY = cfg.platform.registry
|
||||
ORG = cfg.platform.repo.split("/")[0]
|
||||
IMAGE_PREFIX = cfg.platform.image_prefix
|
||||
DOMAIN = cfg.platform.domain
|
||||
HOST_IP = cfg.platform.host_ip
|
||||
|
||||
# ── Placeholder pattern ─────────────────────────────────────────
|
||||
|
||||
# Templates use __REGISTRY__, __ORG__, __IMAGE_PREFIX__, __DOMAIN__, __HOST_IP__
|
||||
REPLACEMENTS = {
|
||||
"__REGISTRY__": REGISTRY,
|
||||
"__ORG__": ORG,
|
||||
"__IMAGE_PREFIX__": IMAGE_PREFIX,
|
||||
"__DOMAIN__": DOMAIN,
|
||||
"__HOST_IP__": HOST_IP,
|
||||
}
|
||||
|
||||
|
||||
def _render(template: str) -> str:
|
||||
"""Replace all __PLACEHOLDER__ tokens in a template string."""
|
||||
result = template
|
||||
for token, value in REPLACEMENTS.items():
|
||||
result = result.replace(token, value)
|
||||
return result
|
||||
|
||||
|
||||
# ── CoreDNS ──────────────────────────────────────────────────────
|
||||
|
||||
SUBDOMAINS = [
|
||||
"", "gitea", "ci", "docs", "notebooks", "api", "s3", "s3console",
|
||||
"grafana", "traefik", "zotero", "nessie", "polaris", "trino",
|
||||
"prometheus", "jaeger", "loki",
|
||||
]
|
||||
|
||||
|
||||
def gen_coredns() -> dict[str, str]:
|
||||
"""Return {filename: content} for coredns files."""
|
||||
lines = []
|
||||
for sub in SUBDOMAINS:
|
||||
fqdn = f"{sub}.{DOMAIN}" if sub else DOMAIN
|
||||
lines.append(f"{HOST_IP} {fqdn}")
|
||||
hosts = "\n".join(lines) + "\n"
|
||||
|
||||
corefile = f"""{DOMAIN} {{
|
||||
hosts /etc/coredns/hosts {{
|
||||
fallthrough
|
||||
}}
|
||||
log
|
||||
}}
|
||||
|
||||
. {{
|
||||
forward . 1.1.1.1 8.8.8.8
|
||||
cache 300
|
||||
}}
|
||||
"""
|
||||
return {
|
||||
"coredns/hosts": hosts,
|
||||
"coredns/Corefile": corefile,
|
||||
}
|
||||
|
||||
|
||||
# ── Woodpecker pipelines ────────────────────────────────────────
|
||||
|
||||
def gen_woodpecker() -> dict[str, str]:
|
||||
"""Return {filename: content} for .woodpecker pipeline files."""
|
||||
templates_dir = ROOT / ".woodpecker" / "templates"
|
||||
results = {}
|
||||
for tmpl in sorted(templates_dir.glob("*.yml")):
|
||||
content = _render(tmpl.read_text())
|
||||
results[f".woodpecker/{tmpl.name}"] = content
|
||||
return results
|
||||
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────
|
||||
|
||||
def generate() -> dict[str, str]:
|
||||
"""Generate all derived config files. Returns {path: content}."""
|
||||
files = {}
|
||||
files.update(gen_coredns())
|
||||
files.update(gen_woodpecker())
|
||||
return files
|
||||
|
||||
|
||||
def write_all(files: dict[str, str]) -> None:
|
||||
for relpath, content in sorted(files.items()):
|
||||
out = ROOT / relpath
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(content)
|
||||
print(f" wrote {relpath}")
|
||||
|
||||
|
||||
def check_all(files: dict[str, str]) -> list[str]:
|
||||
"""Return list of files that differ from expected content."""
|
||||
diffs = []
|
||||
for relpath, expected in sorted(files.items()):
|
||||
out = ROOT / relpath
|
||||
if not out.exists():
|
||||
diffs.append(f"{relpath}: missing")
|
||||
elif out.read_text() != expected:
|
||||
diffs.append(f"{relpath}: out of date")
|
||||
return diffs
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Generate derived config from stack.toml")
|
||||
parser.add_argument("--check", action="store_true", help="Verify files are up-to-date")
|
||||
args = parser.parse_args()
|
||||
|
||||
files = generate()
|
||||
|
||||
if args.check:
|
||||
diffs = check_all(files)
|
||||
if diffs:
|
||||
print("Files out of date:")
|
||||
for d in diffs:
|
||||
print(f" {d}")
|
||||
print("\nRun: uv run python dev/scripts/gen_config.py")
|
||||
return 1
|
||||
print("All generated files are up-to-date.")
|
||||
return 0
|
||||
|
||||
write_all(files)
|
||||
print(f"\nGenerated {len(files)} files from stack.toml")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,67 +0,0 @@
|
||||
"""Generate coredns/hosts and coredns/Corefile from stack.toml.
|
||||
|
||||
Usage::
|
||||
|
||||
uv run python dev/scripts/gen_coredns.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conf import ROOT, cfg
|
||||
|
||||
SUBDOMAINS = [
|
||||
"", # bare domain
|
||||
"gitea",
|
||||
"ci",
|
||||
"docs",
|
||||
"notebooks",
|
||||
"api",
|
||||
"s3",
|
||||
"s3console",
|
||||
"grafana",
|
||||
"traefik",
|
||||
"zotero",
|
||||
"nessie",
|
||||
"polaris",
|
||||
"trino",
|
||||
"prometheus",
|
||||
"jaeger",
|
||||
"loki",
|
||||
]
|
||||
|
||||
OUT_DIR = ROOT / "coredns"
|
||||
|
||||
|
||||
def generate() -> None:
|
||||
domain = cfg.platform.domain
|
||||
host_ip = cfg.platform.host_ip
|
||||
|
||||
# hosts file
|
||||
lines = []
|
||||
for sub in SUBDOMAINS:
|
||||
fqdn = f"{sub}.{domain}" if sub else domain
|
||||
lines.append(f"{host_ip} {fqdn}")
|
||||
(OUT_DIR / "hosts").write_text("\n".join(lines) + "\n")
|
||||
|
||||
# Corefile
|
||||
corefile = f"""{domain} {{
|
||||
hosts /etc/coredns/hosts {{
|
||||
fallthrough
|
||||
}}
|
||||
log
|
||||
}}
|
||||
|
||||
. {{
|
||||
forward . 1.1.1.1 8.8.8.8
|
||||
cache 300
|
||||
}}
|
||||
"""
|
||||
(OUT_DIR / "Corefile").write_text(corefile)
|
||||
|
||||
print(f"Generated coredns/hosts ({len(SUBDOMAINS)} entries) and coredns/Corefile")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate()
|
||||
Reference in New Issue
Block a user