homelab/stack
1
0
Fork 0
Code Issues 20 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

P19: switchable CI backend — fix #142 #143 #144-#149

Browse Source 
stack.toml [ci] section with backend = "woodpecker" | "github".
gen_config.py dispatches to backends/woodpecker.py or
backends/github.py, both implementing emit(images, scans,
platform, ci_cfg) -> dict[str, str].

Woodpecker backend: extracted from monolithic gen_config.py,
all plugin images and docker socket now from [ci.woodpecker].

GitHub backend: generates 6 .github/workflows/*.yml using
actions/checkout, astral-sh/setup-uv, docker/build-push-action,
aquasecurity/trivy-action, softprops/action-gh-release. Matrix
strategy for parallel image builds. GHCR as default registry.

Backend switching: change ci.backend in stack.toml, commit.
gen_config.py emits to the active directory and cleans stale
files from the inactive backend directory.

Pre-commit hook triggers on changes to stack.toml,
gen_config.py, or backends/*.py — stages all generated files.
This commit is contained in:
kert
2026-03-23 20:40:19 -04:00
parent c0514a12cd
commit 55847bbdc1
12 changed files with 1456 additions and 721 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.woodpecker/ci.yml
View File

@@ -1,4 +1,4 @@
# ── Quality gate (generated by gen_config.py) ────────────────────
# ── Quality gate (generated by gen_config.py — woodpecker backend) ──
# DO NOT EDIT — edit stack.toml and run gen_config.py
when:

2
.woodpecker/deploy.yml
View File

@@ -1,4 +1,4 @@
# ── Deploy pipeline (generated by gen_config.py) ────────────────
# ── Deploy pipeline (generated by gen_config.py — woodpecker backend) ──
# DO NOT EDIT — edit stack.toml [images] and run gen_config.py
when:

2
.woodpecker/harden.yml
View File

@@ -1,4 +1,4 @@
# ── Hardening pipeline (generated by gen_config.py) ─────────────
# ── Hardening pipeline (generated by gen_config.py — woodpecker backend) ──
# DO NOT EDIT — edit stack.toml [images] and run gen_config.py
when:

2
.woodpecker/infra-ci.yml
View File

@@ -1,4 +1,4 @@
# ── Infrastructure quality gate (generated by gen_config.py) ────
# ── Infrastructure quality gate (generated by gen_config.py — woodpecker backend) ──
# DO NOT EDIT — edit stack.toml [images] and run gen_config.py
when:

2
.woodpecker/rebuild-all.yml
View File

@@ -1,4 +1,4 @@
# ── Rebuild-all pipeline (generated by gen_config.py) ───────────
# ── Rebuild-all pipeline (generated by gen_config.py — woodpecker backend) ──
# DO NOT EDIT — edit stack.toml [images] and run gen_config.py
when:

2
.woodpecker/release.yml
View File

@@ -1,4 +1,4 @@
# ── Release pipeline (generated by gen_config.py) ────────────────
# ── Release pipeline (generated by gen_config.py — woodpecker backend) ──
# DO NOT EDIT — edit stack.toml and run gen_config.py
when:

6
dev/hooks/pre-commit
View File

@@ -7,12 +7,12 @@ if ! uv run python -c 'import sys' 2>/dev/null; then
uv sync --dev
fi
# Auto-regenerate derived config if stack.toml or templates changed
CONF_CHANGED=$(git diff --cached --name-only -- stack.toml dev/scripts/gen_config.py)
# Auto-regenerate derived config if stack.toml or generator changed
CONF_CHANGED=$(git diff --cached --name-only -- stack.toml 'dev/scripts/gen_config.py' 'dev/scripts/backends/*.py')
if [ -n "$CONF_CHANGED" ]; then
echo "==> pre-commit: regenerating config from stack.toml"
uv run python dev/scripts/gen_config.py
git add .woodpecker/ci.yml .woodpecker/deploy.yml .woodpecker/harden.yml .woodpecker/infra-ci.yml .woodpecker/rebuild-all.yml .woodpecker/release.yml coredns/hosts coredns/Corefile
git add .woodpecker/*.yml .github/workflows/*.yml coredns/hosts coredns/Corefile 2>/dev/null || true
fi
echo "==> pre-commit: ruff check (staged)"

1
dev/scripts/backends/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""CI backend emitters for gen_config.py."""

592
dev/scripts/backends/github.py Normal file
View File

@@ -0,0 +1,592 @@
"""GitHub Actions workflow emitter.
Generates .github/workflows/*.yml from the same image definitions
used by the Woodpecker backend. All output is valid GitHub Actions YAML.
Usage (called by gen_config.py)::
from backends.github import emit
files = emit(images, scans, platform, ci_cfg)
"""
from __future__ import annotations
# ── Constants ─────────────────────────────────────────────────────
_HEADER = (
"# DO NOT EDIT — generated by gen_config.py from stack.toml\n"
"# Re-generate: uv run python dev/scripts/gen_config.py\n"
)
_SHA_SHORT = 'echo "${GITHUB_SHA::8}"'
# ── Helpers ───────────────────────────────────────────────────────
def _indent(text: str, spaces: int = 0) -> str:
"""Indent every line of *text* by *spaces*."""
prefix = " " * spaces
return "\n".join(prefix + line if line else "" for line in text.splitlines())
def _trivy_step(
img: dict,
tag: str,
registry: str,
owner_repo: str,
) -> str:
"""aquasecurity/trivy-action step for one image."""
name = img["name"]
sev = img.get("trivy_severity", "HIGH,CRITICAL")
ec = img.get("trivy_exit_code", 0)
return f"""\
- name: Scan {name}
uses: aquasecurity/trivy-action@master
with:
image-ref: {registry}/{owner_repo}/{name}:{tag}
severity: {sev}
exit-code: "{ec}"
format: json
output: {name}-scan.json"""
def _docker_login_step(registry: str) -> str:
return f"""\
- name: Log in to container registry
uses: docker/login-action@v3
with:
registry: {registry}
username: ${{{{ github.actor }}}}
password: ${{{{ secrets.GITHUB_TOKEN }}}}"""
def _checkout_step() -> str:
return """\
- name: Checkout
uses: actions/checkout@v4"""
def _setup_buildx_step() -> str:
return """\
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3"""
def _setup_uv_step(uv_version: str) -> str:
return f"""\
- name: Set up uv
uses: astral-sh/setup-uv@v4
with:
version: {uv_version}"""
def _build_push_step(
img: dict,
tags_expr: str,
registry: str,
owner_repo: str,
*,
no_cache: bool = False,
load_only: bool = False,
) -> str:
name = img["name"]
push = "false" if load_only else "true"
lines = f"""\
- name: Build {name}
uses: docker/build-push-action@v6
with:
context: {img["context"]}
file: {img["dockerfile"]}
push: {push}
tags: {tags_expr}"""
if load_only:
lines += "\n load: true"
if no_cache:
lines += "\n no-cache: true"
return lines
# ── Workflow generators ───────────────────────────────────────────
def _gen_ci(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
content = f"""\
{_HEADER}
name: CI
on:
push:
branches: ["**"]
pull_request:
jobs:
lint-test:
runs-on: {runner}
steps:
{_checkout_step()}
{_setup_uv_step(uv_version)}
- name: Install dependencies
run: uv sync --dev
- name: Ruff check
run: uv run ruff check src/ tests/ --output-format=concise
- name: Ruff format
run: uv run ruff format --check src/ tests/
- name: Pytest
run: uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=99 -q
- name: Validate generated config
run: uv run python dev/scripts/gen_config.py --check
"""
return (".github/workflows/ci.yml", content)
def _gen_deploy(
images: list[dict],
scans: list[dict],
registry: str,
owner_repo: str,
runner: str,
uv_version: str,
**_kw: object,
) -> tuple[str, str]:
# Matrix include list
matrix_entries = []
for img in images:
matrix_entries.append(
f" - name: {img['name']}\n"
f" dockerfile: {img['dockerfile']}\n"
f" context: {img['context']}"
)
matrix_block = "\n".join(matrix_entries)
# Scan steps (only for scannable images)
scan_steps = []
for img in scans:
scan_steps.append(
_trivy_step(
img,
"${{ env.SHORT_SHA }}",
registry,
owner_repo,
)
)
scan_block = "\n\n".join(scan_steps)
# Report job vuln loop
vuln_loop_files = " ".join(f"{i['name']}-scan.json" for i in scans)
content = f"""\
{_HEADER}
name: Deploy
on:
push:
branches: [main]
jobs:
build-scan:
runs-on: {runner}
strategy:
matrix:
include:
{matrix_block}
steps:
{_checkout_step()}
{_setup_buildx_step()}
{_docker_login_step(registry)}
- name: Compute short SHA
run: echo "SHORT_SHA=${{{{GITHUB_SHA::8}}}}" >> "$GITHUB_ENV"
{
_build_push_step(
{
"name": "${{ matrix.name }}",
"dockerfile": "${{ matrix.dockerfile }}",
"context": "${{ matrix.context }}",
},
"{registry}/{owner_repo}/${{{{ matrix.name }}}}:${{{{ env.SHORT_SHA }}}},{registry}/{owner_repo}/${{{{ matrix.name }}}}:latest".format(
registry=registry, owner_repo=owner_repo
),
registry,
owner_repo,
)
}
scan:
runs-on: {runner}
needs: build-scan
steps:
{_checkout_step()}
- name: Compute short SHA
run: echo "SHORT_SHA=${{{{GITHUB_SHA::8}}}}" >> "$GITHUB_ENV"
{scan_block}
- name: Upload scan results
uses: actions/upload-artifact@v4
with:
name: trivy-scans
path: "*-scan.json"
report-vulns:
runs-on: {runner}
needs: scan
steps:
{_checkout_step()}
{_setup_uv_step(uv_version)}
- name: Download scan results
uses: actions/download-artifact@v4
with:
name: trivy-scans
- name: Report vulnerabilities
env:
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
run: |
uv sync --no-dev
for f in {vuln_loop_files}; do
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
done
"""
return (".github/workflows/deploy.yml", content)
def _gen_harden(
images: list[dict],
scans: list[dict],
registry: str,
owner_repo: str,
runner: str,
uv_version: str,
**_kw: object,
) -> tuple[str, str]:
# Build steps
build_steps = []
for img in images:
tags = f"{registry}/{owner_repo}/{img['name']}:hardened,{registry}/{owner_repo}/{img['name']}:latest"
build_steps.append(
_build_push_step(
img,
tags,
registry,
owner_repo,
no_cache=True,
)
)
build_block = "\n\n".join(build_steps)
# Scan steps
scan_steps = []
for img in scans:
scan_steps.append(_trivy_step(img, "hardened", registry, owner_repo))
scan_block = "\n\n".join(scan_steps)
vuln_loop_files = " ".join(f"{i['name']}-scan.json" for i in scans)
content = f"""\
{_HEADER}
name: Harden
on:
workflow_dispatch:
schedule:
- cron: "0 2 * * 0"
jobs:
build-scan:
runs-on: {runner}
steps:
{_checkout_step()}
{_setup_buildx_step()}
{_docker_login_step(registry)}
{build_block}
scan:
runs-on: {runner}
needs: build-scan
steps:
{_checkout_step()}
{scan_block}
- name: Upload scan results
uses: actions/upload-artifact@v4
with:
name: trivy-scans-harden
path: "*-scan.json"
close-or-report-vulns:
runs-on: {runner}
needs: scan
steps:
{_checkout_step()}
{_setup_uv_step(uv_version)}
- name: Download scan results
uses: actions/download-artifact@v4
with:
name: trivy-scans-harden
- name: Close resolved or file new vuln issues
env:
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
run: |
uv sync --no-dev
for f in {vuln_loop_files}; do
if [ -f "$f" ]; then
uv run python -m api.diag.vuln --close "$f" || \\
uv run python -m api.diag.vuln "$f" || true
fi
done
"""
return (".github/workflows/harden.yml", content)
def _gen_rebuild_all(
images: list[dict],
scans: list[dict],
registry: str,
owner_repo: str,
runner: str,
uv_version: str,
**_kw: object,
) -> tuple[str, str]:
# Build steps
build_steps = []
for img in images:
tags = f"{registry}/{owner_repo}/{img['name']}:${{{{ env.SHORT_SHA }}}},{registry}/{owner_repo}/{img['name']}:latest"
build_steps.append(_build_push_step(img, tags, registry, owner_repo))
build_block = "\n\n".join(build_steps)
# Scan steps
scan_steps = []
for img in scans:
scan_steps.append(
_trivy_step(
img,
"${{ env.SHORT_SHA }}",
registry,
owner_repo,
)
)
scan_block = "\n\n".join(scan_steps)
vuln_loop_files = " ".join(f"{i['name']}-scan.json" for i in scans)
content = f"""\
{_HEADER}
name: Rebuild All
on:
workflow_dispatch:
jobs:
build:
runs-on: {runner}
steps:
{_checkout_step()}
{_setup_buildx_step()}
{_docker_login_step(registry)}
- name: Compute short SHA
run: echo "SHORT_SHA=${{{{GITHUB_SHA::8}}}}" >> "$GITHUB_ENV"
{build_block}
scan:
runs-on: {runner}
needs: build
steps:
{_checkout_step()}
- name: Compute short SHA
run: echo "SHORT_SHA=${{{{GITHUB_SHA::8}}}}" >> "$GITHUB_ENV"
{scan_block}
- name: Upload scan results
uses: actions/upload-artifact@v4
with:
name: trivy-scans-rebuild
path: "*-scan.json"
report-vulns:
runs-on: {runner}
needs: scan
steps:
{_checkout_step()}
{_setup_uv_step(uv_version)}
- name: Download scan results
uses: actions/download-artifact@v4
with:
name: trivy-scans-rebuild
- name: Report vulnerabilities
env:
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
run: |
uv sync --no-dev
for f in {vuln_loop_files}; do
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
done
"""
return (".github/workflows/rebuild-all.yml", content)
def _gen_infra_ci(
images: list[dict],
runner: str,
**_kw: object,
) -> tuple[str, str]:
# Collect all path filters for global trigger
all_paths: list[str] = []
for img in images:
all_paths.extend(img.get("path_filter", []))
paths_block = "\n".join(f" - {p!r}" for p in all_paths) if all_paths else ""
# Per-image hadolint + dry-run build jobs
jobs_block_parts = []
for img in images:
if not img.get("hadolint", True):
continue
name = img["name"]
pf = img.get("path_filter", []) # noqa: F841 — reserved for future per-job filtering
jobs_block_parts.append(f"""\
{name}:
runs-on: {runner}
steps:
{_checkout_step()}
- name: Hadolint {name}
uses: hadolint/hadolint-action@v3.1.0
with:
dockerfile: {img["dockerfile"]}
{_setup_buildx_step()}
{_build_push_step(img, f"ci-test-{name}", "", "", load_only=True)}""")
jobs_block = "\n\n".join(jobs_block_parts)
content = f"""\
{_HEADER}
name: Infra CI
on:
push:
paths:
{paths_block}
pull_request:
paths:
{paths_block}
jobs:
{jobs_block}
"""
return (".github/workflows/infra-ci.yml", content)
def _gen_release(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
content = f"""\
{_HEADER}
name: Release
on:
push:
tags: ["v*"]
jobs:
release:
runs-on: {runner}
permissions:
contents: write
steps:
{_checkout_step()}
{_setup_uv_step(uv_version)}
- name: Build package
run: uv build --out-dir dist/
- name: Create GitHub release
uses: softprops/action-gh-release@v2
with:
files: |
dist/*.whl
dist/*.tar.gz
"""
return (".github/workflows/release.yml", content)
# ── Public API ────────────────────────────────────────────────────
def emit(
images: list[dict],
scans: list[dict],
platform: dict,
ci_cfg: dict,
) -> dict[str, str]:
"""Return {relative_path: content} for all GitHub Actions workflows.
Parameters
----------
images:
Merged image dicts from ``load_images()``.
scans:
Scannable subset (``scan=True``).
platform:
Dict with keys *registry*, *domain*, *host_ip*, *image_prefix*,
*repo* (``"org/name"`` form).
ci_cfg:
Dict from ``cfg.ci.github`` — *runner*, *registry*, *uv_version*.
"""
registry = ci_cfg["registry"]
owner_repo = platform["repo"]
runner = ci_cfg["runner"]
uv_version = ci_cfg["uv_version"]
common = dict(
images=images,
scans=scans,
registry=registry,
owner_repo=owner_repo,
runner=runner,
uv_version=uv_version,
)
files: dict[str, str] = {}
for gen_fn in (
_gen_ci,
_gen_deploy,
_gen_harden,
_gen_rebuild_all,
_gen_infra_ci,
_gen_release,
):
path, content = gen_fn(**common) # type: ignore[arg-type]
files[path] = content
return files

727
dev/scripts/backends/woodpecker.py Normal file
View File

@@ -0,0 +1,727 @@
"""Woodpecker CI backend emitter.
Generates .woodpecker/*.yml pipeline files from image definitions
and platform config.
"""
from __future__ import annotations
def emit(
images: list[dict],
scans: list[dict],
platform: dict,
ci_cfg: dict,
) -> dict[str, str]:
"""Return {relative_path: content} for all Woodpecker pipelines."""
ctx = _Ctx(images, scans, platform, ci_cfg)
return {
".woodpecker/ci.yml": ctx.gen_ci(),
".woodpecker/deploy.yml": ctx.gen_deploy(),
".woodpecker/harden.yml": ctx.gen_harden(),
".woodpecker/infra-ci.yml": ctx.gen_infra_ci(),
".woodpecker/rebuild-all.yml": ctx.gen_rebuild_all(),
".woodpecker/release.yml": ctx.gen_release(),
}
class _Ctx:
"""Holds platform/image context for YAML generation."""
def __init__(
self,
images: list[dict],
scans: list[dict],
platform: dict,
ci_cfg: dict,
) -> None:
self.images = images
self.scans = scans
self.registry = platform["registry"]
self.org = platform["org"]
self.image_prefix = platform["image_prefix"]
self.plugin_buildx = ci_cfg.get(
"plugin_buildx", "woodpeckerci/plugin-docker-buildx:5-insecure"
)
self.plugin_buildx_dryrun = ci_cfg.get(
"plugin_buildx_dryrun", "woodpeckerci/plugin-docker-buildx"
)
self.plugin_s3 = ci_cfg.get("plugin_s3", "woodpeckerci/plugin-s3")
self.plugin_comment = ci_cfg.get(
"plugin_comment", "woodpeckerci/plugin-gitea-comment"
)
self.plugin_release = ci_cfg.get(
"plugin_release", "woodpeckerci/plugin-gitea-release"
)
self.docker_socket = ci_cfg.get("docker_socket", "/run/user/1000/docker.sock")
self.uv_image = ci_cfg.get(
"uv_image", "ghcr.io/astral-sh/uv:python3.13-bookworm-slim"
)
# ── shared fragments ─────────────────────────────────────────
@property
def _variables_block(self) -> str:
return f"""\
variables:
- &buildx_image {self.plugin_buildx}
- &registry {self.registry}
- &org {self.org}
- &image_prefix {self.image_prefix}
- &buildx_base
buildkit_config: |
[registry."{self.registry}"]
http = true
registry: *registry
username:
from_secret: registry_user
password:
from_secret: registry_pass
- &trivy_scan
image: aquasec/trivy:latest
environment:
REGISTRY: *registry
ORG: *org
- &diag_env
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
GITEA_TOKEN:
from_secret: gitea_token
"""
@property
def _failure_reporter(self) -> str:
return f"""\
# ── Failure reporter ─────────────────────────────────────────
- name: report-failure
image: {self.uv_image}
failure: ignore
environment:
<<: *diag_env
commands:
- uv sync --no-dev
- uv run python -m api.diag
when:
- status: [failure]
"""
def _build_step(
self,
img: dict,
tags: list[str],
*,
no_cache: bool = False,
name_prefix: str = "build",
depends: str = "",
) -> str:
n = img["name"]
lines = [
f" - name: {name_prefix}-{n}",
" image: *buildx_image",
" settings:",
" <<: *buildx_base",
f" repo: {self.registry}/{self.org}/{n}",
f" dockerfile: {img['dockerfile']}",
f" context: {img['context']}",
" tags:",
]
for t in tags:
lines.append(f' - "{t}"')
if no_cache:
lines.append(" no_cache: true")
if depends:
lines.append(" depends_on:")
lines.append(f" - {depends}")
return "\n".join(lines)
def _prep_step(self, img: dict, *, env_block: str = "") -> str:
n = img["name"]
cmds = img.get("prep_commands", [])
if not cmds:
return ""
lines = [
f" - name: prep-{n}-context",
" image: alpine:3",
]
if env_block:
lines.append(env_block)
if n == "docs":
lines.append(" volumes:")
lines.append(" - ${CI_WORKSPACE}/data:/host-data:ro")
lines.append(" commands:")
for cmd in cmds:
lines.append(f" - {cmd}")
return "\n".join(lines)
def _scan_step(self, img: dict, tag_expr: str) -> str:
n = img["name"]
sev = img.get("trivy_severity", "HIGH,CRITICAL")
ec = img.get("trivy_exit_code", 0)
return f"""\
- name: scan-{n}
<<: *trivy_scan
commands:
- trivy image --image-src remote --insecure --severity {sev}
--exit-code {ec} --format json -o {n}-scan.json
"$REGISTRY/$ORG/{n}:{tag_expr}"
depends_on: [build-push-{n}]"""
def _s3_upload_step(self, img: dict) -> str:
n = img["name"]
return f"""\
- name: upload-{n}-scan
image: {self.plugin_s3}
failure: ignore
settings:
endpoint: http://rustfs:9000
bucket: gitea
access_key:
from_secret: s3_access_key
secret_key:
from_secret: s3_secret_key
source: "{n}-scan.json"
target: /ci/${{CI_REPO}}/${{CI_COMMIT_SHA:0:8}}/
path_style: true
depends_on:
- scan-{n}"""
def _scan_report_loop(self, imgs: list[dict]) -> str:
files = " ".join(f"{i['name']}-scan.json" for i in imgs)
return f"""\
- |
for f in {files}; do
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
done"""
def _harden_close_loop(self, imgs: list[dict]) -> str:
files = " ".join(f"{i['name']}-scan.json" for i in imgs)
return f"""\
- |
for f in {files}; do
if [ -f "$f" ]; then
uv run python -m api.diag.vuln --close "$f" || \\
uv run python -m api.diag.vuln "$f" || true
fi
done"""
def _path_block(self, pf: list[str]) -> str:
if not pf:
return ""
if len(pf) == 1:
return f' - path: "{pf[0]}"'
return " - path:\n" + "\n".join(f' - "{p}"' for p in pf)
# ── pipeline generators ──────────────────────────────────────
def gen_ci(self) -> str:
return f"""\
# ── Quality gate (generated by gen_config.py — woodpecker backend) ──
# DO NOT EDIT — edit stack.toml and run gen_config.py
when:
- event: [push, pull_request, manual]
steps:
- name: lint
image: {self.uv_image}
environment:
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
commands:
- uv sync --dev
- uv run ruff check src/ tests/ --output-format=concise
- uv run ruff format --check src/ tests/
- name: test
image: {self.uv_image}
environment:
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
commands:
- uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=99 -q 2>&1 | tee pytest.out
- uv run python dev/scripts/coverage_badge.py < pytest.out > coverage.svg
depends_on:
- lint
- name: coverage-badge
image: {self.uv_image}
failure: ignore
environment:
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
GITEA_TOKEN:
from_secret: gitea_token
GITEA_URL:
from_secret: gitea_url
commands:
- uv run python dev/scripts/coverage_badge.py --post pytest.out
depends_on:
- test
- name: commit-badge
image: alpine/git
failure: ignore
environment:
REGISTRY_USER:
from_secret: registry_user
REGISTRY_PASS:
from_secret: registry_pass
commands:
- git config user.name woodpecker-ci
- git config user.email ci@${{CI_REPO_OWNER}}.io
- git add coverage.svg
- git diff --cached --quiet && echo "no change" && exit 0
- git commit -m "update coverage badge [skip ci]"
- git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/${{CI_REPO}}.git HEAD:main
when:
- branch: main
event: push
depends_on:
- test
- name: upload-coverage-badge
image: {self.plugin_s3}
failure: ignore
settings:
endpoint: http://rustfs:9000
bucket: gitea
access_key:
from_secret: s3_access_key
secret_key:
from_secret: s3_secret_key
source: "coverage.svg"
target: /badges/${{CI_REPO}}/
path_style: true
overwrite: true
depends_on:
- test
- name: pr-comment
image: {self.plugin_comment}
settings:
gitea_url:
from_secret: gitea_url
api_key:
from_secret: gitea_token
message: >
**CI** for `${{CI_COMMIT_SHA:0:8}}` —
[pipeline #${{CI_PIPELINE_NUMBER}}](${{CI_PIPELINE_URL}})
when:
- event: pull_request
depends_on:
- test
- coverage-badge
- name: validate-compose
image: docker:cli
volumes:
- {self.docker_socket}:/var/run/docker.sock
commands:
- docker compose config --quiet
- name: validate-gen-config
image: {self.uv_image}
environment:
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
commands:
- uv run python dev/scripts/gen_config.py --check
{self._failure_reporter}
"""
def gen_deploy(self) -> str:
tag_expr = "${CI_COMMIT_SHA:0:8}"
parts = [
"# ── Deploy pipeline (generated by gen_config.py — woodpecker backend) ──",
"# DO NOT EDIT — edit stack.toml [images] and run gen_config.py",
"",
"when:",
" - event: push",
" branch: main",
"",
self._variables_block,
"steps:",
" # ── Python package ──────────────────────────────────────────",
" - name: build-package",
f" image: {self.uv_image}",
" commands:",
" - BASE=$(grep '^version' pyproject.toml | head -1 | sed 's/.*\"\\(.*\\)\"/\\1/')",
' - uv version "$BASE.dev${CI_PIPELINE_NUMBER}" --no-sync',
" - uv build --out-dir dist/",
" - ls -lh dist/",
"",
" - name: publish-package",
f" image: {self.uv_image}",
" environment:",
" REGISTRY_USER:",
" from_secret: registry_user",
" REGISTRY_PASS:",
" from_secret: registry_pass",
" commands:",
' - uv publish --publish-url http://gitea:3000/api/packages/${CI_REPO_OWNER}/pypi --username "$REGISTRY_USER" --password "$REGISTRY_PASS" dist/*',
" depends_on:",
" - build-package",
"",
" # ── Container images ──────────────────────────────────────",
]
for img in self.images:
prep = self._prep_step(img)
if prep:
parts.append(prep)
parts.append("")
parts.append(
self._build_step(
img,
[tag_expr, "latest"],
name_prefix="build-push",
depends=f"prep-{img['name']}-context",
)
)
else:
parts.append(
self._build_step(
img, [tag_expr, "latest"], name_prefix="build-push"
)
)
parts.append("")
parts.append(" # ── Scan images ──────────────────────────────────────────")
for img in self.scans:
parts.append(self._scan_step(img, tag_expr))
parts.append("")
parts.append(" # ── Upload scan results ─────────────────────────────────")
for img in self.scans:
parts.append(self._s3_upload_step(img))
parts.append("")
parts.append(" # ── Report vulnerabilities ──────────────────────────────")
parts.append(" - name: report-vulns")
parts.append(f" image: {self.uv_image}")
parts.append(" failure: ignore")
parts.append(" environment:")
parts.append(" <<: *diag_env")
parts.append(" commands:")
parts.append(" - uv sync --no-dev")
parts.append(self._scan_report_loop(self.scans))
parts.append(" depends_on:")
for i in self.scans:
parts.append(f" - scan-{i['name']}")
parts.append("")
parts.append(" # ── Deploy (disabled — manual only) ────────────────────")
parts.append(" - name: deploy")
parts.append(" image: alpine:3")
parts.append(" commands:")
parts.append(
' - echo "Deploy disabled — run manually after green pipeline"'
)
parts.append(" depends_on:")
parts.append(" - publish-package")
for i in self.scans:
parts.append(f" - scan-{i['name']}")
for i in self.images:
if not i.get("scan", True):
parts.append(f" - build-push-{i['name']}")
parts.append("")
parts.append(self._failure_reporter)
return "\n".join(parts)
def gen_harden(self) -> str:
parts = [
"# ── Hardening pipeline (generated by gen_config.py — woodpecker backend) ──",
"# DO NOT EDIT — edit stack.toml [images] and run gen_config.py",
"",
"when:",
" - event: [manual, cron]",
"",
self._variables_block,
"steps:",
" # ── Build hardened images ──────────────────────────────────",
]
for img in self.images:
parts.append(self._build_step(img, ["hardened", "latest"], no_cache=True))
parts.append("")
parts.append(" # ── Scan hardened images ───────────────────────────────────")
for img in self.scans:
parts.append(self._scan_step(img, "hardened"))
parts.append("")
parts.append(" # ── Close resolved vuln issues OR file new ones ────────────")
parts.append(" - name: close-or-report-vulns")
parts.append(f" image: {self.uv_image}")
parts.append(" environment:")
parts.append(" <<: *diag_env")
parts.append(" commands:")
parts.append(" - uv sync --no-dev")
parts.append(self._harden_close_loop(self.scans))
parts.append(" depends_on:")
for i in self.scans:
parts.append(f" - scan-{i['name']}")
parts.append("")
parts.append(self._failure_reporter)
return "\n".join(parts)
def gen_rebuild_all(self) -> str:
tag_expr = "${CI_COMMIT_SHA:0:8}"
svc_list = " ".join(i["name"] for i in self.images)
parts = [
"# ── Rebuild-all pipeline (generated by gen_config.py — woodpecker backend) ──",
"# DO NOT EDIT — edit stack.toml [images] and run gen_config.py",
"",
"when:",
" - event: manual",
"",
self._variables_block,
"steps:",
]
for img in self.images:
prep = self._prep_step(
img,
env_block=" environment:\n STACK_ROOT:\n from_secret: stack_root",
)
if prep:
parts.append(prep)
parts.append("")
parts.append(
self._build_step(
img,
[tag_expr, "latest"],
name_prefix="build-push",
depends=f"prep-{img['name']}-context",
)
)
else:
parts.append(
self._build_step(
img, [tag_expr, "latest"], name_prefix="build-push"
)
)
parts.append("")
parts.append(" # ── Scan images ──────────────────────────────────────────")
for img in self.scans:
parts.append(self._scan_step(img, tag_expr))
parts.append("")
parts.append(" # ── Report vulnerabilities ──────────────────────────────")
parts.append(" - name: report-vulns")
parts.append(f" image: {self.uv_image}")
parts.append(" failure: ignore")
parts.append(" environment:")
parts.append(" <<: *diag_env")
parts.append(" commands:")
parts.append(" - uv sync --no-dev")
parts.append(self._scan_report_loop(self.scans))
parts.append(" depends_on:")
for i in self.scans:
parts.append(f" - scan-{i['name']}")
parts.append("")
parts.append(" - name: deploy")
parts.append(" image: docker:cli")
parts.append(" environment:")
parts.append(" REGISTRY: *registry")
parts.append(" ORG: *org")
parts.append(" IMAGE_PREFIX: *image_prefix")
parts.append(" STACK_ROOT:")
parts.append(" from_secret: stack_root")
parts.append(" volumes:")
parts.append(f" - {self.docker_socket}:/var/run/docker.sock")
parts.append(
" - ${STACK_ROOT:-/home/kert/stack}:${STACK_ROOT:-/home/kert/stack}"
)
parts.append(" commands:")
parts.append(" - STACK_ROOT=${STACK_ROOT:-/home/kert/stack}")
parts.append(" - cd $STACK_ROOT")
parts.append(" - TAG=${CI_COMMIT_SHA:0:8}")
parts.append(f" - for SVC in {svc_list}; do")
parts.append(" docker pull $REGISTRY/$ORG/$SVC:$TAG &&")
parts.append(
" docker tag $REGISTRY/$ORG/$SVC:$TAG $IMAGE_PREFIX/$SVC:$TAG;"
)
parts.append(" done")
parts.append(
' - sed -i "s/^COMMIT_SHA=.*/COMMIT_SHA=$TAG/" .env 2>/dev/null || echo "COMMIT_SHA=$TAG" >> .env'
)
parts.append(" - docker compose up -d --remove-orphans")
parts.append(" depends_on:")
for i in self.scans:
parts.append(f" - scan-{i['name']}")
for i in self.images:
if not i.get("scan", True):
parts.append(f" - build-push-{i['name']}")
parts.append("")
parts.append(self._failure_reporter)
return "\n".join(parts)
def gen_infra_ci(self) -> str:
parts = [
"# ── Infrastructure quality gate (generated by gen_config.py — woodpecker backend) ──",
"# DO NOT EDIT — edit stack.toml [images] and run gen_config.py",
"",
"when:",
" - event: [push, pull_request, manual]",
"",
"steps:",
" - name: infra-gate",
" image: alpine:3",
" commands:",
' - echo "infra-ci gate passed"',
"",
]
for img in self.images:
if not img.get("hadolint", True):
continue
n = img["name"]
pb = self._path_block(img.get("path_filter", []))
parts.append(f" - name: hadolint-{n}")
parts.append(" image: hadolint/hadolint:latest-debian")
parts.append(" commands:")
parts.append(f" - hadolint {img['dockerfile']}")
if pb:
parts.append(" when:")
parts.append(pb)
parts.append("")
parts.append(f" - name: build-{n}")
parts.append(f" image: {self.plugin_buildx_dryrun}")
parts.append(" settings:")
parts.append(f" dockerfile: {img['dockerfile']}")
parts.append(f" context: {img['context']}")
parts.append(' tags: ["ci-test"]')
parts.append(" daemon_off: true")
parts.append(" dry_run: true")
parts.append(" volumes:")
parts.append(f" - {self.docker_socket}:/var/run/docker.sock")
parts.append(" depends_on:")
parts.append(f" - hadolint-{n}")
if pb:
parts.append(" when:")
parts.append(pb)
parts.append("")
parts.append(" # ── Config validation ────────────────────────────────────")
parts.append("""\
- name: validate-nginx
image: nginx:alpine
commands:
- cp nginx/nginx.conf /etc/nginx/nginx.conf
- nginx -t
when:
- path: "nginx/**"
- name: validate-prometheus
image: prom/prometheus:latest
commands:
- promtool check config prometheus/prometheus.yml
when:
- path: "prometheus/**"
- name: validate-traefik-static
image: cytopia/yamllint:latest
commands:
- yamllint -d relaxed traefik/traefik.yml
when:
- path: "traefik/traefik.yml"
- name: validate-traefik-template
image: traefik:v3.3
environment:
DOMAIN: ci-test.example.com
commands:
- |
timeout 5 traefik \\
--providers.file.directory=traefik/dynamic \\
--api.dashboard=false \\
--log.level=DEBUG 2>&1 | head -80 || true
- echo "Traefik template syntax OK"
when:
- path: "traefik/dynamic/**"
- name: validate-loki
image: cytopia/yamllint:latest
commands:
- yamllint -d relaxed loki/
when:
- path: "loki/**"
- name: validate-trino
image: alpine:3
commands:
- |
OK=true
for f in trino/etc/config.properties trino/etc/node.properties trino/etc/jvm.config; do
if [ ! -f "$f" ]; then echo "MISSING: $f"; OK=false; fi
done
ls trino/etc/catalog/*.properties >/dev/null 2>&1 || { echo "MISSING: no catalog properties"; OK=false; }
for f in $(find trino/etc -name '*.properties'); do
while IFS= read -r line; do
case "$line" in ''|'#'*|'!'*) continue ;; esac
echo "$line" | grep -q '=' || { echo "BAD LINE in $f: $line"; OK=false; }
done < "$f"
done
$OK && echo "Trino config validation passed"
$OK
when:
- path: "trino/etc/**"
""")
parts.append(self._failure_reporter)
return "\n".join(parts)
def gen_release(self) -> str:
return f"""\
# ── Release pipeline (generated by gen_config.py — woodpecker backend) ──
# DO NOT EDIT — edit stack.toml and run gen_config.py
when:
- event: tag
variables:
- &diag_env
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
GITEA_TOKEN:
from_secret: gitea_token
steps:
- name: build-package
image: {self.uv_image}
commands:
- uv build --out-dir dist/
- ls -lh dist/
- name: release
image: {self.plugin_release}
settings:
api_key:
from_secret: gitea_token
base_url:
from_secret: gitea_url
files:
- dist/*.whl
- dist/*.tar.gz
title: "${{CI_COMMIT_TAG}}"
depends_on:
- build-package
{self._failure_reporter}
"""

822
dev/scripts/gen_config.py
View File

@@ -1,32 +1,40 @@
"""Generate all derived config files from stack.toml.
Reads [platform], [services], and [images] from stack.toml and renders:
- coredns/hosts + coredns/Corefile
- .woodpecker/ci.yml, deploy.yml, harden.yml, infra-ci.yml,
rebuild-all.yml, release.yml
Reads [platform], [services], [images], and [ci] from stack.toml.
Dispatches to the active CI backend emitter (woodpecker or github)
for pipeline YAML, plus generates coredns config.
Usage::
uv run python dev/scripts/gen_config.py # generate all
uv run python dev/scripts/gen_config.py --check # verify up-to-date
uv run python dev/scripts/gen_config.py # generate all
uv run python dev/scripts/gen_config.py --check # verify up-to-date
uv run python dev/scripts/gen_config.py --backend X # override backend
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
from conf import ROOT, cfg
# ── Platform values ──────────────────────────────────────────────
REGISTRY = cfg.platform.registry
ORG = cfg.platform.repo.split("/")[0]
IMAGE_PREFIX = cfg.platform.image_prefix
DOMAIN = cfg.platform.domain
HOST_IP = cfg.platform.host_ip
def _platform_dict() -> dict:
return {
"registry": cfg.platform.registry,
"org": cfg.platform.repo.split("/")[0],
"image_prefix": cfg.platform.image_prefix,
"domain": DOMAIN,
"host_ip": HOST_IP,
"repo": cfg.platform.repo,
}
# ── Image definitions ────────────────────────────────────────────
@@ -41,702 +49,30 @@ def load_images() -> list[dict]:
return images
def scannable() -> list[dict]:
return [i for i in load_images() if i.get("scan", True)]
# ── YAML fragments ───────────────────────────────────────────────
# Woodpecker relies on YAML anchors which can't be expressed via
# yaml.dump(). We use f-string templates with loops instead.
_VARIABLES_BLOCK = f"""\
variables:
- &buildx_image woodpeckerci/plugin-docker-buildx:5-insecure
- &registry {REGISTRY}
- &org {ORG}
- &image_prefix {IMAGE_PREFIX}
- &buildx_base
buildkit_config: |
[registry."{REGISTRY}"]
http = true
registry: *registry
username:
from_secret: registry_user
password:
from_secret: registry_pass
- &trivy_scan
image: aquasec/trivy:latest
environment:
REGISTRY: *registry
ORG: *org
- &diag_env
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
GITEA_TOKEN:
from_secret: gitea_token
"""
_FAILURE_REPORTER = """\
# ── Failure reporter ─────────────────────────────────────────
- name: report-failure
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
failure: ignore
environment:
<<: *diag_env
commands:
- uv sync --no-dev
- uv run python -m api.diag
when:
- status: [failure]
"""
def _build_step(
img: dict,
tags: list[str],
*,
no_cache: bool = False,
name_prefix: str = "build",
depends: str = "",
) -> str:
n = img["name"]
lines = [
f" - name: {name_prefix}-{n}",
" image: *buildx_image",
" settings:",
" <<: *buildx_base",
f" repo: {REGISTRY}/{ORG}/{n}",
f" dockerfile: {img['dockerfile']}",
f" context: {img['context']}",
" tags:",
]
for t in tags:
lines.append(f' - "{t}"')
if no_cache:
lines.append(" no_cache: true")
if depends:
lines.append(f" depends_on:")
lines.append(f" - {depends}")
return "\n".join(lines)
def _prep_step(img: dict, *, env_block: str = "") -> str:
"""Generate a prep step for images that need pre-build commands."""
n = img["name"]
cmds = img.get("prep_commands", [])
if not cmds:
return ""
lines = [
f" - name: prep-{n}-context",
" image: alpine:3",
]
if env_block:
lines.append(env_block)
if n == "docs":
lines.append(" volumes:")
lines.append(" - ${CI_WORKSPACE}/data:/host-data:ro")
lines.append(" commands:")
for cmd in cmds:
lines.append(f" - {cmd}")
return "\n".join(lines)
def _scan_step(img: dict, tag_expr: str) -> str:
n = img["name"]
sev = img.get("trivy_severity", "HIGH,CRITICAL")
ec = img.get("trivy_exit_code", 0)
return f"""\
- name: scan-{n}
<<: *trivy_scan
commands:
- trivy image --image-src remote --insecure --severity {sev}
--exit-code {ec} --format json -o {n}-scan.json
"$REGISTRY/$ORG/{n}:{tag_expr}"
depends_on: [build-push-{n}]"""
def _s3_upload_step(img: dict) -> str:
n = img["name"]
return f"""\
- name: upload-{n}-scan
image: woodpeckerci/plugin-s3
failure: ignore
settings:
endpoint: http://rustfs:9000
bucket: gitea
access_key:
from_secret: s3_access_key
secret_key:
from_secret: s3_secret_key
source: "{n}-scan.json"
target: /ci/${{CI_REPO}}/${{CI_COMMIT_SHA:0:8}}/
path_style: true
depends_on:
- scan-{n}"""
def _scan_report_loop(images: list[dict]) -> str:
files = " ".join(f"{i['name']}-scan.json" for i in images)
return f"""\
- |
for f in {files}; do
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
done"""
def _harden_close_loop(images: list[dict]) -> str:
files = " ".join(f"{i['name']}-scan.json" for i in images)
return f"""\
- |
for f in {files}; do
if [ -f "$f" ]; then
uv run python -m api.diag.vuln --close "$f" || \\
uv run python -m api.diag.vuln "$f" || true
fi
done"""
# ── Pipeline generators ──────────────────────────────────────────
def gen_harden() -> str:
images = load_images()
scans = scannable()
scan_deps = "\n".join(f" - scan-{i['name']}" for i in scans)
parts = [
"# ── Hardening pipeline (generated by gen_config.py) ─────────────",
"# DO NOT EDIT — edit stack.toml [images] and run gen_config.py",
"",
"when:",
" - event: [manual, cron]",
"",
_VARIABLES_BLOCK,
"steps:",
" # ── Build hardened images ──────────────────────────────────",
]
for img in images:
parts.append(_build_step(img, ["hardened", "latest"], no_cache=True))
parts.append("")
parts.append(" # ── Scan hardened images ───────────────────────────────────")
for img in scans:
parts.append(_scan_step(img, "hardened"))
parts.append("")
parts.append(" # ── Close resolved vuln issues OR file new ones ────────────")
parts.append(" - name: close-or-report-vulns")
parts.append(" image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim")
parts.append(" environment:")
parts.append(" <<: *diag_env")
parts.append(" commands:")
parts.append(" - uv sync --no-dev")
parts.append(_harden_close_loop(scans))
parts.append(" depends_on:")
for i in scans:
parts.append(f" - scan-{i['name']}")
parts.append("")
parts.append(_FAILURE_REPORTER)
return "\n".join(parts)
def gen_deploy() -> str:
images = load_images()
scans = scannable()
parts = [
"# ── Deploy pipeline (generated by gen_config.py) ────────────────",
"# DO NOT EDIT — edit stack.toml [images] and run gen_config.py",
"",
"when:",
" - event: push",
" branch: main",
"",
_VARIABLES_BLOCK,
"steps:",
" # ── Python package ──────────────────────────────────────────",
" - name: build-package",
" image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim",
" commands:",
" - BASE=$(grep '^version' pyproject.toml | head -1 | sed 's/.*\"\\(.*\\)\"/\\1/')",
' - uv version "$BASE.dev${CI_PIPELINE_NUMBER}" --no-sync',
" - uv build --out-dir dist/",
" - ls -lh dist/",
"",
" - name: publish-package",
" image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim",
" environment:",
" REGISTRY_USER:",
" from_secret: registry_user",
" REGISTRY_PASS:",
" from_secret: registry_pass",
" commands:",
f' - uv publish --publish-url http://gitea:3000/api/packages/${{CI_REPO_OWNER}}/pypi --username "$REGISTRY_USER" --password "$REGISTRY_PASS" dist/*',
" depends_on:",
" - build-package",
"",
]
parts.append(" # ── Container images ──────────────────────────────────────")
tag_expr = '${CI_COMMIT_SHA:0:8}'
for img in images:
prep = _prep_step(img)
if prep:
parts.append(prep)
parts.append("")
dep = f"prep-{img['name']}-context"
parts.append(
_build_step(
img,
[tag_expr, "latest"],
name_prefix="build-push",
depends=dep,
)
)
else:
parts.append(
_build_step(img, [tag_expr, "latest"], name_prefix="build-push")
)
parts.append("")
parts.append(" # ── Scan images ──────────────────────────────────────────")
for img in scans:
parts.append(_scan_step(img, tag_expr))
parts.append("")
parts.append(" # ── Upload scan results ─────────────────────────────────")
for img in scans:
parts.append(_s3_upload_step(img))
parts.append("")
parts.append(" # ── Report vulnerabilities ──────────────────────────────")
parts.append(" - name: report-vulns")
parts.append(" image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim")
parts.append(" failure: ignore")
parts.append(" environment:")
parts.append(" <<: *diag_env")
parts.append(" commands:")
parts.append(" - uv sync --no-dev")
parts.append(_scan_report_loop(scans))
parts.append(" depends_on:")
for i in scans:
parts.append(f" - scan-{i['name']}")
parts.append("")
# Deploy step (disabled)
scan_deps = "\n".join(f" - scan-{i['name']}" for i in scans)
build_deps = "\n".join(f" - build-push-{i['name']}" for i in images if not i.get("scan", True))
parts.append(" # ── Deploy (disabled — manual only) ────────────────────")
parts.append(" - name: deploy")
parts.append(" image: alpine:3")
parts.append(" commands:")
parts.append(' - echo "Deploy disabled — run manually after green pipeline"')
parts.append(" depends_on:")
parts.append(" - publish-package")
for i in scans:
parts.append(f" - scan-{i['name']}")
for i in images:
if not i.get("scan", True):
parts.append(f" - build-push-{i['name']}")
parts.append("")
parts.append(_FAILURE_REPORTER)
return "\n".join(parts)
def gen_rebuild_all() -> str:
images = load_images()
scans = scannable()
tag_expr = '${CI_COMMIT_SHA:0:8}'
parts = [
"# ── Rebuild-all pipeline (generated by gen_config.py) ───────────",
"# DO NOT EDIT — edit stack.toml [images] and run gen_config.py",
"",
"when:",
" - event: manual",
"",
_VARIABLES_BLOCK,
"steps:",
]
for img in images:
prep = _prep_step(
img,
env_block=" environment:\n STACK_ROOT:\n from_secret: stack_root",
)
if prep:
parts.append(prep)
parts.append("")
parts.append(
_build_step(
img,
[tag_expr, "latest"],
name_prefix="build-push",
depends=f"prep-{img['name']}-context",
)
)
else:
parts.append(
_build_step(img, [tag_expr, "latest"], name_prefix="build-push")
)
parts.append("")
parts.append(" # ── Scan images ──────────────────────────────────────────")
for img in scans:
parts.append(_scan_step(img, tag_expr))
parts.append("")
parts.append(" # ── Report vulnerabilities ──────────────────────────────")
parts.append(" - name: report-vulns")
parts.append(" image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim")
parts.append(" failure: ignore")
parts.append(" environment:")
parts.append(" <<: *diag_env")
parts.append(" commands:")
parts.append(" - uv sync --no-dev")
parts.append(_scan_report_loop(scans))
parts.append(" depends_on:")
for i in scans:
parts.append(f" - scan-{i['name']}")
parts.append("")
# Deploy step
svc_list = " ".join(i["name"] for i in images)
parts.append(" - name: deploy")
parts.append(" image: docker:cli")
parts.append(" environment:")
parts.append(" REGISTRY: *registry")
parts.append(" ORG: *org")
parts.append(" IMAGE_PREFIX: *image_prefix")
parts.append(" STACK_ROOT:")
parts.append(" from_secret: stack_root")
parts.append(" volumes:")
parts.append(" - /run/user/1000/docker.sock:/var/run/docker.sock")
parts.append(" - ${STACK_ROOT:-/home/kert/stack}:${STACK_ROOT:-/home/kert/stack}")
parts.append(" commands:")
parts.append(" - STACK_ROOT=${STACK_ROOT:-/home/kert/stack}")
parts.append(" - cd $STACK_ROOT")
parts.append(" - TAG=${CI_COMMIT_SHA:0:8}")
parts.append(f" - for SVC in {svc_list}; do")
parts.append(" docker pull $REGISTRY/$ORG/$SVC:$TAG &&")
parts.append(" docker tag $REGISTRY/$ORG/$SVC:$TAG $IMAGE_PREFIX/$SVC:$TAG;")
parts.append(" done")
parts.append(' - sed -i "s/^COMMIT_SHA=.*/COMMIT_SHA=$TAG/" .env 2>/dev/null || echo "COMMIT_SHA=$TAG" >> .env')
parts.append(" - docker compose up -d --remove-orphans")
parts.append(" depends_on:")
for i in scans:
parts.append(f" - scan-{i['name']}")
for i in images:
if not i.get("scan", True):
parts.append(f" - build-push-{i['name']}")
parts.append("")
parts.append(_FAILURE_REPORTER)
return "\n".join(parts)
def gen_infra_ci() -> str:
images = load_images()
parts = [
"# ── Infrastructure quality gate (generated by gen_config.py) ────",
"# DO NOT EDIT — edit stack.toml [images] and run gen_config.py",
"",
"when:",
" - event: [push, pull_request, manual]",
"",
"steps:",
" - name: infra-gate",
" image: alpine:3",
" commands:",
' - echo "infra-ci gate passed"',
"",
]
for img in images:
if not img.get("hadolint", True):
continue
n = img["name"]
pf = img.get("path_filter", [])
path_block = ""
if pf:
if len(pf) == 1:
path_block = f' - path: "{pf[0]}"'
else:
path_block = " - path:\n" + "\n".join(
f' - "{p}"' for p in pf
)
parts.append(f" - name: hadolint-{n}")
parts.append(" image: hadolint/hadolint:latest-debian")
parts.append(" commands:")
parts.append(f" - hadolint {img['dockerfile']}")
if path_block:
parts.append(" when:")
parts.append(path_block)
parts.append("")
parts.append(f" - name: build-{n}")
parts.append(" image: woodpeckerci/plugin-docker-buildx")
parts.append(" settings:")
parts.append(f" dockerfile: {img['dockerfile']}")
parts.append(f" context: {img['context']}")
parts.append(' tags: ["ci-test"]')
parts.append(" daemon_off: true")
parts.append(" dry_run: true")
parts.append(" volumes:")
parts.append(" - /run/user/1000/docker.sock:/var/run/docker.sock")
parts.append(" depends_on:")
parts.append(f" - hadolint-{n}")
if path_block:
parts.append(" when:")
parts.append(path_block)
parts.append("")
# Static validation steps
parts.append(" # ── Config validation ────────────────────────────────────")
parts.append("""\
- name: validate-nginx
image: nginx:alpine
commands:
- cp nginx/nginx.conf /etc/nginx/nginx.conf
- nginx -t
when:
- path: "nginx/**"
- name: validate-prometheus
image: prom/prometheus:latest
commands:
- promtool check config prometheus/prometheus.yml
when:
- path: "prometheus/**"
- name: validate-traefik-static
image: cytopia/yamllint:latest
commands:
- yamllint -d relaxed traefik/traefik.yml
when:
- path: "traefik/traefik.yml"
- name: validate-traefik-template
image: traefik:v3.3
environment:
DOMAIN: ci-test.example.com
commands:
- |
timeout 5 traefik \\
--providers.file.directory=traefik/dynamic \\
--api.dashboard=false \\
--log.level=DEBUG 2>&1 | head -80 || true
- echo "Traefik template syntax OK"
when:
- path: "traefik/dynamic/**"
- name: validate-loki
image: cytopia/yamllint:latest
commands:
- yamllint -d relaxed loki/
when:
- path: "loki/**"
- name: validate-trino
image: alpine:3
commands:
- |
OK=true
for f in trino/etc/config.properties trino/etc/node.properties trino/etc/jvm.config; do
if [ ! -f "$f" ]; then echo "MISSING: $f"; OK=false; fi
done
ls trino/etc/catalog/*.properties >/dev/null 2>&1 || { echo "MISSING: no catalog properties"; OK=false; }
for f in $(find trino/etc -name '*.properties'); do
while IFS= read -r line; do
case "$line" in ''|'#'*|'!'*) continue ;; esac
echo "$line" | grep -q '=' || { echo "BAD LINE in $f: $line"; OK=false; }
done < "$f"
done
$OK && echo "Trino config validation passed"
$OK
when:
- path: "trino/etc/**"
""")
parts.append(_FAILURE_REPORTER)
return "\n".join(parts)
def gen_ci() -> str:
return f"""\
# ── Quality gate (generated by gen_config.py) ────────────────────
# DO NOT EDIT — edit stack.toml and run gen_config.py
when:
- event: [push, pull_request, manual]
steps:
- name: lint
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
environment:
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
commands:
- uv sync --dev
- uv run ruff check src/ tests/ --output-format=concise
- uv run ruff format --check src/ tests/
- name: test
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
environment:
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
commands:
- uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=99 -q 2>&1 | tee pytest.out
- uv run python dev/scripts/coverage_badge.py < pytest.out > coverage.svg
depends_on:
- lint
- name: coverage-badge
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
failure: ignore
environment:
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
GITEA_TOKEN:
from_secret: gitea_token
GITEA_URL:
from_secret: gitea_url
commands:
- uv run python dev/scripts/coverage_badge.py --post pytest.out
depends_on:
- test
- name: commit-badge
image: alpine/git
failure: ignore
environment:
REGISTRY_USER:
from_secret: registry_user
REGISTRY_PASS:
from_secret: registry_pass
commands:
- git config user.name woodpecker-ci
- git config user.email ci@${{CI_REPO_OWNER}}.io
- git add coverage.svg
- git diff --cached --quiet && echo "no change" && exit 0
- git commit -m "update coverage badge [skip ci]"
- git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/${{CI_REPO}}.git HEAD:main
when:
- branch: main
event: push
depends_on:
- test
- name: upload-coverage-badge
image: woodpeckerci/plugin-s3
failure: ignore
settings:
endpoint: http://rustfs:9000
bucket: gitea
access_key:
from_secret: s3_access_key
secret_key:
from_secret: s3_secret_key
source: "coverage.svg"
target: /badges/${{CI_REPO}}/
path_style: true
overwrite: true
depends_on:
- test
- name: pr-comment
image: woodpeckerci/plugin-gitea-comment
settings:
gitea_url:
from_secret: gitea_url
api_key:
from_secret: gitea_token
message: >
**CI** for `${{CI_COMMIT_SHA:0:8}}` —
[pipeline #${{CI_PIPELINE_NUMBER}}](${{CI_PIPELINE_URL}})
when:
- event: pull_request
depends_on:
- test
- coverage-badge
- name: validate-compose
image: docker:cli
volumes:
- /run/user/1000/docker.sock:/var/run/docker.sock
commands:
- docker compose config --quiet
- name: validate-gen-config
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
environment:
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
commands:
- uv run python dev/scripts/gen_config.py --check
{_FAILURE_REPORTER}
"""
def gen_release() -> str:
return f"""\
# ── Release pipeline (generated by gen_config.py) ────────────────
# DO NOT EDIT — edit stack.toml and run gen_config.py
when:
- event: tag
variables:
- &diag_env
UV_PYTHON_PREFERENCE: only-system
UV_LINK_MODE: copy
UV_PROJECT_ENVIRONMENT: .venv
GITEA_TOKEN:
from_secret: gitea_token
steps:
- name: build-package
image: ghcr.io/astral-sh/uv:python3.13-bookworm-slim
commands:
- uv build --out-dir dist/
- ls -lh dist/
- name: release
image: woodpeckerci/plugin-gitea-release
settings:
api_key:
from_secret: gitea_token
base_url:
from_secret: gitea_url
files:
- dist/*.whl
- dist/*.tar.gz
title: "${{CI_COMMIT_TAG}}"
depends_on:
- build-package
{_FAILURE_REPORTER}
"""
def scannable(images: list[dict]) -> list[dict]:
return [i for i in images if i.get("scan", True)]
# ── CoreDNS ──────────────────────────────────────────────────────
SUBDOMAINS = [
"", "gitea", "ci", "docs", "notebooks", "api", "s3", "s3console",
"grafana", "traefik", "zotero", "nessie", "polaris", "trino",
"prometheus", "jaeger", "loki",
"",
"gitea",
"ci",
"docs",
"notebooks",
"api",
"s3",
"s3console",
"grafana",
"traefik",
"zotero",
"nessie",
"polaris",
"trino",
"prometheus",
"jaeger",
"loki",
]
@@ -762,18 +98,63 @@ def gen_coredns() -> dict[str, str]:
return {"coredns/hosts": hosts, "coredns/Corefile": corefile}
# ── Backend dispatch ─────────────────────────────────────────────
STALE_DIRS = {
"woodpecker": ".github/workflows",
"github": ".woodpecker",
}
def _emit_backend(backend: str) -> dict[str, str]:
"""Dispatch to the correct CI backend emitter."""
images = load_images()
scans = scannable(images)
platform = _platform_dict()
ci_data = cfg._data.get("ci", {})
ci_cfg = ci_data.get(backend, {})
if backend == "woodpecker":
from backends.woodpecker import emit
elif backend == "github":
from backends.github import emit
else:
print(f"Unknown CI backend: {backend}", file=sys.stderr)
sys.exit(1)
return emit(images, scans, platform, ci_cfg)
def _clean_stale(backend: str, generated_files: dict[str, str]) -> list[str]:
"""Return stale files from the inactive backend directory."""
stale_dir = STALE_DIRS.get(backend, "")
if not stale_dir:
return []
stale_path = ROOT / stale_dir
if not stale_path.exists():
return []
removed = []
for f in sorted(stale_path.glob("*.yml")):
rel = str(f.relative_to(ROOT))
if rel not in generated_files:
# Only remove files that look generated
content = f.read_text()
if "generated by gen_config.py" in content:
removed.append(rel)
return removed
# ── Main ─────────────────────────────────────────────────────────
def generate() -> dict[str, str]:
def generate(backend: str | None = None) -> dict[str, str]:
if backend is None:
backend = cfg._data.get("ci", {}).get("backend", "woodpecker")
files: dict[str, str] = {}
files.update(gen_coredns())
files[".woodpecker/ci.yml"] = gen_ci()
files[".woodpecker/deploy.yml"] = gen_deploy()
files[".woodpecker/harden.yml"] = gen_harden()
files[".woodpecker/infra-ci.yml"] = gen_infra_ci()
files[".woodpecker/rebuild-all.yml"] = gen_rebuild_all()
files[".woodpecker/release.yml"] = gen_release()
files.update(_emit_backend(backend))
return files
@@ -797,11 +178,20 @@ def check_all(files: dict[str, str]) -> list[str]:
def main() -> int:
parser = argparse.ArgumentParser(description="Generate config from stack.toml")
parser.add_argument("--check", action="store_true")
parser = argparse.ArgumentParser(
description="Generate derived config from stack.toml [ci] backend"
)
parser.add_argument(
"--check",
action="store_true",
help="Verify files are up-to-date (exit 1 if not)",
)
parser.add_argument(
"--backend", default=None, help="Override CI backend (woodpecker|github)"
)
args = parser.parse_args()
files = generate()
files = generate(args.backend)
if args.check:
diffs = check_all(files)
@@ -815,7 +205,15 @@ def main() -> int:
return 0
write_all(files)
print(f"\nGenerated {len(files)} files from stack.toml")
# Clean stale files from inactive backend
backend = args.backend or cfg._data.get("ci", {}).get("backend", "woodpecker")
stale = _clean_stale(backend, files)
for s in stale:
(ROOT / s).unlink()
print(f" removed stale {s}")
print(f"\nGenerated {len(files)} files from stack.toml (backend: {backend})")
return 0

17
stack.toml
View File

@@ -108,6 +108,23 @@ port = 8000
secret = "" # override via STACK_API_SECRET env var
workers = 1
[ci]
backend = "woodpecker"
[ci.woodpecker]
plugin_buildx = "woodpeckerci/plugin-docker-buildx:5-insecure"
plugin_buildx_dryrun = "woodpeckerci/plugin-docker-buildx"
plugin_s3 = "woodpeckerci/plugin-s3"
plugin_comment = "woodpeckerci/plugin-gitea-comment"
plugin_release = "woodpeckerci/plugin-gitea-release"
docker_socket = "/run/user/1000/docker.sock"
uv_image = "ghcr.io/astral-sh/uv:python3.13-bookworm-slim"
[ci.github]
runner = "ubuntu-latest"
registry = "ghcr.io"
uv_version = "latest"
[lint]
line_length = 88