ci: modular actions via gen_config + fix module sync
Some checks failed
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
CI / lint (push) Successful in 30s
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Failing after 12s
Deploy / api (push) Successful in 42s
Infra CI / zotero (push) Successful in 13s
Infra CI / docs (push) Successful in 12s
Infra CI / api (push) Successful in 11s
Infra CI / mc (push) Successful in 11s
Package Supply Chain / pkg-supply-chain (push) Failing after 57s
Deploy / report (push) Has been cancelled
CI / test (push) Has been cancelled
Some checks failed
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
CI / lint (push) Successful in 30s
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Failing after 12s
Deploy / api (push) Successful in 42s
Infra CI / zotero (push) Successful in 13s
Infra CI / docs (push) Successful in 12s
Infra CI / api (push) Successful in 11s
Infra CI / mc (push) Successful in 11s
Package Supply Chain / pkg-supply-chain (push) Failing after 57s
Deploy / report (push) Has been cancelled
CI / test (push) Has been cancelled
gen_config.py changes: - CI: split into lint (fast) → test (needs lint). Drop skinny-install matrix (14 jobs → 2 jobs per push) - Deploy: per-image jobs with path filters from stack.toml. Each image only builds when its Dockerfile or source files change. Python-only commits skip all 5 image builds entirely. - All workflows regenerated from stack.toml pyproject.toml: - Add mail + prisma to module-name (fixes module sync check) tests/conftest.py: - Session-scoped zotero_db fixture: create_db() once per session instead of ~200 times
This commit is contained in:
@@ -1,3 +1,6 @@
|
|||||||
|
# DO NOT EDIT — generated by gen_config.py from stack.toml
|
||||||
|
# Re-generate: uv run python dev/scripts/gen_config.py
|
||||||
|
|
||||||
name: CI
|
name: CI
|
||||||
|
|
||||||
on:
|
on:
|
||||||
@@ -9,7 +12,8 @@ jobs:
|
|||||||
lint:
|
lint:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- name: Checkout
|
||||||
|
uses: https://github.com/actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up uv
|
- name: Set up uv
|
||||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||||
@@ -32,7 +36,8 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: lint
|
needs: lint
|
||||||
steps:
|
steps:
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- name: Checkout
|
||||||
|
uses: https://github.com/actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up uv
|
- name: Set up uv
|
||||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||||
@@ -50,6 +55,7 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
|
GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
|
uv sync --no-dev --quiet 2>/dev/null || true
|
||||||
uv run python -m api.diag.ci \
|
uv run python -m api.diag.ci \
|
||||||
--workflow "CI" --job "test" \
|
--workflow "CI" --job "test" \
|
||||||
--run "${{ github.run_number }}" \
|
--run "${{ github.run_number }}" \
|
||||||
|
|||||||
@@ -1,142 +1,209 @@
|
|||||||
|
# DO NOT EDIT — generated by gen_config.py from stack.toml
|
||||||
|
# Re-generate: uv run python dev/scripts/gen_config.py
|
||||||
|
|
||||||
name: Deploy
|
name: Deploy
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
|
|
||||||
# Shared setup anchors
|
|
||||||
env:
|
|
||||||
CRANE_INSECURE: "true"
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# ── Per-image build → scan → push ─────────────────────────────
|
|
||||||
|
|
||||||
notebooks:
|
notebooks:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: >-
|
if: >-
|
||||||
contains(github.event.head_commit.modified, 'notebooks/') ||
|
contains(github.event.head_commit.modified, 'notebooks') ||
|
||||||
contains(github.event.head_commit.modified, 'infra/images/notebooks.Dockerfile') ||
|
contains(github.event.head_commit.modified, 'infra/marimo') ||
|
||||||
contains(github.event.head_commit.modified, 'infra/marimo/') ||
|
contains(github.event.head_commit.modified, 'infra/images/notebooks.Dockerfile')
|
||||||
contains(github.event.head_commit.modified, 'pyproject.toml')
|
|
||||||
steps:
|
steps:
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- name: Checkout
|
||||||
- run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
uses: https://github.com/actions/checkout@v4
|
||||||
- run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
|
||||||
- run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
- name: Install crane
|
||||||
- run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
||||||
- name: Build
|
|
||||||
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build .
|
- name: Log in to registry
|
||||||
- name: Scan
|
run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --trivyignores .trivyignore --scanners vuln local/notebooks:build
|
env:
|
||||||
- name: Push
|
CRANE_INSECURE: "true"
|
||||||
|
|
||||||
|
|
||||||
|
- name: Install trivy
|
||||||
|
run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
||||||
|
|
||||||
|
- name: Scan notebooks
|
||||||
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o notebooks-scan.json local/notebooks:build
|
||||||
|
|
||||||
|
- name: Compute short SHA
|
||||||
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
- name: Build notebooks
|
||||||
|
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||||
|
|
||||||
|
- name: Push notebooks
|
||||||
run: |
|
run: |
|
||||||
docker save local/notebooks:build -o /tmp/img.tar
|
docker save local/notebooks:build -o /tmp/notebooks.tar
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/notebooks:${{ env.SHORT_SHA }} --insecure
|
crane push /tmp/notebooks.tar git:3000/homelab/stack/notebooks:${{ env.SHORT_SHA }} --insecure
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/notebooks:latest --insecure
|
crane push /tmp/notebooks.tar git:3000/homelab/stack/notebooks:latest --insecure
|
||||||
|
|
||||||
zotero:
|
zotero:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: >-
|
if: >-
|
||||||
contains(github.event.head_commit.modified, 'infra/images/zotero.Dockerfile') ||
|
contains(github.event.head_commit.modified, 'data/zotero') ||
|
||||||
contains(github.event.head_commit.modified, 'data/zotero/')
|
contains(github.event.head_commit.modified, 'infra/images/zotero.Dockerfile')
|
||||||
steps:
|
steps:
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- name: Checkout
|
||||||
- run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
uses: https://github.com/actions/checkout@v4
|
||||||
- run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
|
||||||
- run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
- name: Install crane
|
||||||
- run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
||||||
- name: Build
|
|
||||||
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build data/zotero
|
- name: Log in to registry
|
||||||
- name: Scan
|
run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --trivyignores .trivyignore --scanners vuln local/zotero:build
|
env:
|
||||||
- name: Push
|
CRANE_INSECURE: "true"
|
||||||
|
|
||||||
|
|
||||||
|
- name: Install trivy
|
||||||
|
run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
||||||
|
|
||||||
|
- name: Scan zotero
|
||||||
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o zotero-scan.json local/zotero:build
|
||||||
|
|
||||||
|
- name: Compute short SHA
|
||||||
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
- name: Build zotero
|
||||||
|
run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build data/zotero/
|
||||||
|
|
||||||
|
- name: Push zotero
|
||||||
run: |
|
run: |
|
||||||
docker save local/zotero:build -o /tmp/img.tar
|
docker save local/zotero:build -o /tmp/zotero.tar
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/zotero:${{ env.SHORT_SHA }} --insecure
|
crane push /tmp/zotero.tar git:3000/homelab/stack/zotero:${{ env.SHORT_SHA }} --insecure
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/zotero:latest --insecure
|
crane push /tmp/zotero.tar git:3000/homelab/stack/zotero:latest --insecure
|
||||||
|
|
||||||
docs:
|
docs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: >-
|
if: >-
|
||||||
contains(github.event.head_commit.modified, 'infra/images/docs.Dockerfile') ||
|
contains(github.event.head_commit.modified, 'docs') ||
|
||||||
contains(github.event.head_commit.modified, 'src/') ||
|
contains(github.event.head_commit.modified, 'infra/images/docs.Dockerfile')
|
||||||
contains(github.event.head_commit.modified, 'docs/')
|
|
||||||
steps:
|
steps:
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- name: Checkout
|
||||||
- run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
uses: https://github.com/actions/checkout@v4
|
||||||
- run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
|
||||||
- run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
- name: Install crane
|
||||||
- run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
||||||
- name: Build
|
|
||||||
|
- name: Log in to registry
|
||||||
|
run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
||||||
|
env:
|
||||||
|
CRANE_INSECURE: "true"
|
||||||
|
|
||||||
|
|
||||||
|
- name: Install trivy
|
||||||
|
run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
||||||
|
|
||||||
|
- name: Scan docs
|
||||||
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o docs-scan.json local/docs:build
|
||||||
|
|
||||||
|
- name: Compute short SHA
|
||||||
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
- name: Build docs
|
||||||
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
|
run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
|
||||||
- name: Scan
|
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --trivyignores .trivyignore --scanners vuln local/docs:build
|
- name: Push docs
|
||||||
- name: Push
|
|
||||||
run: |
|
run: |
|
||||||
docker save local/docs:build -o /tmp/img.tar
|
docker save local/docs:build -o /tmp/docs.tar
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/docs:${{ env.SHORT_SHA }} --insecure
|
crane push /tmp/docs.tar git:3000/homelab/stack/docs:${{ env.SHORT_SHA }} --insecure
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/docs:latest --insecure
|
crane push /tmp/docs.tar git:3000/homelab/stack/docs:latest --insecure
|
||||||
|
|
||||||
api:
|
api:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: >-
|
if: >-
|
||||||
contains(github.event.head_commit.modified, 'infra/images/api.Dockerfile') ||
|
contains(github.event.head_commit.modified, 'infra/images/api.Dockerfile') ||
|
||||||
contains(github.event.head_commit.modified, 'src/api/') ||
|
contains(github.event.head_commit.modified, 'src') ||
|
||||||
contains(github.event.head_commit.modified, 'pyproject.toml')
|
contains(github.event.head_commit.modified, 'pyproject.toml')
|
||||||
steps:
|
steps:
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- name: Checkout
|
||||||
- run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
uses: https://github.com/actions/checkout@v4
|
||||||
- run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
|
||||||
- run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
- name: Install crane
|
||||||
- run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
||||||
- name: Build
|
|
||||||
|
- name: Log in to registry
|
||||||
|
run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
||||||
|
env:
|
||||||
|
CRANE_INSECURE: "true"
|
||||||
|
|
||||||
|
|
||||||
|
- name: Install trivy
|
||||||
|
run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
||||||
|
|
||||||
|
- name: Scan api
|
||||||
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o api-scan.json local/api:build
|
||||||
|
|
||||||
|
- name: Compute short SHA
|
||||||
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
- name: Build api
|
||||||
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
|
run: docker build -f infra/images/api.Dockerfile -t local/api:build .
|
||||||
- name: Scan
|
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --trivyignores .trivyignore --scanners vuln local/api:build
|
- name: Push api
|
||||||
- name: Push
|
|
||||||
run: |
|
run: |
|
||||||
docker save local/api:build -o /tmp/img.tar
|
docker save local/api:build -o /tmp/api.tar
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/api:${{ env.SHORT_SHA }} --insecure
|
crane push /tmp/api.tar git:3000/homelab/stack/api:${{ env.SHORT_SHA }} --insecure
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/api:latest --insecure
|
crane push /tmp/api.tar git:3000/homelab/stack/api:latest --insecure
|
||||||
|
|
||||||
mc:
|
mc:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: >-
|
if: >-
|
||||||
contains(github.event.head_commit.modified, 'infra/images/mc.Dockerfile') ||
|
contains(github.event.head_commit.modified, 'infra/rustfs') ||
|
||||||
contains(github.event.head_commit.modified, 'infra/rustfs/')
|
contains(github.event.head_commit.modified, 'infra/images/mc.Dockerfile')
|
||||||
steps:
|
steps:
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- name: Checkout
|
||||||
- run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
uses: https://github.com/actions/checkout@v4
|
||||||
- run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
|
||||||
- run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
- name: Install crane
|
||||||
- name: Build
|
run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
||||||
|
|
||||||
|
- name: Log in to registry
|
||||||
|
run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}"
|
||||||
|
env:
|
||||||
|
CRANE_INSECURE: "true"
|
||||||
|
|
||||||
|
|
||||||
|
- name: Compute short SHA
|
||||||
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
- name: Build mc
|
||||||
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
|
||||||
- name: Push
|
|
||||||
|
- name: Push mc
|
||||||
run: |
|
run: |
|
||||||
docker save local/mc:build -o /tmp/img.tar
|
docker save local/mc:build -o /tmp/mc.tar
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/mc:${{ env.SHORT_SHA }} --insecure
|
crane push /tmp/mc.tar git:3000/homelab/stack/mc:${{ env.SHORT_SHA }} --insecure
|
||||||
crane push /tmp/img.tar git:3000/homelab/stack/mc:latest --insecure
|
crane push /tmp/mc.tar git:3000/homelab/stack/mc:latest --insecure
|
||||||
|
|
||||||
# ── Restart services that got new images ───────────────────────
|
report:
|
||||||
|
|
||||||
restart:
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [notebooks, zotero, docs, api, mc]
|
needs: [notebooks, zotero, docs, api, mc]
|
||||||
if: always() && !cancelled()
|
if: always() && !cancelled()
|
||||||
steps:
|
steps:
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- name: Checkout
|
||||||
|
uses: https://github.com/actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up uv
|
- name: Set up uv
|
||||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||||
env:
|
env:
|
||||||
UV_INSTALL_DIR: /usr/local/bin
|
UV_INSTALL_DIR: /usr/local/bin
|
||||||
- run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
|
||||||
- name: Deploy changed services
|
- name: Report
|
||||||
env:
|
env:
|
||||||
GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
|
GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
uv sync --no-dev --quiet 2>/dev/null || true
|
uv sync --no-dev --quiet 2>/dev/null || true
|
||||||
uv run python -m api.diag.ci \
|
uv run python -m api.diag.ci \
|
||||||
--workflow "Deploy" --job "restart" \
|
--workflow "Deploy" --job "report" \
|
||||||
--run "${{ github.run_number }}" \
|
--run "${{ github.run_number }}" \
|
||||||
--sha "${{ github.sha }}" \
|
--sha "${{ github.sha }}" \
|
||||||
--ref "${{ github.ref }}" || true
|
--ref "${{ github.ref }}" || true
|
||||||
|
|||||||
@@ -77,16 +77,16 @@ jobs:
|
|||||||
crane push /tmp/mc.tar git:3000/homelab/stack/mc:latest --insecure
|
crane push /tmp/mc.tar git:3000/homelab/stack/mc:latest --insecure
|
||||||
|
|
||||||
- name: Scan notebooks
|
- name: Scan notebooks
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --format json -o notebooks-scan.json local/notebooks:build
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o notebooks-scan.json local/notebooks:build
|
||||||
|
|
||||||
- name: Scan zotero
|
- name: Scan zotero
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --format json -o zotero-scan.json local/zotero:build
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o zotero-scan.json local/zotero:build
|
||||||
|
|
||||||
- name: Scan docs
|
- name: Scan docs
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --format json -o docs-scan.json local/docs:build
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o docs-scan.json local/docs:build
|
||||||
|
|
||||||
- name: Scan api
|
- name: Scan api
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --format json -o api-scan.json local/api:build
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o api-scan.json local/api:build
|
||||||
|
|
||||||
- name: Close resolved or file new vuln issues
|
- name: Close resolved or file new vuln issues
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ jobs:
|
|||||||
|
|
||||||
|
|
||||||
- name: Build notebooks
|
- name: Build notebooks
|
||||||
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build .
|
run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
|
||||||
|
|
||||||
- name: File failure issue
|
- name: File failure issue
|
||||||
if: failure()
|
if: failure()
|
||||||
|
|||||||
@@ -78,16 +78,16 @@ jobs:
|
|||||||
crane push /tmp/mc.tar git:3000/homelab/stack/mc:latest --insecure
|
crane push /tmp/mc.tar git:3000/homelab/stack/mc:latest --insecure
|
||||||
|
|
||||||
- name: Scan notebooks
|
- name: Scan notebooks
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --format json -o notebooks-scan.json local/notebooks:build
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o notebooks-scan.json local/notebooks:build
|
||||||
|
|
||||||
- name: Scan zotero
|
- name: Scan zotero
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --format json -o zotero-scan.json local/zotero:build
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o zotero-scan.json local/zotero:build
|
||||||
|
|
||||||
- name: Scan docs
|
- name: Scan docs
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --format json -o docs-scan.json local/docs:build
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o docs-scan.json local/docs:build
|
||||||
|
|
||||||
- name: Scan api
|
- name: Scan api
|
||||||
run: trivy image --severity HIGH,CRITICAL --exit-code 1 --format json -o api-scan.json local/api:build
|
run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o api-scan.json local/api:build
|
||||||
|
|
||||||
- name: Report vulnerabilities
|
- name: Report vulnerabilities
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -148,7 +148,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
lint-test:
|
lint:
|
||||||
runs-on: {runner}
|
runs-on: {runner}
|
||||||
steps:
|
steps:
|
||||||
{_checkout_step()}
|
{_checkout_step()}
|
||||||
@@ -164,37 +164,24 @@ jobs:
|
|||||||
- name: Ruff format
|
- name: Ruff format
|
||||||
run: uv run ruff format --check src/ tests/
|
run: uv run ruff format --check src/ tests/
|
||||||
|
|
||||||
- name: Pytest
|
|
||||||
run: uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under={coverage_threshold} -q
|
|
||||||
|
|
||||||
- name: Validate generated config
|
- name: Validate generated config
|
||||||
run: uv run python dev/scripts/gen_config.py --check
|
run: uv run python dev/scripts/gen_config.py --check
|
||||||
|
|
||||||
{_failure_step("CI", "lint-test")}
|
test:
|
||||||
|
|
||||||
skinny-install:
|
|
||||||
runs-on: {runner}
|
runs-on: {runner}
|
||||||
strategy:
|
needs: lint
|
||||||
matrix:
|
|
||||||
extra: [conf, aco, api, bcda, bib, bls, ccw, cli, cms, opps, perf, pfs, rex]
|
|
||||||
steps:
|
steps:
|
||||||
{_checkout_step()}
|
{_checkout_step()}
|
||||||
|
|
||||||
{_setup_uv_step(uv_version)}
|
{_setup_uv_step(uv_version)}
|
||||||
|
|
||||||
- name: Install stack[${{{{ matrix.extra }}}}]
|
- name: Install dependencies
|
||||||
run: uv sync --no-dev --extra ${{{{ matrix.extra }}}}
|
run: uv sync --dev
|
||||||
|
|
||||||
- name: Verify import
|
- name: Pytest
|
||||||
run: uv run python -c "import ${{{{ matrix.extra }}}}"
|
run: uv run pytest tests/ -x --cov=src --cov-report=term-missing --cov-fail-under={coverage_threshold} -q
|
||||||
|
|
||||||
- name: Run module tests
|
{_failure_step("CI", "test")}
|
||||||
run: |
|
|
||||||
if [ -d "tests/${{{{ matrix.extra }}}}" ]; then
|
|
||||||
uv run pytest "tests/${{{{ matrix.extra }}}}/" -x -q || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
{_failure_step("CI", "skinny-install")}
|
|
||||||
"""
|
"""
|
||||||
return (".gitea/workflows/ci.yml", content)
|
return (".gitea/workflows/ci.yml", content)
|
||||||
|
|
||||||
@@ -208,24 +195,54 @@ def _gen_deploy(
|
|||||||
uv_version: str,
|
uv_version: str,
|
||||||
**_kw: object,
|
**_kw: object,
|
||||||
) -> tuple[str, str]:
|
) -> tuple[str, str]:
|
||||||
# Build steps — sequential (Gitea matrix has edge cases)
|
scan_set = {s["name"] for s in scans}
|
||||||
build_steps = []
|
|
||||||
|
# Generate one job per image with path filters
|
||||||
|
job_blocks = []
|
||||||
|
job_names = []
|
||||||
for img in images:
|
for img in images:
|
||||||
|
name = img["name"]
|
||||||
|
job_names.append(name)
|
||||||
|
paths = img.get("path_filter", [])
|
||||||
|
# Gitea Actions doesn't support `paths:` on push without branches,
|
||||||
|
# so use `if: contains(...)` on the modified files list.
|
||||||
|
def _clean_path(p: str) -> str:
|
||||||
|
return p.rstrip("*").rstrip("/")
|
||||||
|
|
||||||
|
path_checks = " ||\n ".join(
|
||||||
|
f"contains(github.event.head_commit.modified, '{_clean_path(p)}')"
|
||||||
|
for p in paths
|
||||||
|
)
|
||||||
tags = (
|
tags = (
|
||||||
f"{registry}/{owner_repo}/{img['name']}:${{{{ env.SHORT_SHA }}}},"
|
f"{registry}/{owner_repo}/{name}:${{{{ env.SHORT_SHA }}}},"
|
||||||
f"{registry}/{owner_repo}/{img['name']}:latest"
|
f"{registry}/{owner_repo}/{name}:latest"
|
||||||
)
|
)
|
||||||
build_steps.append(_build_push_step(img, tags, registry, owner_repo))
|
scan_step = ""
|
||||||
build_block = "\n\n".join(build_steps)
|
if name in scan_set:
|
||||||
|
scan_step = f"""
|
||||||
|
|
||||||
scan_steps = []
|
{_install_trivy_step()}
|
||||||
for img in scans:
|
|
||||||
scan_steps.append(
|
|
||||||
_trivy_step(img, "${{ env.SHORT_SHA }}", registry, owner_repo)
|
|
||||||
)
|
|
||||||
scan_block = "\n\n".join(scan_steps)
|
|
||||||
|
|
||||||
vuln_files = " ".join(f"{i['name']}-scan.json" for i in scans)
|
{_trivy_step(img, "${{{{ env.SHORT_SHA }}}}", registry, owner_repo)}"""
|
||||||
|
|
||||||
|
job_blocks.append(f"""\
|
||||||
|
{name}:
|
||||||
|
runs-on: {runner}
|
||||||
|
if: >-
|
||||||
|
{path_checks}
|
||||||
|
steps:
|
||||||
|
{_checkout_step()}
|
||||||
|
|
||||||
|
{_docker_login_step(registry)}
|
||||||
|
{scan_step}
|
||||||
|
|
||||||
|
- name: Compute short SHA
|
||||||
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
{_build_push_step(img, tags, registry, owner_repo)}""")
|
||||||
|
|
||||||
|
jobs_block = "\n\n".join(job_blocks)
|
||||||
|
needs_list = ", ".join(job_names)
|
||||||
|
|
||||||
content = f"""\
|
content = f"""\
|
||||||
{_HEADER}
|
{_HEADER}
|
||||||
@@ -236,34 +253,27 @@ on:
|
|||||||
branches: [main]
|
branches: [main]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-scan-report:
|
{jobs_block}
|
||||||
|
|
||||||
|
report:
|
||||||
runs-on: {runner}
|
runs-on: {runner}
|
||||||
|
needs: [{needs_list}]
|
||||||
|
if: always() && !cancelled()
|
||||||
steps:
|
steps:
|
||||||
{_checkout_step()}
|
{_checkout_step()}
|
||||||
|
|
||||||
{_docker_login_step(registry)}
|
|
||||||
|
|
||||||
{_install_trivy_step()}
|
|
||||||
|
|
||||||
{_setup_uv_step(uv_version)}
|
{_setup_uv_step(uv_version)}
|
||||||
|
|
||||||
- name: Compute short SHA
|
- name: Report
|
||||||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
{build_block}
|
|
||||||
|
|
||||||
{scan_block}
|
|
||||||
|
|
||||||
- name: Report vulnerabilities
|
|
||||||
env:
|
env:
|
||||||
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
||||||
run: |
|
run: |
|
||||||
uv sync --no-dev
|
uv sync --no-dev --quiet 2>/dev/null || true
|
||||||
for f in {vuln_files}; do
|
uv run python -m api.diag.ci \\
|
||||||
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
|
--workflow "Deploy" --job "report" \\
|
||||||
done
|
--run "${{{{ github.run_number }}}}" \\
|
||||||
|
--sha "${{{{ github.sha }}}}" \\
|
||||||
{_failure_step("Deploy", "build-scan-report")}
|
--ref "${{{{ github.ref }}}}" || true
|
||||||
"""
|
"""
|
||||||
return (".gitea/workflows/deploy.yml", content)
|
return (".gitea/workflows/deploy.yml", content)
|
||||||
|
|
||||||
|
|||||||
@@ -186,6 +186,6 @@ markers = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[tool.uv.build-backend]
|
[tool.uv.build-backend]
|
||||||
module-name = ["aco", "api", "bcda", "bib", "bls", "ccw", "cli", "cms", "conf", "opps", "perf", "pfs", "rec", "rex", "sem"]
|
module-name = ["aco", "api", "bcda", "bib", "bls", "ccw", "cli", "cms", "conf", "mail", "opps", "perf", "pfs", "prisma", "rec", "rex", "sem"]
|
||||||
namespace = true
|
namespace = true
|
||||||
source-exclude = ["compose.yml","infra/**","data/**","notebooks/**","tuva/**","assets/**","docs/**","dev/**","bundle/**","cloud/**"]
|
source-exclude = ["compose.yml","infra/**","data/**","notebooks/**","tuva/**","assets/**","docs/**","dev/**","bundle/**","cloud/**"]
|
||||||
|
|||||||
Reference in New Issue
Block a user