Add CI/CD pipeline and documentation

2026-02-04 19:51:32 -05:00
parent 906d3aa318
commit 21dce22688
5 changed files with 496 additions and 2 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,6 @@ zotero/data/

 # Marimo cache
 notebooks/__marimo__/
+.deploy_key
+.env
+.deploy_key.pub
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@@ -0,0 +1,58 @@
+when:
+  - event: [push, pull_request]
+
+steps:
+  - name: validate
+    image: docker:cli
+    commands:
+      - docker compose config --quiet
+
+  - name: build-notebooks
+    image: docker:dind
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    commands:
+      - docker build -t localhost:3000/homelab/notebooks:${CI_COMMIT_SHA:0:8} ./notebooks
+      - docker tag localhost:3000/homelab/notebooks:${CI_COMMIT_SHA:0:8} localhost:3000/homelab/notebooks:latest
+    when:
+      - path: "notebooks/**"
+
+  - name: build-zotero
+    image: docker:dind
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    commands:
+      - docker build -t localhost:3000/homelab/zotero:${CI_COMMIT_SHA:0:8} ./zotero
+      - docker tag localhost:3000/homelab/zotero:${CI_COMMIT_SHA:0:8} localhost:3000/homelab/zotero:latest
+    when:
+      - path: "zotero/**"
+
+  - name: push-images
+    image: docker:dind
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      REGISTRY_USER:
+        from_secret: registry_user
+      REGISTRY_PASS:
+        from_secret: registry_pass
+    commands:
+      - echo "$REGISTRY_PASS" | docker login localhost:3000 -u "$REGISTRY_USER" --password-stdin
+      - docker push localhost:3000/homelab/notebooks:${CI_COMMIT_SHA:0:8} || true
+      - docker push localhost:3000/homelab/notebooks:latest || true
+      - docker push localhost:3000/homelab/zotero:${CI_COMMIT_SHA:0:8} || true
+      - docker push localhost:3000/homelab/zotero:latest || true
+    when:
+      - event: push
+        branch: main
+
+  - name: deploy
+    image: docker:cli
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    commands:
+      - docker compose pull
+      - docker compose up -d
+    when:
+      - event: push
+        branch: main
--- a/README.md
+++ b/README.md
@@ -0,0 +1,276 @@
+# Homelab Stack
+
+Self-hosted infrastructure stack with GPU support, S3-compatible storage, Git server with container registry, CI/CD, and research tools.
+
+## Services
+
+| Service | URL | Description |
+|---------|-----|-------------|
+| Gitea | http://gitea.home.fhirworx.io:3000 | Git server with LFS + container registry |
+| Woodpecker CI | http://ci.home.fhirworx.io:8000 | CI/CD pipelines |
+| RustFS | http://s3.home.fhirworx.io:9000 | S3-compatible object storage |
+| RustFS Console | http://s3.home.fhirworx.io:9001 | Storage management UI |
+| Notebooks | http://notebooks.home.fhirworx.io:2718 | GPU-accelerated Jupyter |
+| Zotero | http://zotero.home.fhirworx.io:8080 | Reference manager with VNC |
+| PostgreSQL | localhost:5432 | Database (Bitnami hardened) |
+
+## Prerequisites
+
+- Docker with rootless mode
+- NVIDIA GPU with container toolkit
+- Fix for rootless NVIDIA containers:
+  ```bash
+  sudo sed -i 's/#no-cgroups = false/no-cgroups = true/' /etc/nvidia-container-runtime/config.toml
+  ```
+
+## DNS Setup
+
+Add to `/etc/hosts` (replace with your server's LAN IP):
+
+```bash
+cat << 'EOF' | sudo tee -a /etc/hosts
+
+# homelab stack
+192.168.1.192 gitea.home.fhirworx.io
+192.168.1.192 ci.home.fhirworx.io
+192.168.1.192 s3.home.fhirworx.io
+192.168.1.192 notebooks.home.fhirworx.io
+192.168.1.192 zotero.home.fhirworx.io
+EOF
+```
+
+Or configure these records on your router/DNS server.
+
+## Setup
+
+### 1. Environment Variables
+
+Create `.env` file:
+
+```bash
+POSTGRES_PASSWORD=<your_password>
+RUSTFS_ACCESS_KEY=<your_access_key>
+RUSTFS_SECRET_KEY=<your_secret_key>
+GITEA_S3_ACCESS_KEY=<gitea_s3_user>
+GITEA_S3_SECRET_KEY=<gitea_s3_password>
+GITEA_DB_PASSWORD=<gitea_db_password>
+GITEA_TOKEN=<gitea_api_token>
+WOODPECKER_DB_PASSWORD=<woodpecker_db_password>
+WOODPECKER_AGENT_SECRET=<generated_secret>
+WOODPECKER_GITEA_CLIENT=<oauth_client_id>
+WOODPECKER_GITEA_SECRET=<oauth_client_secret>
+```
+
+Generate agent secret:
+```bash
+openssl rand -hex 32
+```
+
+### 2. Start Core Services
+
+```bash
+docker compose up -d postgres rustfs
+```
+
+### 3. Configure RustFS
+
+1. Access console at http://s3.home.fhirworx.io:9001
+2. Create user `git` with S3 credentials matching `GITEA_S3_ACCESS_KEY` and `GITEA_S3_SECRET_KEY`
+3. Create buckets: `gitea`, `gitea-lfs`, `gitea-packages`
+4. Apply IAM policy to `git` user:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetBucketLocation",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads"
+      ],
+      "Resource": [
+        "arn:aws:s3:::gitea",
+        "arn:aws:s3:::gitea-lfs",
+        "arn:aws:s3:::gitea-packages"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:DeleteObject",
+        "s3:ListMultipartUploadParts",
+        "s3:AbortMultipartUpload"
+      ],
+      "Resource": [
+        "arn:aws:s3:::gitea/*",
+        "arn:aws:s3:::gitea-lfs/*",
+        "arn:aws:s3:::gitea-packages/*"
+      ]
+    }
+  ]
+}
+```
+
+### 4. Configure PostgreSQL
+
+Create databases and users:
+
+```bash
+# Gitea database
+docker exec -e PGPASSWORD=<postgres_password> postgres psql -U postgres -c "CREATE DATABASE gitea;"
+docker exec -e PGPASSWORD=<postgres_password> postgres psql -U postgres -c "CREATE USER git WITH PASSWORD '<gitea_db_password>'; ALTER DATABASE gitea OWNER TO git;"
+
+# Woodpecker database
+docker exec -e PGPASSWORD=<postgres_password> postgres psql -U postgres -c "CREATE DATABASE woodpecker;"
+docker exec -e PGPASSWORD=<postgres_password> postgres psql -U postgres -c "CREATE USER woodpecker WITH PASSWORD '<woodpecker_db_password>'; ALTER DATABASE woodpecker OWNER TO woodpecker;"
+```
+
+### 5. Start Gitea
+
+```bash
+docker compose up -d gitea
+```
+
+### 6. Configure Gitea
+
+1. Complete initial setup at http://gitea.home.fhirworx.io:3000
+2. Create organization `homelab`
+3. Create repository `stack`
+4. Generate API token for container registry access
+5. Create OAuth2 application for Woodpecker:
+   - Name: `Woodpecker CI`
+   - Redirect URI: `http://ci.home.fhirworx.io:8000/authorize`
+   - Copy Client ID and Secret to `.env`
+
+### 7. Start Woodpecker CI
+
+```bash
+docker compose up -d woodpecker-server woodpecker-agent
+```
+
+Add secrets in Woodpecker UI (http://ci.home.fhirworx.io:8000):
+- `registry_user` - Gitea username
+- `registry_pass` - Gitea token
+
+### 8. SSH and GPG Keys
+
+Generate SSH key:
+```bash
+ssh-keygen -t ed25519 -C "user@homelab" -f ~/.ssh/gitea_ed25519 -N ""
+```
+
+Generate GPG key:
+```bash
+gpg --batch --gen-key <<EOF
+Key-Type: eddsa
+Key-Curve: ed25519
+Key-Usage: sign
+Subkey-Type: ecdh
+Subkey-Curve: cv25519
+Subkey-Usage: encrypt
+Name-Real: <your_name>
+Name-Email: <your_email>
+Expire-Date: 2y
+%no-protection
+%commit
+EOF
+```
+
+Configure Git:
+```bash
+git config --global user.name "<your_name>"
+git config --global user.email "<your_email>"
+git config --global user.signingkey <GPG_KEY_ID>
+git config --global commit.gpgsign true
+```
+
+Add to `~/.ssh/config`:
+```
+Host gitea
+    HostName gitea.home.fhirworx.io
+    Port 2222
+    User git
+    IdentityFile ~/.ssh/gitea_ed25519
+    IdentitiesOnly yes
+```
+
+Add host key:
+```bash
+ssh-keyscan -p 2222 gitea.home.fhirworx.io >> ~/.ssh/known_hosts
+```
+
+Add SSH and GPG public keys to Gitea: http://gitea.home.fhirworx.io:3000/user/settings/keys
+
+### 9. Deploy Key (for CI/CD)
+
+```bash
+ssh-keygen -t ed25519 -C "deploy@homelab/stack" -f .deploy_key -N ""
+```
+
+Add `.deploy_key.pub` to repository deploy keys: http://gitea.home.fhirworx.io:3000/homelab/stack/settings/keys
+
+### 10. Push to Gitea
+
+```bash
+git remote add gitea gitea:homelab/stack.git
+git push -u gitea main
+```
+
+### 11. Container Registry
+
+Login:
+```bash
+echo "<GITEA_TOKEN>" | docker login localhost:3000 -u <username> --password-stdin
+```
+
+Tag and push images:
+```bash
+docker tag <image>:latest localhost:3000/homelab/<image>:latest
+docker push localhost:3000/homelab/<image>:latest
+```
+
+### 12. Start All Services
+
+```bash
+docker compose up -d
+```
+
+## Container Registry Images
+
+All images are hosted in the Gitea container registry:
+
+- `localhost:3000/homelab/postgresql:latest`
+- `localhost:3000/homelab/rustfs:latest`
+- `localhost:3000/homelab/gitea:latest-rootless`
+- `localhost:3000/homelab/woodpecker-server:latest`
+- `localhost:3000/homelab/woodpecker-agent:latest`
+- `localhost:3000/homelab/notebooks:latest`
+- `localhost:3000/homelab/zotero:latest`
+
+## CI/CD Pipeline
+
+The `.woodpecker.yml` pipeline:
+
+1. **validate** - Validates docker compose config
+2. **build-notebooks** - Builds notebooks image (on changes to `notebooks/`)
+3. **build-zotero** - Builds zotero image (on changes to `zotero/`)
+4. **push-images** - Pushes to container registry (on main branch)
+5. **deploy** - Pulls and restarts services (on main branch)
+
+## Endpoints
+
+| Service | URL |
+|---------|-----|
+| Gitea Web | http://gitea.home.fhirworx.io:3000 |
+| Gitea SSH | ssh://git@gitea.home.fhirworx.io:2222 |
+| Container Registry | http://gitea.home.fhirworx.io:3000/v2/ |
+| Woodpecker CI | http://ci.home.fhirworx.io:8000 |
+| RustFS S3 API | http://s3.home.fhirworx.io:9000 |
+| RustFS Console | http://s3.home.fhirworx.io:9001 |
+| PostgreSQL | localhost:5432 |
+| Notebooks | http://notebooks.home.fhirworx.io:2718 |
+| Zotero | http://zotero.home.fhirworx.io:8080 |
--- a/compose.yml
+++ b/compose.yml
@@ -1,6 +1,122 @@
 services:
+  rustfs:
+    image: localhost:3000/homelab/rustfs:latest
+    container_name: rustfs
+    user: "10001:10001"
+    environment:
+      - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY}
+      - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY}
+    volumes:
+      - rustfs_data:/data
+      - rustfs_logs:/logs
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+
+  postgres:
+    image: localhost:3000/homelab/postgresql:latest
+    container_name: postgres
+    environment:
+      - POSTGRESQL_PASSWORD=${POSTGRES_PASSWORD:-changeme}
+      - POSTGRESQL_DATABASE=gitea
+    volumes:
+      - postgres_data:/bitnami/postgresql
+    ports:
+      - "5432:5432"
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+
+  gitea:
+    image: localhost:3000/homelab/gitea:latest-rootless
+    container_name: gitea
+    environment:
+      - GITEA__database__DB_TYPE=postgres
+      - GITEA__database__HOST=postgres:5432
+      - GITEA__database__NAME=gitea
+      - GITEA__database__USER=git
+      - GITEA__database__PASSWD=${GITEA_DB_PASSWORD:-git_password}
+      - GITEA__storage__STORAGE_TYPE=minio
+      - GITEA__storage__MINIO_ENDPOINT=rustfs:9000
+      - GITEA__storage__MINIO_ACCESS_KEY_ID=${GITEA_S3_ACCESS_KEY:-git}
+      - GITEA__storage__MINIO_SECRET_ACCESS_KEY=${GITEA_S3_SECRET_KEY}
+      - GITEA__storage__MINIO_BUCKET=gitea
+      - GITEA__storage__MINIO_USE_SSL=false
+      - GITEA__lfs__STORAGE_TYPE=minio
+      - GITEA__lfs__MINIO_ENDPOINT=rustfs:9000
+      - GITEA__lfs__MINIO_ACCESS_KEY_ID=${GITEA_S3_ACCESS_KEY:-git}
+      - GITEA__lfs__MINIO_SECRET_ACCESS_KEY=${GITEA_S3_SECRET_KEY}
+      - GITEA__lfs__MINIO_BUCKET=gitea-lfs
+      - GITEA__lfs__MINIO_USE_SSL=false
+      - GITEA__packages__ENABLED=true
+      - GITEA__packages__STORAGE_TYPE=minio
+      - GITEA__packages__MINIO_ENDPOINT=rustfs:9000
+      - GITEA__packages__MINIO_ACCESS_KEY_ID=${GITEA_S3_ACCESS_KEY:-git}
+      - GITEA__packages__MINIO_SECRET_ACCESS_KEY=${GITEA_S3_SECRET_KEY}
+      - GITEA__packages__MINIO_BUCKET=gitea-packages
+      - GITEA__packages__MINIO_USE_SSL=false
+      - GITEA__server__DOMAIN=gitea.home.fhirworx.io
+      - GITEA__server__ROOT_URL=http://gitea.home.fhirworx.io:3000/
+      - GITEA__server__SSH_DOMAIN=gitea.home.fhirworx.io
+    volumes:
+      - gitea_data:/var/lib/gitea
+      - gitea_config:/etc/gitea
+    ports:
+      - "3000:3000"
+      - "2222:2222"
+    depends_on:
+      - postgres
+      - rustfs
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+  woodpecker-server:
+    image: localhost:3000/homelab/woodpecker-server:v2.7.3
+    container_name: woodpecker-server
+    environment:
+      - WOODPECKER_HOST=http://ci.home.fhirworx.io:8000
+      - WOODPECKER_WEBHOOK_HOST=http://woodpecker-server:8000
+      - WOODPECKER_LOG_LEVEL=debug
+      - WOODPECKER_OPEN=true
+      - WOODPECKER_GITEA=true
+      - WOODPECKER_GITEA_URL=http://gitea:3000
+      - WOODPECKER_GITEA_CLIENT=${WOODPECKER_GITEA_CLIENT}
+      - WOODPECKER_GITEA_SECRET=${WOODPECKER_GITEA_SECRET}
+      - WOODPECKER_AGENT_SECRET=${WOODPECKER_AGENT_SECRET}
+      - WOODPECKER_DATABASE_DRIVER=postgres
+      - WOODPECKER_DATABASE_DATASOURCE=postgres://woodpecker:${WOODPECKER_DB_PASSWORD}@postgres:5432/woodpecker?sslmode=disable
+    volumes:
+      - woodpecker_data:/var/lib/woodpecker
+    ports:
+      - "8000:8000"
+    depends_on:
+      - gitea
+      - postgres
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+
+  woodpecker-agent:
+    image: localhost:3000/homelab/woodpecker-agent:v2.7.3
+    container_name: woodpecker-agent
+    environment:
+      - WOODPECKER_SERVER=woodpecker-server:9000
+      - WOODPECKER_AGENT_SECRET=${WOODPECKER_AGENT_SECRET}
+      - WOODPECKER_MAX_WORKFLOWS=4
+      - DOCKER_HOST=unix:///var/run/docker.sock
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    depends_on:
+      - woodpecker-server
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+
  notebooks:
-    build: ./notebooks
+    image: localhost:3000/homelab/notebooks:latest
    container_name: notebooks
    ports:
      - "2718:2718"
@@ -18,7 +134,7 @@ services:
    restart: unless-stopped

  zotero:
-    build: ./zotero
+    image: localhost:3000/homelab/zotero:latest
    container_name: zotero
    runtime: nvidia
    stdin_open: true
@@ -50,3 +166,11 @@ services:
              count: 1
              capabilities: [gpu]
    restart: unless-stopped
+
+volumes:
+  postgres_data:
+  rustfs_data:
+  rustfs_logs:
+  gitea_data:
+  gitea_config:
+  woodpecker_data:
--- a/rustfs/iam.json
+++ b/rustfs/iam.json
@@ -0,0 +1,33 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Effect": "Allow",
+        "Action": [
+          "s3:GetBucketLocation",
+          "s3:ListBucket",
+          "s3:ListBucketMultipartUploads"
+        ],
+        "Resource": [
+          "arn:aws:s3:::gitea",
+          "arn:aws:s3:::gitea-lfs",
+          "arn:aws:s3:::gitea-packages"
+        ]
+      },
+      {
+        "Effect": "Allow",
+        "Action": [
+          "s3:GetObject",
+          "s3:PutObject",
+          "s3:DeleteObject",
+          "s3:ListMultipartUploadParts",
+          "s3:AbortMultipartUpload"
+        ],
+        "Resource": [
+          "arn:aws:s3:::gitea/*",
+          "arn:aws:s3:::gitea-lfs/*",
+          "arn:aws:s3:::gitea-packages/*"
+        ]
+      }
+    ]
+  }