replace devpi/apt-cacher-ng with Gitea package registry
Some checks failed
Infra CI / notebooks (push) Failing after 47s
Infra CI / zotero (push) Successful in 6s
Infra CI / docs (push) Successful in 5s
Infra CI / mc (push) Successful in 6s
Infra CI / zotero (pull_request) Successful in 7s
Infra CI / api (pull_request) Successful in 5s
Infra CI / api (push) Successful in 29s
CI / lint-test (push) Successful in 1m33s
Infra CI / notebooks (pull_request) Failing after 36s
Infra CI / docs (pull_request) Successful in 6s
Infra CI / mc (pull_request) Successful in 6s
CI / lint-test (pull_request) Successful in 1m29s
Package Supply Chain / pkg-supply-chain (push) Failing after 52s
Some checks failed
Infra CI / notebooks (push) Failing after 47s
Infra CI / zotero (push) Successful in 6s
Infra CI / docs (push) Successful in 5s
Infra CI / mc (push) Successful in 6s
Infra CI / zotero (pull_request) Successful in 7s
Infra CI / api (pull_request) Successful in 5s
Infra CI / api (push) Successful in 29s
CI / lint-test (push) Successful in 1m33s
Infra CI / notebooks (pull_request) Failing after 36s
Infra CI / docs (pull_request) Successful in 6s
Infra CI / mc (pull_request) Successful in 6s
CI / lint-test (pull_request) Successful in 1m29s
Package Supply Chain / pkg-supply-chain (push) Failing after 52s
closes #160, closes #161, closes #162, closes #163, refs #164 - Remove apt-cacher-ng and devpi containers from compose.yml - pkg_mirror_sync.py: downloads from upstream, uploads to Gitea's built-in PyPI and Debian registries via API - Dockerfiles: add PYPI_INDEX_URL build-arg for local registry - CI build steps pass --build-arg PYPI_INDEX_URL for mirror builds - stack.toml: add [mirrors] section pointing at Gitea endpoints - Add packages.homelab.fhirworx.io subdomain (traefik + coredns) - Verified: 39 PyPI wheels + 146 Debian .debs uploaded to Gitea
This commit is contained in:
@@ -34,7 +34,7 @@ jobs:
|
||||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build notebooks
|
||||
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
|
||||
- name: Push notebooks
|
||||
run: |
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
||||
|
||||
- name: Build zotero
|
||||
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
|
||||
- name: Push zotero
|
||||
run: |
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
||||
|
||||
- name: Build docs
|
||||
run: docker build -f docs/Dockerfile -t local/docs:build .
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f docs/Dockerfile -t local/docs:build .
|
||||
|
||||
- name: Push docs
|
||||
run: |
|
||||
@@ -61,7 +61,7 @@ jobs:
|
||||
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
||||
|
||||
- name: Build api
|
||||
run: docker build -f api/Dockerfile -t local/api:build .
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f api/Dockerfile -t local/api:build .
|
||||
|
||||
- name: Push api
|
||||
run: |
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
||||
|
||||
- name: Build mc
|
||||
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
|
||||
- name: Push mc
|
||||
run: |
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
UV_INSTALL_DIR: /usr/local/bin
|
||||
|
||||
- name: Build notebooks
|
||||
run: docker build --no-cache -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
|
||||
- name: Push notebooks
|
||||
run: |
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
||||
|
||||
- name: Build zotero
|
||||
run: docker build --no-cache -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
|
||||
- name: Push zotero
|
||||
run: |
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
||||
|
||||
- name: Build docs
|
||||
run: docker build --no-cache -f docs/Dockerfile -t local/docs:build .
|
||||
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f docs/Dockerfile -t local/docs:build .
|
||||
|
||||
- name: Push docs
|
||||
run: |
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
||||
|
||||
- name: Build api
|
||||
run: docker build --no-cache -f api/Dockerfile -t local/api:build .
|
||||
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f api/Dockerfile -t local/api:build .
|
||||
|
||||
- name: Push api
|
||||
run: |
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
||||
|
||||
- name: Build mc
|
||||
run: docker build --no-cache -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
|
||||
- name: Push mc
|
||||
run: |
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
|
||||
|
||||
- name: Build notebooks
|
||||
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
|
||||
zotero:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -54,7 +54,7 @@ jobs:
|
||||
|
||||
|
||||
- name: Build zotero
|
||||
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
|
||||
docs:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
|
||||
|
||||
- name: Build docs
|
||||
run: docker build -f docs/Dockerfile -t local/docs:build .
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f docs/Dockerfile -t local/docs:build .
|
||||
|
||||
api:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
|
||||
|
||||
- name: Build api
|
||||
run: docker build -f api/Dockerfile -t local/api:build .
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f api/Dockerfile -t local/api:build .
|
||||
|
||||
mc:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -102,4 +102,4 @@ jobs:
|
||||
|
||||
|
||||
- name: Build mc
|
||||
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
|
||||
@@ -36,7 +36,9 @@ jobs:
|
||||
- name: Check manifest freshness
|
||||
run: uv run python dev/scripts/pkg_inventory.py --check
|
||||
|
||||
- name: Sync mirrors
|
||||
- name: Sync to Gitea package registry
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
run: uv run python dev/scripts/pkg_mirror_sync.py
|
||||
|
||||
- name: Drift detection
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build notebooks
|
||||
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f notebooks/Dockerfile -t local/notebooks:build notebooks/
|
||||
|
||||
- name: Push notebooks
|
||||
run: |
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
|
||||
|
||||
- name: Build zotero
|
||||
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f zotero/Dockerfile -t local/zotero:build zotero/
|
||||
|
||||
- name: Push zotero
|
||||
run: |
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
|
||||
|
||||
- name: Build docs
|
||||
run: docker build -f docs/Dockerfile -t local/docs:build .
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f docs/Dockerfile -t local/docs:build .
|
||||
|
||||
- name: Push docs
|
||||
run: |
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
|
||||
|
||||
- name: Build api
|
||||
run: docker build -f api/Dockerfile -t local/api:build .
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f api/Dockerfile -t local/api:build .
|
||||
|
||||
- name: Push api
|
||||
run: |
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
|
||||
|
||||
- name: Build mc
|
||||
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
|
||||
|
||||
- name: Push mc
|
||||
run: |
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -3,6 +3,7 @@
|
||||
|
||||
# Data directories (user data, not tracked)
|
||||
data/
|
||||
mirrors/cache/
|
||||
zotero/data/
|
||||
zotero/profiles/
|
||||
tuva/
|
||||
|
||||
@@ -3,6 +3,9 @@ FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Local package registry (Gitea) — set via --build-arg to pull from mirror
|
||||
ARG PYPI_INDEX_URL=""
|
||||
|
||||
# Patch base image CVEs
|
||||
RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -13,7 +16,8 @@ COPY src/ src/
|
||||
# Install the package (no dev deps)
|
||||
ENV UV_PYTHON_PREFERENCE=only-system \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=.venv
|
||||
UV_PROJECT_ENVIRONMENT=.venv \
|
||||
UV_INDEX_URL=${PYPI_INDEX_URL}
|
||||
RUN uv sync --no-dev && uv pip install -e .
|
||||
|
||||
# Config
|
||||
|
||||
28
compose.yml
28
compose.yml
@@ -600,32 +600,6 @@ services:
|
||||
- no-new-privileges:true
|
||||
restart: unless-stopped
|
||||
|
||||
# Package mirrors — local caches for reproducible builds
|
||||
apt-cache:
|
||||
image: sameersbn/apt-cacher-ng:latest
|
||||
container_name: apt-cache
|
||||
networks:
|
||||
- ci
|
||||
volumes:
|
||||
- apt_cache_data:/var/cache/apt-cacher-ng
|
||||
- ./mirrors/apt/acng.conf:/etc/apt-cacher-ng/acng.conf:ro
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
restart: unless-stopped
|
||||
|
||||
devpi:
|
||||
image: muccg/devpi:latest
|
||||
container_name: devpi
|
||||
networks:
|
||||
- ci
|
||||
volumes:
|
||||
- devpi_data:/var/lib/devpi
|
||||
environment:
|
||||
- DEVPI_PASSWORD=${DEVPI_PASSWORD:-changeme}
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
rustfs_data:
|
||||
@@ -638,5 +612,3 @@ volumes:
|
||||
loki_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
apt_cache_data:
|
||||
devpi_data:
|
||||
|
||||
@@ -16,3 +16,4 @@
|
||||
192.168.1.192 loki.homelab.fhirworx.io
|
||||
192.168.1.192 s3.homelab.fhirworx.io
|
||||
192.168.1.192 s3console.homelab.fhirworx.io
|
||||
192.168.1.192 packages.homelab.fhirworx.io
|
||||
|
||||
@@ -58,9 +58,11 @@ def _build_push_step(
|
||||
tags = [t.strip() for t in tags_expr.split(",") if t.strip()]
|
||||
local_tag = f"local/{name}:build"
|
||||
cache_flag = " --no-cache" if no_cache else ""
|
||||
# Pass Gitea PyPI registry as index URL when set
|
||||
mirror_args = ' --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}"'
|
||||
lines = f"""\
|
||||
- name: Build {name}
|
||||
run: docker build{cache_flag} -f {img["dockerfile"]} -t {local_tag} {img["context"]}"""
|
||||
run: docker build{cache_flag}{mirror_args} -f {img["dockerfile"]} -t {local_tag} {img["context"]}"""
|
||||
if not load_only and tags:
|
||||
# Save to tarball, then crane push each tag (daemonless)
|
||||
push_cmds = "\n ".join(
|
||||
@@ -439,7 +441,9 @@ jobs:
|
||||
- name: Check manifest freshness
|
||||
run: uv run python dev/scripts/pkg_inventory.py --check
|
||||
|
||||
- name: Sync mirrors
|
||||
- name: Sync to Gitea package registry
|
||||
env:
|
||||
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
|
||||
run: uv run python dev/scripts/pkg_mirror_sync.py
|
||||
|
||||
- name: Drift detection
|
||||
|
||||
@@ -1,28 +1,124 @@
|
||||
"""Synchronize local package mirrors from the package manifest.
|
||||
"""Download packages from upstream and upload to Gitea package registry.
|
||||
|
||||
Reads data/pkg-manifest.json and ensures apt-cache, apk-mirror, and
|
||||
devpi mirrors contain exactly the packages we need.
|
||||
Reads data/pkg-manifest.json, downloads each package from its upstream
|
||||
source (PyPI, Debian repos, Alpine repos), then pushes to Gitea's
|
||||
built-in package registry so all builds pull exclusively from Gitea.
|
||||
|
||||
Usage:
|
||||
uv run python dev/scripts/pkg_mirror_sync.py # sync all
|
||||
uv run python dev/scripts/pkg_mirror_sync.py --type apt # sync apt only
|
||||
uv run python dev/scripts/pkg_mirror_sync.py --type pypi # sync pypi only
|
||||
uv run python dev/scripts/pkg_mirror_sync.py --dry-run # show what would change
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
|
||||
CACHE_DIR = ROOT / "mirrors" / "cache"
|
||||
|
||||
# Mirror config directories
|
||||
APT_MIRROR_DIR = ROOT / "mirrors" / "apt"
|
||||
APK_MIRROR_DIR = ROOT / "mirrors" / "apk"
|
||||
PYPI_MIRROR_DIR = ROOT / "mirrors" / "pypi"
|
||||
# Gitea registry config — loaded from env or .env file
|
||||
GITEA_URL = ""
|
||||
GITEA_TOKEN = ""
|
||||
GITEA_OWNER = "homelab"
|
||||
|
||||
|
||||
def _load_env() -> None:
|
||||
global GITEA_URL, GITEA_TOKEN, GITEA_OWNER
|
||||
GITEA_URL = os.environ.get("GITEA_URL", "")
|
||||
GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "")
|
||||
|
||||
if not GITEA_TOKEN:
|
||||
env_file = ROOT / ".env"
|
||||
if env_file.exists():
|
||||
for line in env_file.read_text().splitlines():
|
||||
if line.startswith("GITEA_TOKEN="):
|
||||
GITEA_TOKEN = line.split("=", 1)[1].strip().strip('"').strip("'")
|
||||
|
||||
if not GITEA_URL:
|
||||
# Read from stack.toml [services]
|
||||
toml_file = ROOT / "stack.toml"
|
||||
if toml_file.exists():
|
||||
for line in toml_file.read_text().splitlines():
|
||||
if line.strip().startswith("gitea"):
|
||||
parts = line.split("=", 1)
|
||||
if len(parts) == 2:
|
||||
GITEA_URL = parts[1].strip().strip('"').strip("'")
|
||||
break
|
||||
if not GITEA_URL:
|
||||
GITEA_URL = "http://gitea:3000"
|
||||
|
||||
|
||||
def _api_via_docker(method: str, path: str, file_path: str = "") -> tuple[int, str]:
|
||||
"""Call Gitea API via docker exec (handles DNS resolution)."""
|
||||
url = f"http://localhost:3000/api/v1/{path}"
|
||||
cmd = [
|
||||
"docker", "exec", "gitea", "curl", "-s", "-w", "\n%{http_code}",
|
||||
"-H", f"Authorization: token {GITEA_TOKEN}",
|
||||
]
|
||||
if method == "PUT" and file_path:
|
||||
# For file uploads, we need to copy the file into the container first
|
||||
tmp_name = f"/tmp/pkg_upload_{os.path.basename(file_path)}"
|
||||
subprocess.run(
|
||||
["docker", "cp", file_path, f"gitea:{tmp_name}"],
|
||||
capture_output=True, timeout=60,
|
||||
)
|
||||
cmd += ["-X", "PUT", "--upload-file", tmp_name, url]
|
||||
elif method == "GET":
|
||||
cmd += [url]
|
||||
else:
|
||||
cmd += ["-X", method, url]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
if result.returncode != 0:
|
||||
return 500, result.stderr[:200]
|
||||
|
||||
lines = result.stdout.strip().rsplit("\n", 1)
|
||||
body = lines[0] if len(lines) > 1 else ""
|
||||
status = int(lines[-1]) if lines[-1].isdigit() else 500
|
||||
return status, body
|
||||
|
||||
|
||||
def _gitea_pkg_exists(pkg_type: str, name: str) -> bool:
|
||||
"""Check if a package already exists in Gitea registry."""
|
||||
path = f"packages/{GITEA_OWNER}/{pkg_type}?q={name}&limit=1"
|
||||
status, body = _api_via_docker("GET", path)
|
||||
if status == 200:
|
||||
try:
|
||||
data = json.loads(body)
|
||||
return len(data) > 0
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _parse_dist_filename(filename: str) -> tuple[str, str]:
|
||||
"""Extract package name and version from a wheel or sdist filename.
|
||||
|
||||
Examples:
|
||||
pyasn1-0.6.3-py3-none-any.whl -> (pyasn1, 0.6.3)
|
||||
sqlglot-26.0.0.tar.gz -> (sqlglot, 26.0.0)
|
||||
"""
|
||||
import re
|
||||
|
||||
# Wheel: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl
|
||||
m = re.match(r"^(.+?)-(\d+[^-]*)-", filename)
|
||||
if m:
|
||||
return m.group(1).replace("_", "-").lower(), m.group(2)
|
||||
|
||||
# Sdist: {name}-{version}.tar.gz or {name}-{version}.zip
|
||||
m = re.match(r"^(.+?)-(\d+\S+?)\.(?:tar\.gz|zip)$", filename)
|
||||
if m:
|
||||
return m.group(1).replace("_", "-").lower(), m.group(2)
|
||||
|
||||
return "", ""
|
||||
|
||||
|
||||
def load_manifest() -> dict:
|
||||
@@ -39,13 +135,6 @@ def _flat_apt_packages(manifest: dict) -> list[str]:
|
||||
return sorted(pkgs)
|
||||
|
||||
|
||||
def _flat_apk_packages(manifest: dict) -> list[str]:
|
||||
pkgs: set[str] = set()
|
||||
for pkg_list in manifest.get("apk", {}).values():
|
||||
pkgs.update(pkg_list)
|
||||
return sorted(pkgs)
|
||||
|
||||
|
||||
def _flat_pypi_packages(manifest: dict) -> list[dict]:
|
||||
seen: set[str] = set()
|
||||
pkgs: list[dict] = []
|
||||
@@ -57,168 +146,257 @@ def _flat_pypi_packages(manifest: dict) -> list[dict]:
|
||||
return sorted(pkgs, key=lambda p: p["name"])
|
||||
|
||||
|
||||
def sync_apt(manifest: dict, *, dry_run: bool = False) -> None:
|
||||
"""Generate apt-cacher-ng pre-seed package list."""
|
||||
pkgs = _flat_apt_packages(manifest)
|
||||
if not pkgs:
|
||||
print(" apt: no packages to mirror")
|
||||
return
|
||||
|
||||
APT_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
seed_file = APT_MIRROR_DIR / "package-list.txt"
|
||||
content = "\n".join(pkgs) + "\n"
|
||||
|
||||
if dry_run:
|
||||
print(f" apt: would write {len(pkgs)} packages to {seed_file}")
|
||||
for p in pkgs:
|
||||
print(f" - {p}")
|
||||
return
|
||||
|
||||
seed_file.write_text(content)
|
||||
print(f" apt: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
|
||||
|
||||
# Generate apt-cacher-ng config for pre-seeding
|
||||
config = APT_MIRROR_DIR / "acng.conf"
|
||||
config.write_text(
|
||||
"# apt-cacher-ng config — generated by pkg_mirror_sync.py\n"
|
||||
"CacheDir: /var/cache/apt-cacher-ng\n"
|
||||
"LogDir: /var/log/apt-cacher-ng\n"
|
||||
"Port: 3142\n"
|
||||
"PassThroughPattern: .*\n"
|
||||
)
|
||||
print(f" apt: wrote config to {config.relative_to(ROOT)}")
|
||||
|
||||
|
||||
def sync_apk(manifest: dict, *, dry_run: bool = False) -> None:
|
||||
"""Generate apk mirror seed list."""
|
||||
pkgs = _flat_apk_packages(manifest)
|
||||
|
||||
APK_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
seed_file = APK_MIRROR_DIR / "package-list.txt"
|
||||
|
||||
# Even if no explicit apk add packages, Alpine base images need
|
||||
# their index for `apk upgrade`
|
||||
content = "# Alpine packages to pre-cache (includes base for apk upgrade)\n"
|
||||
if pkgs:
|
||||
content += "\n".join(pkgs) + "\n"
|
||||
else:
|
||||
content += "# No explicit packages — mirror Alpine index only\n"
|
||||
|
||||
if dry_run:
|
||||
print(f" apk: would write {len(pkgs)} packages to {seed_file}")
|
||||
return
|
||||
|
||||
seed_file.write_text(content)
|
||||
print(f" apk: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
|
||||
|
||||
|
||||
def sync_pypi(manifest: dict, *, dry_run: bool = False) -> None:
|
||||
"""Generate devpi pre-seed requirements file."""
|
||||
def sync_pypi(manifest: dict, *, dry_run: bool = False) -> dict:
|
||||
"""Download Python wheels from PyPI and upload to Gitea."""
|
||||
pkgs = _flat_pypi_packages(manifest)
|
||||
if not pkgs:
|
||||
print(" pypi: no packages to mirror")
|
||||
return
|
||||
return {"uploaded": 0, "skipped": 0, "failed": []}
|
||||
|
||||
PYPI_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
seed_file = PYPI_MIRROR_DIR / "requirements.txt"
|
||||
stats = {"uploaded": 0, "skipped": 0, "failed": []}
|
||||
|
||||
lines = [
|
||||
"# PyPI packages to mirror — generated by pkg_mirror_sync.py",
|
||||
"# Feed to: devpi-server --mirror, or pip download -r",
|
||||
]
|
||||
for pkg in pkgs:
|
||||
spec = pkg["name"]
|
||||
if pkg.get("extras"):
|
||||
spec += f"[{pkg['extras']}]"
|
||||
if pkg.get("version"):
|
||||
spec += pkg["version"]
|
||||
lines.append(spec)
|
||||
with tempfile.TemporaryDirectory(prefix="pkg_pypi_") as tmpdir:
|
||||
# Build requirements spec
|
||||
specs = []
|
||||
for pkg in pkgs:
|
||||
spec = pkg["name"]
|
||||
if pkg.get("extras"):
|
||||
spec += f"[{pkg['extras']}]"
|
||||
if pkg.get("version"):
|
||||
spec += pkg["version"]
|
||||
specs.append(spec)
|
||||
|
||||
content = "\n".join(lines) + "\n"
|
||||
req_file = Path(tmpdir) / "requirements.txt"
|
||||
req_file.write_text("\n".join(specs) + "\n")
|
||||
|
||||
if dry_run:
|
||||
print(f" pypi: would write {len(pkgs)} packages to {seed_file}")
|
||||
for p in pkgs:
|
||||
print(f" - {p['name']}{p.get('version', '')}")
|
||||
return
|
||||
if dry_run:
|
||||
print(f" pypi: would download and upload {len(specs)} packages:")
|
||||
for s in specs:
|
||||
print(f" - {s}")
|
||||
return stats
|
||||
|
||||
seed_file.write_text(content)
|
||||
print(f" pypi: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
|
||||
|
||||
# Generate devpi config
|
||||
config = PYPI_MIRROR_DIR / "devpi.conf"
|
||||
config.write_text(
|
||||
"# devpi config — generated by pkg_mirror_sync.py\n"
|
||||
"[devpi-server]\n"
|
||||
"serverdir = /var/lib/devpi\n"
|
||||
"port = 3141\n"
|
||||
"host = 0.0.0.0\n"
|
||||
"\n"
|
||||
"[mirror]\n"
|
||||
"type = mirror\n"
|
||||
"url = https://pypi.org/simple/\n"
|
||||
)
|
||||
print(f" pypi: wrote config to {config.relative_to(ROOT)}")
|
||||
|
||||
|
||||
def warm_devpi(manifest: dict, *, dry_run: bool = False) -> None:
|
||||
"""Warm devpi cache by downloading packages listed in the manifest.
|
||||
|
||||
Requires devpi to be running. Skips if devpi-server is unreachable.
|
||||
"""
|
||||
seed_file = PYPI_MIRROR_DIR / "requirements.txt"
|
||||
if not seed_file.exists():
|
||||
return
|
||||
|
||||
if dry_run:
|
||||
print(" pypi: would warm devpi cache from requirements.txt")
|
||||
return
|
||||
|
||||
try:
|
||||
# Download wheels/sdists from upstream PyPI
|
||||
print(f" pypi: downloading {len(specs)} packages from PyPI...")
|
||||
dl_dir = Path(tmpdir) / "downloads"
|
||||
dl_dir.mkdir()
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable, "-m", "pip", "download",
|
||||
"--no-deps", "--dest", str(PYPI_MIRROR_DIR / "cache"),
|
||||
"-r", str(seed_file),
|
||||
"uvx", "pip", "download",
|
||||
"--no-deps", "--dest", str(dl_dir),
|
||||
"-r", str(req_file),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
capture_output=True, text=True, timeout=600,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(" pypi: warmed cache successfully")
|
||||
if result.returncode != 0:
|
||||
print(f" pypi: download failed: {result.stderr[:500]}")
|
||||
stats["failed"].append("pip-download")
|
||||
return stats
|
||||
|
||||
# Upload to Gitea PyPI registry (requires name, version, sha256_digest)
|
||||
dist_files = list(dl_dir.iterdir())
|
||||
print(f" pypi: uploading {len(dist_files)} files to Gitea...")
|
||||
upload_url = f"http://localhost:3000/api/packages/{GITEA_OWNER}/pypi"
|
||||
for dist in sorted(dist_files):
|
||||
# Parse name and version from filename
|
||||
pkg_name, pkg_version = _parse_dist_filename(dist.name)
|
||||
if not pkg_name:
|
||||
print(f" SKIP: {dist.name} (can't parse name/version)")
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
# Compute SHA-256
|
||||
import hashlib
|
||||
|
||||
sha256 = hashlib.sha256(dist.read_bytes()).hexdigest()
|
||||
|
||||
# Copy file into gitea container and upload
|
||||
tmp_name = f"/tmp/pkg_{dist.name}"
|
||||
subprocess.run(
|
||||
["docker", "cp", str(dist), f"gitea:{tmp_name}"],
|
||||
capture_output=True, timeout=60,
|
||||
)
|
||||
result_up = subprocess.run(
|
||||
[
|
||||
"docker", "exec", "gitea", "curl", "-s",
|
||||
"-w", "\n%{http_code}",
|
||||
"-H", f"Authorization: token {GITEA_TOKEN}",
|
||||
"-F", f"content=@{tmp_name}",
|
||||
"-F", f"name={pkg_name}",
|
||||
"-F", f"version={pkg_version}",
|
||||
"-F", f"sha256_digest={sha256}",
|
||||
upload_url,
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
lines = result_up.stdout.strip().rsplit("\n", 1)
|
||||
status = int(lines[-1]) if lines[-1].isdigit() else 500
|
||||
if status in (201, 409):
|
||||
label = "uploaded" if status == 201 else "exists"
|
||||
print(f" {label}: {dist.name}")
|
||||
if status == 201:
|
||||
stats["uploaded"] += 1
|
||||
else:
|
||||
stats["skipped"] += 1
|
||||
else:
|
||||
body = lines[0] if len(lines) > 1 else ""
|
||||
print(f" FAILED ({status}): {dist.name} — {body[:120]}")
|
||||
stats["failed"].append(dist.name)
|
||||
# Cleanup
|
||||
subprocess.run(
|
||||
["docker", "exec", "gitea", "rm", "-f", tmp_name],
|
||||
capture_output=True, timeout=10,
|
||||
)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
|
||||
"""Download .deb packages and upload to Gitea Debian registry."""
|
||||
pkgs = _flat_apt_packages(manifest)
|
||||
if not pkgs:
|
||||
print(" apt: no packages to mirror")
|
||||
return {"uploaded": 0, "skipped": 0, "failed": []}
|
||||
|
||||
stats = {"uploaded": 0, "skipped": 0, "failed": []}
|
||||
|
||||
if dry_run:
|
||||
print(f" apt: would download and upload {len(pkgs)} packages:")
|
||||
for p in pkgs:
|
||||
print(f" - {p}")
|
||||
return stats
|
||||
|
||||
# Use a subdirectory under mirrors/ for apt downloads (avoids tmpdir
|
||||
# permission issues with Docker volume mounts)
|
||||
dl_dir = ROOT / "mirrors" / "cache" / "apt"
|
||||
dl_dir.mkdir(parents=True, exist_ok=True)
|
||||
# Clean previous downloads
|
||||
for old in dl_dir.glob("*.deb"):
|
||||
old.unlink()
|
||||
|
||||
# Download .debs via Docker — try each package individually to handle
|
||||
# third-party packages (e.g. zotero) that aren't in base Debian repos
|
||||
print(f" apt: downloading {len(pkgs)} packages...")
|
||||
# Build a script that tries each package, skipping failures
|
||||
install_cmds = " && ".join(
|
||||
f"(apt-get install --reinstall --download-only -y {p} 2>/dev/null || "
|
||||
f"echo 'SKIP: {p} (not in base repos)')"
|
||||
for p in pkgs
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"-v", f"{dl_dir}:/debs",
|
||||
"debian:bookworm-slim",
|
||||
"bash", "-c",
|
||||
f"apt-get update -qq 2>/dev/null && {install_cmds}; "
|
||||
f"cp /var/cache/apt/archives/*.deb /debs/ 2>/dev/null || true; "
|
||||
f"chmod 644 /debs/*.deb 2>/dev/null || true",
|
||||
],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
)
|
||||
|
||||
deb_files = list(dl_dir.glob("*.deb"))
|
||||
if not deb_files:
|
||||
print(" apt: no .deb files downloaded")
|
||||
stats["skipped"] = len(pkgs)
|
||||
return stats
|
||||
|
||||
print(f" apt: uploading {len(deb_files)} .deb files to Gitea...")
|
||||
for deb in sorted(deb_files):
|
||||
tmp_name = f"/tmp/pkg_{deb.name}"
|
||||
subprocess.run(
|
||||
["docker", "cp", str(deb), f"gitea:{tmp_name}"],
|
||||
capture_output=True, timeout=60,
|
||||
)
|
||||
result_up = subprocess.run(
|
||||
[
|
||||
"docker", "exec", "gitea", "curl", "-s",
|
||||
"-w", "\n%{http_code}",
|
||||
"-H", f"Authorization: token {GITEA_TOKEN}",
|
||||
"--upload-file", tmp_name,
|
||||
f"http://localhost:3000/api/packages/{GITEA_OWNER}"
|
||||
f"/debian/pool/bookworm/main/upload",
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
lines = result_up.stdout.strip().rsplit("\n", 1)
|
||||
status = int(lines[-1]) if lines[-1].isdigit() else 500
|
||||
if status in (201, 409):
|
||||
label = "uploaded" if status == 201 else "exists"
|
||||
print(f" {label}: {deb.name}")
|
||||
if status == 201:
|
||||
stats["uploaded"] += 1
|
||||
else:
|
||||
stats["skipped"] += 1
|
||||
else:
|
||||
print(f" pypi: cache warm failed (non-fatal): {result.stderr[:200]}")
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired) as e:
|
||||
print(f" pypi: cache warm skipped: {e}")
|
||||
body = lines[0] if len(lines) > 1 else ""
|
||||
print(f" FAILED ({status}): {deb.name} — {body[:120]}")
|
||||
stats["failed"].append(deb.name)
|
||||
subprocess.run(
|
||||
["docker", "exec", "gitea", "rm", "-f", tmp_name],
|
||||
capture_output=True, timeout=10,
|
||||
)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
|
||||
"""Download .apk packages and upload to Gitea Alpine registry.
|
||||
|
||||
Currently we have no explicit apk add packages (only apk upgrade),
|
||||
so this is primarily for future use when Alpine packages are added.
|
||||
"""
|
||||
# We don't have explicit apk packages right now — just apk upgrade
|
||||
# which updates base packages. We'd need to mirror the full Alpine
|
||||
# repo to support that, which isn't practical. Instead, we'll handle
|
||||
# this by pinning the nginx:alpine image version.
|
||||
if dry_run:
|
||||
print(" apk: no explicit packages to mirror (apk upgrade uses base repo)")
|
||||
else:
|
||||
print(" apk: skipped — no explicit apk packages in manifest")
|
||||
return {"uploaded": 0, "skipped": 0, "failed": []}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Sync package mirrors from manifest")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Download packages from upstream and upload to Gitea registry"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--type",
|
||||
choices=["apt", "apk", "pypi", "all"],
|
||||
default="all",
|
||||
help="Which mirror type to sync",
|
||||
help="Which package type to sync",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show what would change")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest = load_manifest()
|
||||
print("Syncing mirrors from pkg-manifest.json...")
|
||||
_load_env()
|
||||
if not GITEA_TOKEN:
|
||||
print("ERROR: GITEA_TOKEN not set. Set it in .env or environment.")
|
||||
raise SystemExit(2)
|
||||
|
||||
if args.type in ("apt", "all"):
|
||||
sync_apt(manifest, dry_run=args.dry_run)
|
||||
if args.type in ("apk", "all"):
|
||||
sync_apk(manifest, dry_run=args.dry_run)
|
||||
manifest = load_manifest()
|
||||
print("Syncing packages to Gitea registry...")
|
||||
|
||||
results: dict[str, dict] = {}
|
||||
if args.type in ("pypi", "all"):
|
||||
sync_pypi(manifest, dry_run=args.dry_run)
|
||||
results["pypi"] = sync_pypi(manifest, dry_run=args.dry_run)
|
||||
if args.type in ("apt", "all"):
|
||||
results["apt"] = sync_apt(manifest, dry_run=args.dry_run)
|
||||
if args.type in ("apk", "all"):
|
||||
results["apk"] = sync_apk(manifest, dry_run=args.dry_run)
|
||||
|
||||
if not args.dry_run:
|
||||
print("\nMirror seed files generated. Start mirrors with:")
|
||||
print(" docker compose up -d apt-cache devpi")
|
||||
print("\n=== Summary ===")
|
||||
total_up = sum(r.get("uploaded", 0) for r in results.values())
|
||||
total_skip = sum(r.get("skipped", 0) for r in results.values())
|
||||
total_fail = sum(len(r.get("failed", [])) for r in results.values())
|
||||
print(f" Uploaded: {total_up} Skipped: {total_skip} Failed: {total_fail}")
|
||||
|
||||
if total_fail:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# Alpine packages to pre-cache (includes base for apk upgrade)
|
||||
# No explicit packages — mirror Alpine index only
|
||||
@@ -1,5 +0,0 @@
|
||||
# apt-cacher-ng config — generated by pkg_mirror_sync.py
|
||||
CacheDir: /var/cache/apt-cacher-ng
|
||||
LogDir: /var/log/apt-cacher-ng
|
||||
Port: 3142
|
||||
PassThroughPattern: .*
|
||||
@@ -1,5 +0,0 @@
|
||||
build-essential
|
||||
ca-certificates
|
||||
curl
|
||||
git
|
||||
zotero
|
||||
@@ -1,9 +0,0 @@
|
||||
# devpi config — generated by pkg_mirror_sync.py
|
||||
[devpi-server]
|
||||
serverdir = /var/lib/devpi
|
||||
port = 3141
|
||||
host = 0.0.0.0
|
||||
|
||||
[mirror]
|
||||
type = mirror
|
||||
url = https://pypi.org/simple/
|
||||
@@ -1,41 +0,0 @@
|
||||
# PyPI packages to mirror — generated by pkg_mirror_sync.py
|
||||
# Feed to: devpi-server --mirror, or pip download -r
|
||||
altair>=6.0.0
|
||||
coverage>=7.13.4
|
||||
cryptography>=46.0.5
|
||||
cudf-polars-cu12
|
||||
databricks-cli>=0.18.0
|
||||
databricks-sdk>=0.85.0
|
||||
dbt-core==1.10.15
|
||||
dbt-duckdb>=1.10,<1.11
|
||||
duckdb>=1.0.0
|
||||
fastapi>=0.135.1
|
||||
fastexcel>=0.19.0
|
||||
fsspec>=2024.1.0
|
||||
griffe
|
||||
httpx>=0.28.1
|
||||
marimo>=0.20.0
|
||||
narwhals>=2.17.0
|
||||
numpy
|
||||
obstore>=0.9.2
|
||||
openpyxl>=3.1.5
|
||||
pandas>=3.0.1
|
||||
pdfplumber>=0.11.9
|
||||
pillow>=12.1.1
|
||||
polars>=1.38.1
|
||||
pyarrow>=23.0.0
|
||||
pyasn1>=0.6.3
|
||||
pydantic
|
||||
pyiceberg[s3,pyarrow]>=0.7.0
|
||||
pyjwt>=2.12.0
|
||||
pytest>=9.0.2
|
||||
pytest-cov>=7.0.0
|
||||
pyzotero
|
||||
ruff>=0.11.0
|
||||
s3fs>=2026.2.0
|
||||
sqlglot>=26.0.0
|
||||
trino>=0.328.0
|
||||
typer>=0.24.1
|
||||
uv-build>=0.7,<1
|
||||
uvicorn>=0.41.0
|
||||
vega-datasets
|
||||
@@ -6,6 +6,9 @@ ARG USER_UID=1000
|
||||
ARG USER_GID=1000
|
||||
ARG PYTHON_VERSION=3.13
|
||||
|
||||
# Local package registry (Gitea) — set via --build-arg to pull from mirror
|
||||
ARG PYPI_INDEX_URL=""
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
HOME=/home/kert \
|
||||
PATH="/home/kert/.local/bin:${PATH}" \
|
||||
@@ -34,6 +37,7 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uvx /usr/local/bin/uvx
|
||||
WORKDIR /home/${USERNAME}
|
||||
|
||||
# Initialize uv project and install dependencies
|
||||
ENV UV_INDEX_URL=${PYPI_INDEX_URL}
|
||||
RUN uv python install ${PYTHON_VERSION} \
|
||||
&& uv init workspace --python ${PYTHON_VERSION} \
|
||||
&& cd workspace \
|
||||
|
||||
@@ -24,7 +24,7 @@ subdomains = [
|
||||
"dashboard", "docs", "gitea", "ci", "notebooks", "zotero",
|
||||
"webdav", "api", "nessie", "trino", "polaris",
|
||||
"grafana", "prometheus", "jaeger", "loki",
|
||||
"s3", "s3console",
|
||||
"s3", "s3console", "packages",
|
||||
]
|
||||
|
||||
[services]
|
||||
@@ -114,6 +114,13 @@ port = 8000
|
||||
secret = "" # override via STACK_API_SECRET env var
|
||||
workers = 1
|
||||
|
||||
[mirrors]
|
||||
# Gitea package registry — single source for all mirrored packages
|
||||
owner = "homelab"
|
||||
pypi_index = "http://gitea:3000/api/packages/homelab/pypi/simple/"
|
||||
debian_source = "http://gitea:3000/api/packages/homelab/debian"
|
||||
alpine_repo = "http://gitea:3000/api/packages/homelab/alpine"
|
||||
|
||||
[ci]
|
||||
backend = "gitea"
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
{{- $multi := dict
|
||||
"rustfs-api" (dict "container" "rustfs" "port" "9000" "subdomain" "s3" "theme" false "mw" "local-only,infra-headers")
|
||||
"rustfs-console" (dict "container" "rustfs" "port" "9001" "subdomain" "s3console" "theme" true "mw" "local-only,infra-headers")
|
||||
"gitea-packages" (dict "container" "gitea" "port" "3000" "subdomain" "packages" "theme" false "mw" "local-only,secure-headers")
|
||||
-}}
|
||||
http:
|
||||
middlewares:
|
||||
|
||||
Reference in New Issue
Block a user