replace devpi/apt-cacher-ng with Gitea package registry
Some checks failed
Infra CI / notebooks (push) Failing after 47s
Infra CI / zotero (push) Successful in 6s
Infra CI / docs (push) Successful in 5s
Infra CI / mc (push) Successful in 6s
Infra CI / zotero (pull_request) Successful in 7s
Infra CI / api (pull_request) Successful in 5s
Infra CI / api (push) Successful in 29s
CI / lint-test (push) Successful in 1m33s
Infra CI / notebooks (pull_request) Failing after 36s
Infra CI / docs (pull_request) Successful in 6s
Infra CI / mc (pull_request) Successful in 6s
CI / lint-test (pull_request) Successful in 1m29s
Package Supply Chain / pkg-supply-chain (push) Failing after 52s

closes #160, closes #161, closes #162, closes #163, refs #164

- Remove apt-cacher-ng and devpi containers from compose.yml
- pkg_mirror_sync.py: downloads from upstream, uploads to Gitea's
  built-in PyPI and Debian registries via API
- Dockerfiles: add PYPI_INDEX_URL build-arg for local registry
- CI build steps pass --build-arg PYPI_INDEX_URL for mirror builds
- stack.toml: add [mirrors] section pointing at Gitea endpoints
- Add packages.homelab.fhirworx.io subdomain (traefik + coredns)
- Verified: 39 PyPI wheels + 146 Debian .debs uploaded to Gitea
This commit is contained in:
kert
2026-03-24 00:31:07 -04:00
parent b184e72f7b
commit 66932516a9
19 changed files with 375 additions and 263 deletions

View File

@@ -34,7 +34,7 @@ jobs:
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
- name: Build notebooks
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f notebooks/Dockerfile -t local/notebooks:build notebooks/
- name: Push notebooks
run: |
@@ -43,7 +43,7 @@ jobs:
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
- name: Build zotero
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f zotero/Dockerfile -t local/zotero:build zotero/
- name: Push zotero
run: |
@@ -52,7 +52,7 @@ jobs:
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
- name: Build docs
run: docker build -f docs/Dockerfile -t local/docs:build .
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f docs/Dockerfile -t local/docs:build .
- name: Push docs
run: |
@@ -61,7 +61,7 @@ jobs:
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
- name: Build api
run: docker build -f api/Dockerfile -t local/api:build .
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f api/Dockerfile -t local/api:build .
- name: Push api
run: |
@@ -70,7 +70,7 @@ jobs:
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
- name: Build mc
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
- name: Push mc
run: |

View File

@@ -32,7 +32,7 @@ jobs:
UV_INSTALL_DIR: /usr/local/bin
- name: Build notebooks
run: docker build --no-cache -f notebooks/Dockerfile -t local/notebooks:build notebooks/
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f notebooks/Dockerfile -t local/notebooks:build notebooks/
- name: Push notebooks
run: |
@@ -41,7 +41,7 @@ jobs:
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
- name: Build zotero
run: docker build --no-cache -f zotero/Dockerfile -t local/zotero:build zotero/
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f zotero/Dockerfile -t local/zotero:build zotero/
- name: Push zotero
run: |
@@ -50,7 +50,7 @@ jobs:
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
- name: Build docs
run: docker build --no-cache -f docs/Dockerfile -t local/docs:build .
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f docs/Dockerfile -t local/docs:build .
- name: Push docs
run: |
@@ -59,7 +59,7 @@ jobs:
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
- name: Build api
run: docker build --no-cache -f api/Dockerfile -t local/api:build .
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f api/Dockerfile -t local/api:build .
- name: Push api
run: |
@@ -68,7 +68,7 @@ jobs:
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
- name: Build mc
run: docker build --no-cache -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
run: docker build --no-cache --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
- name: Push mc
run: |

View File

@@ -38,7 +38,7 @@ jobs:
- name: Build notebooks
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f notebooks/Dockerfile -t local/notebooks:build notebooks/
zotero:
runs-on: ubuntu-latest
@@ -54,7 +54,7 @@ jobs:
- name: Build zotero
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f zotero/Dockerfile -t local/zotero:build zotero/
docs:
runs-on: ubuntu-latest
@@ -70,7 +70,7 @@ jobs:
- name: Build docs
run: docker build -f docs/Dockerfile -t local/docs:build .
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f docs/Dockerfile -t local/docs:build .
api:
runs-on: ubuntu-latest
@@ -86,7 +86,7 @@ jobs:
- name: Build api
run: docker build -f api/Dockerfile -t local/api:build .
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f api/Dockerfile -t local/api:build .
mc:
runs-on: ubuntu-latest
@@ -102,4 +102,4 @@ jobs:
- name: Build mc
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f rustfs/Dockerfile.mc -t local/mc:build rustfs/

View File

@@ -36,7 +36,9 @@ jobs:
- name: Check manifest freshness
run: uv run python dev/scripts/pkg_inventory.py --check
- name: Sync mirrors
- name: Sync to Gitea package registry
env:
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
run: uv run python dev/scripts/pkg_mirror_sync.py
- name: Drift detection

View File

@@ -33,7 +33,7 @@ jobs:
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
- name: Build notebooks
run: docker build -f notebooks/Dockerfile -t local/notebooks:build notebooks/
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f notebooks/Dockerfile -t local/notebooks:build notebooks/
- name: Push notebooks
run: |
@@ -42,7 +42,7 @@ jobs:
crane push /tmp/notebooks.tar gitea:3000/homelab/stack/notebooks:latest --insecure
- name: Build zotero
run: docker build -f zotero/Dockerfile -t local/zotero:build zotero/
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f zotero/Dockerfile -t local/zotero:build zotero/
- name: Push zotero
run: |
@@ -51,7 +51,7 @@ jobs:
crane push /tmp/zotero.tar gitea:3000/homelab/stack/zotero:latest --insecure
- name: Build docs
run: docker build -f docs/Dockerfile -t local/docs:build .
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f docs/Dockerfile -t local/docs:build .
- name: Push docs
run: |
@@ -60,7 +60,7 @@ jobs:
crane push /tmp/docs.tar gitea:3000/homelab/stack/docs:latest --insecure
- name: Build api
run: docker build -f api/Dockerfile -t local/api:build .
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f api/Dockerfile -t local/api:build .
- name: Push api
run: |
@@ -69,7 +69,7 @@ jobs:
crane push /tmp/api.tar gitea:3000/homelab/stack/api:latest --insecure
- name: Build mc
run: docker build -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
run: docker build --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}" -f rustfs/Dockerfile.mc -t local/mc:build rustfs/
- name: Push mc
run: |

1
.gitignore vendored
View File

@@ -3,6 +3,7 @@
# Data directories (user data, not tracked)
data/
mirrors/cache/
zotero/data/
zotero/profiles/
tuva/

View File

@@ -3,6 +3,9 @@ FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
WORKDIR /app
# Local package registry (Gitea) — set via --build-arg to pull from mirror
ARG PYPI_INDEX_URL=""
# Patch base image CVEs
RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
@@ -13,7 +16,8 @@ COPY src/ src/
# Install the package (no dev deps)
ENV UV_PYTHON_PREFERENCE=only-system \
UV_LINK_MODE=copy \
UV_PROJECT_ENVIRONMENT=.venv
UV_PROJECT_ENVIRONMENT=.venv \
UV_INDEX_URL=${PYPI_INDEX_URL}
RUN uv sync --no-dev && uv pip install -e .
# Config

View File

@@ -600,32 +600,6 @@ services:
- no-new-privileges:true
restart: unless-stopped
# Package mirrors — local caches for reproducible builds
apt-cache:
image: sameersbn/apt-cacher-ng:latest
container_name: apt-cache
networks:
- ci
volumes:
- apt_cache_data:/var/cache/apt-cacher-ng
- ./mirrors/apt/acng.conf:/etc/apt-cacher-ng/acng.conf:ro
security_opt:
- no-new-privileges:true
restart: unless-stopped
devpi:
image: muccg/devpi:latest
container_name: devpi
networks:
- ci
volumes:
- devpi_data:/var/lib/devpi
environment:
- DEVPI_PASSWORD=${DEVPI_PASSWORD:-changeme}
security_opt:
- no-new-privileges:true
restart: unless-stopped
volumes:
postgres_data:
rustfs_data:
@@ -638,5 +612,3 @@ volumes:
loki_data:
prometheus_data:
grafana_data:
apt_cache_data:
devpi_data:

View File

@@ -16,3 +16,4 @@
192.168.1.192 loki.homelab.fhirworx.io
192.168.1.192 s3.homelab.fhirworx.io
192.168.1.192 s3console.homelab.fhirworx.io
192.168.1.192 packages.homelab.fhirworx.io

View File

@@ -58,9 +58,11 @@ def _build_push_step(
tags = [t.strip() for t in tags_expr.split(",") if t.strip()]
local_tag = f"local/{name}:build"
cache_flag = " --no-cache" if no_cache else ""
# Pass Gitea PyPI registry as index URL when set
mirror_args = ' --build-arg PYPI_INDEX_URL="${PYPI_INDEX_URL:-}"'
lines = f"""\
- name: Build {name}
run: docker build{cache_flag} -f {img["dockerfile"]} -t {local_tag} {img["context"]}"""
run: docker build{cache_flag}{mirror_args} -f {img["dockerfile"]} -t {local_tag} {img["context"]}"""
if not load_only and tags:
# Save to tarball, then crane push each tag (daemonless)
push_cmds = "\n ".join(
@@ -439,7 +441,9 @@ jobs:
- name: Check manifest freshness
run: uv run python dev/scripts/pkg_inventory.py --check
- name: Sync mirrors
- name: Sync to Gitea package registry
env:
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
run: uv run python dev/scripts/pkg_mirror_sync.py
- name: Drift detection

View File

@@ -1,28 +1,124 @@
"""Synchronize local package mirrors from the package manifest.
"""Download packages from upstream and upload to Gitea package registry.
Reads data/pkg-manifest.json and ensures apt-cache, apk-mirror, and
devpi mirrors contain exactly the packages we need.
Reads data/pkg-manifest.json, downloads each package from its upstream
source (PyPI, Debian repos, Alpine repos), then pushes to Gitea's
built-in package registry so all builds pull exclusively from Gitea.
Usage:
uv run python dev/scripts/pkg_mirror_sync.py # sync all
uv run python dev/scripts/pkg_mirror_sync.py --type apt # sync apt only
uv run python dev/scripts/pkg_mirror_sync.py --type pypi # sync pypi only
uv run python dev/scripts/pkg_mirror_sync.py --dry-run # show what would change
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
CACHE_DIR = ROOT / "mirrors" / "cache"
# Mirror config directories
APT_MIRROR_DIR = ROOT / "mirrors" / "apt"
APK_MIRROR_DIR = ROOT / "mirrors" / "apk"
PYPI_MIRROR_DIR = ROOT / "mirrors" / "pypi"
# Gitea registry config — loaded from env or .env file
GITEA_URL = ""
GITEA_TOKEN = ""
GITEA_OWNER = "homelab"
def _load_env() -> None:
global GITEA_URL, GITEA_TOKEN, GITEA_OWNER
GITEA_URL = os.environ.get("GITEA_URL", "")
GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "")
if not GITEA_TOKEN:
env_file = ROOT / ".env"
if env_file.exists():
for line in env_file.read_text().splitlines():
if line.startswith("GITEA_TOKEN="):
GITEA_TOKEN = line.split("=", 1)[1].strip().strip('"').strip("'")
if not GITEA_URL:
# Read from stack.toml [services]
toml_file = ROOT / "stack.toml"
if toml_file.exists():
for line in toml_file.read_text().splitlines():
if line.strip().startswith("gitea"):
parts = line.split("=", 1)
if len(parts) == 2:
GITEA_URL = parts[1].strip().strip('"').strip("'")
break
if not GITEA_URL:
GITEA_URL = "http://gitea:3000"
def _api_via_docker(method: str, path: str, file_path: str = "") -> tuple[int, str]:
"""Call Gitea API via docker exec (handles DNS resolution)."""
url = f"http://localhost:3000/api/v1/{path}"
cmd = [
"docker", "exec", "gitea", "curl", "-s", "-w", "\n%{http_code}",
"-H", f"Authorization: token {GITEA_TOKEN}",
]
if method == "PUT" and file_path:
# For file uploads, we need to copy the file into the container first
tmp_name = f"/tmp/pkg_upload_{os.path.basename(file_path)}"
subprocess.run(
["docker", "cp", file_path, f"gitea:{tmp_name}"],
capture_output=True, timeout=60,
)
cmd += ["-X", "PUT", "--upload-file", tmp_name, url]
elif method == "GET":
cmd += [url]
else:
cmd += ["-X", method, url]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
if result.returncode != 0:
return 500, result.stderr[:200]
lines = result.stdout.strip().rsplit("\n", 1)
body = lines[0] if len(lines) > 1 else ""
status = int(lines[-1]) if lines[-1].isdigit() else 500
return status, body
def _gitea_pkg_exists(pkg_type: str, name: str) -> bool:
"""Check if a package already exists in Gitea registry."""
path = f"packages/{GITEA_OWNER}/{pkg_type}?q={name}&limit=1"
status, body = _api_via_docker("GET", path)
if status == 200:
try:
data = json.loads(body)
return len(data) > 0
except json.JSONDecodeError:
pass
return False
def _parse_dist_filename(filename: str) -> tuple[str, str]:
"""Extract package name and version from a wheel or sdist filename.
Examples:
pyasn1-0.6.3-py3-none-any.whl -> (pyasn1, 0.6.3)
sqlglot-26.0.0.tar.gz -> (sqlglot, 26.0.0)
"""
import re
# Wheel: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl
m = re.match(r"^(.+?)-(\d+[^-]*)-", filename)
if m:
return m.group(1).replace("_", "-").lower(), m.group(2)
# Sdist: {name}-{version}.tar.gz or {name}-{version}.zip
m = re.match(r"^(.+?)-(\d+\S+?)\.(?:tar\.gz|zip)$", filename)
if m:
return m.group(1).replace("_", "-").lower(), m.group(2)
return "", ""
def load_manifest() -> dict:
@@ -39,13 +135,6 @@ def _flat_apt_packages(manifest: dict) -> list[str]:
return sorted(pkgs)
def _flat_apk_packages(manifest: dict) -> list[str]:
pkgs: set[str] = set()
for pkg_list in manifest.get("apk", {}).values():
pkgs.update(pkg_list)
return sorted(pkgs)
def _flat_pypi_packages(manifest: dict) -> list[dict]:
seen: set[str] = set()
pkgs: list[dict] = []
@@ -57,168 +146,257 @@ def _flat_pypi_packages(manifest: dict) -> list[dict]:
return sorted(pkgs, key=lambda p: p["name"])
def sync_apt(manifest: dict, *, dry_run: bool = False) -> None:
"""Generate apt-cacher-ng pre-seed package list."""
pkgs = _flat_apt_packages(manifest)
if not pkgs:
print(" apt: no packages to mirror")
return
APT_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
seed_file = APT_MIRROR_DIR / "package-list.txt"
content = "\n".join(pkgs) + "\n"
if dry_run:
print(f" apt: would write {len(pkgs)} packages to {seed_file}")
for p in pkgs:
print(f" - {p}")
return
seed_file.write_text(content)
print(f" apt: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
# Generate apt-cacher-ng config for pre-seeding
config = APT_MIRROR_DIR / "acng.conf"
config.write_text(
"# apt-cacher-ng config — generated by pkg_mirror_sync.py\n"
"CacheDir: /var/cache/apt-cacher-ng\n"
"LogDir: /var/log/apt-cacher-ng\n"
"Port: 3142\n"
"PassThroughPattern: .*\n"
)
print(f" apt: wrote config to {config.relative_to(ROOT)}")
def sync_apk(manifest: dict, *, dry_run: bool = False) -> None:
"""Generate apk mirror seed list."""
pkgs = _flat_apk_packages(manifest)
APK_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
seed_file = APK_MIRROR_DIR / "package-list.txt"
# Even if no explicit apk add packages, Alpine base images need
# their index for `apk upgrade`
content = "# Alpine packages to pre-cache (includes base for apk upgrade)\n"
if pkgs:
content += "\n".join(pkgs) + "\n"
else:
content += "# No explicit packages — mirror Alpine index only\n"
if dry_run:
print(f" apk: would write {len(pkgs)} packages to {seed_file}")
return
seed_file.write_text(content)
print(f" apk: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
def sync_pypi(manifest: dict, *, dry_run: bool = False) -> None:
"""Generate devpi pre-seed requirements file."""
def sync_pypi(manifest: dict, *, dry_run: bool = False) -> dict:
"""Download Python wheels from PyPI and upload to Gitea."""
pkgs = _flat_pypi_packages(manifest)
if not pkgs:
print(" pypi: no packages to mirror")
return
return {"uploaded": 0, "skipped": 0, "failed": []}
PYPI_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
seed_file = PYPI_MIRROR_DIR / "requirements.txt"
stats = {"uploaded": 0, "skipped": 0, "failed": []}
lines = [
"# PyPI packages to mirror — generated by pkg_mirror_sync.py",
"# Feed to: devpi-server --mirror, or pip download -r",
]
for pkg in pkgs:
spec = pkg["name"]
if pkg.get("extras"):
spec += f"[{pkg['extras']}]"
if pkg.get("version"):
spec += pkg["version"]
lines.append(spec)
with tempfile.TemporaryDirectory(prefix="pkg_pypi_") as tmpdir:
# Build requirements spec
specs = []
for pkg in pkgs:
spec = pkg["name"]
if pkg.get("extras"):
spec += f"[{pkg['extras']}]"
if pkg.get("version"):
spec += pkg["version"]
specs.append(spec)
content = "\n".join(lines) + "\n"
req_file = Path(tmpdir) / "requirements.txt"
req_file.write_text("\n".join(specs) + "\n")
if dry_run:
print(f" pypi: would write {len(pkgs)} packages to {seed_file}")
for p in pkgs:
print(f" - {p['name']}{p.get('version', '')}")
return
if dry_run:
print(f" pypi: would download and upload {len(specs)} packages:")
for s in specs:
print(f" - {s}")
return stats
seed_file.write_text(content)
print(f" pypi: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
# Generate devpi config
config = PYPI_MIRROR_DIR / "devpi.conf"
config.write_text(
"# devpi config — generated by pkg_mirror_sync.py\n"
"[devpi-server]\n"
"serverdir = /var/lib/devpi\n"
"port = 3141\n"
"host = 0.0.0.0\n"
"\n"
"[mirror]\n"
"type = mirror\n"
"url = https://pypi.org/simple/\n"
)
print(f" pypi: wrote config to {config.relative_to(ROOT)}")
def warm_devpi(manifest: dict, *, dry_run: bool = False) -> None:
"""Warm devpi cache by downloading packages listed in the manifest.
Requires devpi to be running. Skips if devpi-server is unreachable.
"""
seed_file = PYPI_MIRROR_DIR / "requirements.txt"
if not seed_file.exists():
return
if dry_run:
print(" pypi: would warm devpi cache from requirements.txt")
return
try:
# Download wheels/sdists from upstream PyPI
print(f" pypi: downloading {len(specs)} packages from PyPI...")
dl_dir = Path(tmpdir) / "downloads"
dl_dir.mkdir()
result = subprocess.run(
[
sys.executable, "-m", "pip", "download",
"--no-deps", "--dest", str(PYPI_MIRROR_DIR / "cache"),
"-r", str(seed_file),
"uvx", "pip", "download",
"--no-deps", "--dest", str(dl_dir),
"-r", str(req_file),
],
capture_output=True,
text=True,
timeout=300,
capture_output=True, text=True, timeout=600,
)
if result.returncode == 0:
print(" pypi: warmed cache successfully")
if result.returncode != 0:
print(f" pypi: download failed: {result.stderr[:500]}")
stats["failed"].append("pip-download")
return stats
# Upload to Gitea PyPI registry (requires name, version, sha256_digest)
dist_files = list(dl_dir.iterdir())
print(f" pypi: uploading {len(dist_files)} files to Gitea...")
upload_url = f"http://localhost:3000/api/packages/{GITEA_OWNER}/pypi"
for dist in sorted(dist_files):
# Parse name and version from filename
pkg_name, pkg_version = _parse_dist_filename(dist.name)
if not pkg_name:
print(f" SKIP: {dist.name} (can't parse name/version)")
stats["skipped"] += 1
continue
# Compute SHA-256
import hashlib
sha256 = hashlib.sha256(dist.read_bytes()).hexdigest()
# Copy file into gitea container and upload
tmp_name = f"/tmp/pkg_{dist.name}"
subprocess.run(
["docker", "cp", str(dist), f"gitea:{tmp_name}"],
capture_output=True, timeout=60,
)
result_up = subprocess.run(
[
"docker", "exec", "gitea", "curl", "-s",
"-w", "\n%{http_code}",
"-H", f"Authorization: token {GITEA_TOKEN}",
"-F", f"content=@{tmp_name}",
"-F", f"name={pkg_name}",
"-F", f"version={pkg_version}",
"-F", f"sha256_digest={sha256}",
upload_url,
],
capture_output=True, text=True, timeout=120,
)
lines = result_up.stdout.strip().rsplit("\n", 1)
status = int(lines[-1]) if lines[-1].isdigit() else 500
if status in (201, 409):
label = "uploaded" if status == 201 else "exists"
print(f" {label}: {dist.name}")
if status == 201:
stats["uploaded"] += 1
else:
stats["skipped"] += 1
else:
body = lines[0] if len(lines) > 1 else ""
print(f" FAILED ({status}): {dist.name} — {body[:120]}")
stats["failed"].append(dist.name)
# Cleanup
subprocess.run(
["docker", "exec", "gitea", "rm", "-f", tmp_name],
capture_output=True, timeout=10,
)
return stats
def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
"""Download .deb packages and upload to Gitea Debian registry."""
pkgs = _flat_apt_packages(manifest)
if not pkgs:
print(" apt: no packages to mirror")
return {"uploaded": 0, "skipped": 0, "failed": []}
stats = {"uploaded": 0, "skipped": 0, "failed": []}
if dry_run:
print(f" apt: would download and upload {len(pkgs)} packages:")
for p in pkgs:
print(f" - {p}")
return stats
# Use a subdirectory under mirrors/ for apt downloads (avoids tmpdir
# permission issues with Docker volume mounts)
dl_dir = ROOT / "mirrors" / "cache" / "apt"
dl_dir.mkdir(parents=True, exist_ok=True)
# Clean previous downloads
for old in dl_dir.glob("*.deb"):
old.unlink()
# Download .debs via Docker — try each package individually to handle
# third-party packages (e.g. zotero) that aren't in base Debian repos
print(f" apt: downloading {len(pkgs)} packages...")
# Build a script that tries each package, skipping failures
install_cmds = " && ".join(
f"(apt-get install --reinstall --download-only -y {p} 2>/dev/null || "
f"echo 'SKIP: {p} (not in base repos)')"
for p in pkgs
)
subprocess.run(
[
"docker", "run", "--rm",
"-v", f"{dl_dir}:/debs",
"debian:bookworm-slim",
"bash", "-c",
f"apt-get update -qq 2>/dev/null && {install_cmds}; "
f"cp /var/cache/apt/archives/*.deb /debs/ 2>/dev/null || true; "
f"chmod 644 /debs/*.deb 2>/dev/null || true",
],
capture_output=True, text=True, timeout=300,
)
deb_files = list(dl_dir.glob("*.deb"))
if not deb_files:
print(" apt: no .deb files downloaded")
stats["skipped"] = len(pkgs)
return stats
print(f" apt: uploading {len(deb_files)} .deb files to Gitea...")
for deb in sorted(deb_files):
tmp_name = f"/tmp/pkg_{deb.name}"
subprocess.run(
["docker", "cp", str(deb), f"gitea:{tmp_name}"],
capture_output=True, timeout=60,
)
result_up = subprocess.run(
[
"docker", "exec", "gitea", "curl", "-s",
"-w", "\n%{http_code}",
"-H", f"Authorization: token {GITEA_TOKEN}",
"--upload-file", tmp_name,
f"http://localhost:3000/api/packages/{GITEA_OWNER}"
f"/debian/pool/bookworm/main/upload",
],
capture_output=True, text=True, timeout=120,
)
lines = result_up.stdout.strip().rsplit("\n", 1)
status = int(lines[-1]) if lines[-1].isdigit() else 500
if status in (201, 409):
label = "uploaded" if status == 201 else "exists"
print(f" {label}: {deb.name}")
if status == 201:
stats["uploaded"] += 1
else:
stats["skipped"] += 1
else:
print(f" pypi: cache warm failed (non-fatal): {result.stderr[:200]}")
except (FileNotFoundError, subprocess.TimeoutExpired) as e:
print(f" pypi: cache warm skipped: {e}")
body = lines[0] if len(lines) > 1 else ""
print(f" FAILED ({status}): {deb.name} — {body[:120]}")
stats["failed"].append(deb.name)
subprocess.run(
["docker", "exec", "gitea", "rm", "-f", tmp_name],
capture_output=True, timeout=10,
)
return stats
def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
"""Download .apk packages and upload to Gitea Alpine registry.
Currently we have no explicit apk add packages (only apk upgrade),
so this is primarily for future use when Alpine packages are added.
"""
# We don't have explicit apk packages right now — just apk upgrade
# which updates base packages. We'd need to mirror the full Alpine
# repo to support that, which isn't practical. Instead, we'll handle
# this by pinning the nginx:alpine image version.
if dry_run:
print(" apk: no explicit packages to mirror (apk upgrade uses base repo)")
else:
print(" apk: skipped — no explicit apk packages in manifest")
return {"uploaded": 0, "skipped": 0, "failed": []}
def main() -> None:
import argparse
parser = argparse.ArgumentParser(description="Sync package mirrors from manifest")
parser = argparse.ArgumentParser(
description="Download packages from upstream and upload to Gitea registry"
)
parser.add_argument(
"--type",
choices=["apt", "apk", "pypi", "all"],
default="all",
help="Which mirror type to sync",
help="Which package type to sync",
)
parser.add_argument("--dry-run", action="store_true", help="Show what would change")
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
manifest = load_manifest()
print("Syncing mirrors from pkg-manifest.json...")
_load_env()
if not GITEA_TOKEN:
print("ERROR: GITEA_TOKEN not set. Set it in .env or environment.")
raise SystemExit(2)
if args.type in ("apt", "all"):
sync_apt(manifest, dry_run=args.dry_run)
if args.type in ("apk", "all"):
sync_apk(manifest, dry_run=args.dry_run)
manifest = load_manifest()
print("Syncing packages to Gitea registry...")
results: dict[str, dict] = {}
if args.type in ("pypi", "all"):
sync_pypi(manifest, dry_run=args.dry_run)
results["pypi"] = sync_pypi(manifest, dry_run=args.dry_run)
if args.type in ("apt", "all"):
results["apt"] = sync_apt(manifest, dry_run=args.dry_run)
if args.type in ("apk", "all"):
results["apk"] = sync_apk(manifest, dry_run=args.dry_run)
if not args.dry_run:
print("\nMirror seed files generated. Start mirrors with:")
print(" docker compose up -d apt-cache devpi")
print("\n=== Summary ===")
total_up = sum(r.get("uploaded", 0) for r in results.values())
total_skip = sum(r.get("skipped", 0) for r in results.values())
total_fail = sum(len(r.get("failed", [])) for r in results.values())
print(f" Uploaded: {total_up} Skipped: {total_skip} Failed: {total_fail}")
if total_fail:
raise SystemExit(1)
if __name__ == "__main__":

View File

@@ -1,2 +0,0 @@
# Alpine packages to pre-cache (includes base for apk upgrade)
# No explicit packages — mirror Alpine index only

View File

@@ -1,5 +0,0 @@
# apt-cacher-ng config — generated by pkg_mirror_sync.py
CacheDir: /var/cache/apt-cacher-ng
LogDir: /var/log/apt-cacher-ng
Port: 3142
PassThroughPattern: .*

View File

@@ -1,5 +0,0 @@
build-essential
ca-certificates
curl
git
zotero

View File

@@ -1,9 +0,0 @@
# devpi config — generated by pkg_mirror_sync.py
[devpi-server]
serverdir = /var/lib/devpi
port = 3141
host = 0.0.0.0
[mirror]
type = mirror
url = https://pypi.org/simple/

View File

@@ -1,41 +0,0 @@
# PyPI packages to mirror — generated by pkg_mirror_sync.py
# Feed to: devpi-server --mirror, or pip download -r
altair>=6.0.0
coverage>=7.13.4
cryptography>=46.0.5
cudf-polars-cu12
databricks-cli>=0.18.0
databricks-sdk>=0.85.0
dbt-core==1.10.15
dbt-duckdb>=1.10,<1.11
duckdb>=1.0.0
fastapi>=0.135.1
fastexcel>=0.19.0
fsspec>=2024.1.0
griffe
httpx>=0.28.1
marimo>=0.20.0
narwhals>=2.17.0
numpy
obstore>=0.9.2
openpyxl>=3.1.5
pandas>=3.0.1
pdfplumber>=0.11.9
pillow>=12.1.1
polars>=1.38.1
pyarrow>=23.0.0
pyasn1>=0.6.3
pydantic
pyiceberg[s3,pyarrow]>=0.7.0
pyjwt>=2.12.0
pytest>=9.0.2
pytest-cov>=7.0.0
pyzotero
ruff>=0.11.0
s3fs>=2026.2.0
sqlglot>=26.0.0
trino>=0.328.0
typer>=0.24.1
uv-build>=0.7,<1
uvicorn>=0.41.0
vega-datasets

View File

@@ -6,6 +6,9 @@ ARG USER_UID=1000
ARG USER_GID=1000
ARG PYTHON_VERSION=3.13
# Local package registry (Gitea) — set via --build-arg to pull from mirror
ARG PYPI_INDEX_URL=""
ENV DEBIAN_FRONTEND=noninteractive \
HOME=/home/kert \
PATH="/home/kert/.local/bin:${PATH}" \
@@ -34,6 +37,7 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uvx /usr/local/bin/uvx
WORKDIR /home/${USERNAME}
# Initialize uv project and install dependencies
ENV UV_INDEX_URL=${PYPI_INDEX_URL}
RUN uv python install ${PYTHON_VERSION} \
&& uv init workspace --python ${PYTHON_VERSION} \
&& cd workspace \

View File

@@ -24,7 +24,7 @@ subdomains = [
"dashboard", "docs", "gitea", "ci", "notebooks", "zotero",
"webdav", "api", "nessie", "trino", "polaris",
"grafana", "prometheus", "jaeger", "loki",
"s3", "s3console",
"s3", "s3console", "packages",
]
[services]
@@ -114,6 +114,13 @@ port = 8000
secret = "" # override via STACK_API_SECRET env var
workers = 1
[mirrors]
# Gitea package registry — single source for all mirrored packages
owner = "homelab"
pypi_index = "http://gitea:3000/api/packages/homelab/pypi/simple/"
debian_source = "http://gitea:3000/api/packages/homelab/debian"
alpine_repo = "http://gitea:3000/api/packages/homelab/alpine"
[ci]
backend = "gitea"

View File

@@ -19,6 +19,7 @@
{{- $multi := dict
"rustfs-api" (dict "container" "rustfs" "port" "9000" "subdomain" "s3" "theme" false "mw" "local-only,infra-headers")
"rustfs-console" (dict "container" "rustfs" "port" "9001" "subdomain" "s3console" "theme" true "mw" "local-only,infra-headers")
"gitea-packages" (dict "container" "gitea" "port" "3000" "subdomain" "packages" "theme" false "mw" "local-only,secure-headers")
-}}
http:
middlewares: