fix apk mirroring: enumerate installed packages from Alpine images, upload to Gitea
All checks were successful
CI / lint-test (push) Successful in 1m19s
CI / lint-test (pull_request) Successful in 1m30s
Infra CI / zotero (pull_request) Successful in 6s
Infra CI / docs (pull_request) Successful in 6s
Infra CI / api (pull_request) Successful in 7s
Infra CI / mc (pull_request) Successful in 6s
Infra CI / notebooks (pull_request) Successful in 2m48s
All checks were successful
CI / lint-test (push) Successful in 1m19s
CI / lint-test (pull_request) Successful in 1m30s
Infra CI / zotero (pull_request) Successful in 6s
Infra CI / docs (pull_request) Successful in 6s
Infra CI / api (pull_request) Successful in 7s
Infra CI / mc (pull_request) Successful in 6s
Infra CI / notebooks (pull_request) Successful in 2m48s
closes #161 - Scan nginx:alpine for all 72 installed packages via `apk info` - Fetch each package individually (handles custom repos gracefully) - 67 .apk files uploaded to Gitea Alpine registry (v3.23/main) - 5 nginx modules from custom repos skipped (baked into base image)
This commit is contained in:
@@ -341,20 +341,129 @@ def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
|
||||
|
||||
|
||||
def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
|
||||
"""Download .apk packages and upload to Gitea Alpine registry.
|
||||
"""Download .apk packages from Alpine base images and upload to Gitea.
|
||||
|
||||
Currently we have no explicit apk add packages (only apk upgrade),
|
||||
so this is primarily for future use when Alpine packages are added.
|
||||
Even when no explicit `apk add` packages exist, `apk upgrade` in
|
||||
Dockerfiles pulls updates for every installed package. We mirror
|
||||
all installed packages from Alpine-based images so builds can run
|
||||
fully offline.
|
||||
"""
|
||||
# We don't have explicit apk packages right now — just apk upgrade
|
||||
# which updates base packages. We'd need to mirror the full Alpine
|
||||
# repo to support that, which isn't practical. Instead, we'll handle
|
||||
# this by pinning the nginx:alpine image version.
|
||||
stats = {"uploaded": 0, "skipped": 0, "failed": []}
|
||||
|
||||
# Find Alpine-based images from the manifest
|
||||
alpine_images: list[str] = []
|
||||
for rel, images in manifest.get("base_images", {}).items():
|
||||
for img in images:
|
||||
if "alpine" in img.lower():
|
||||
alpine_images.append(img)
|
||||
|
||||
if not alpine_images:
|
||||
print(" apk: no Alpine-based images in manifest")
|
||||
return stats
|
||||
|
||||
# Get the list of installed packages from each Alpine image
|
||||
all_pkgs: set[str] = set()
|
||||
for img in alpine_images:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm", img,
|
||||
"sh", "-c", "apk info 2>/dev/null",
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
for line in result.stdout.strip().splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
all_pkgs.add(line)
|
||||
|
||||
if not all_pkgs:
|
||||
print(" apk: could not enumerate packages from Alpine images")
|
||||
return stats
|
||||
|
||||
if dry_run:
|
||||
print(" apk: no explicit packages to mirror (apk upgrade uses base repo)")
|
||||
else:
|
||||
print(" apk: skipped — no explicit apk packages in manifest")
|
||||
return {"uploaded": 0, "skipped": 0, "failed": []}
|
||||
print(f" apk: would download and upload {len(all_pkgs)} packages from Alpine images:")
|
||||
for p in sorted(all_pkgs):
|
||||
print(f" - {p}")
|
||||
return stats
|
||||
|
||||
# Get Alpine version from first image
|
||||
ver_result = subprocess.run(
|
||||
["docker", "run", "--rm", alpine_images[0],
|
||||
"cat", "/etc/alpine-release"],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
alpine_version = "v" + ".".join(
|
||||
ver_result.stdout.strip().split(".")[:2]
|
||||
) if ver_result.returncode == 0 else "v3.23"
|
||||
|
||||
# Download .apk files
|
||||
dl_dir = ROOT / "mirrors" / "cache" / "apk"
|
||||
dl_dir.mkdir(parents=True, exist_ok=True)
|
||||
for old in dl_dir.glob("*.apk"):
|
||||
old.unlink()
|
||||
|
||||
# Fetch each package individually — some (e.g. nginx modules) come
|
||||
# from repos not in the default Alpine config, so batch fetch fails.
|
||||
fetch_cmds = "; ".join(
|
||||
f'apk fetch --no-cache -o /out {p} 2>/dev/null || true'
|
||||
for p in sorted(all_pkgs)
|
||||
)
|
||||
print(f" apk: downloading {len(all_pkgs)} packages from Alpine {alpine_version}...")
|
||||
subprocess.run(
|
||||
[
|
||||
"docker", "run", "--rm",
|
||||
"-v", f"{dl_dir}:/out",
|
||||
alpine_images[0],
|
||||
"sh", "-c",
|
||||
f"{fetch_cmds}; chmod 644 /out/*.apk 2>/dev/null || true",
|
||||
],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
)
|
||||
|
||||
apk_files = list(dl_dir.glob("*.apk"))
|
||||
if not apk_files:
|
||||
print(" apk: no .apk files downloaded")
|
||||
stats["skipped"] = len(all_pkgs)
|
||||
return stats
|
||||
|
||||
print(f" apk: uploading {len(apk_files)} .apk files to Gitea...")
|
||||
for apk in sorted(apk_files):
|
||||
tmp_name = f"/tmp/pkg_{apk.name}"
|
||||
subprocess.run(
|
||||
["docker", "cp", str(apk), f"gitea:{tmp_name}"],
|
||||
capture_output=True, timeout=60,
|
||||
)
|
||||
result_up = subprocess.run(
|
||||
[
|
||||
"docker", "exec", "gitea", "curl", "-s",
|
||||
"-w", "\n%{http_code}",
|
||||
"-H", f"Authorization: token {GITEA_TOKEN}",
|
||||
"--upload-file", tmp_name,
|
||||
f"http://localhost:3000/api/packages/{GITEA_OWNER}"
|
||||
f"/alpine/{alpine_version}/main",
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
lines = result_up.stdout.strip().rsplit("\n", 1)
|
||||
status = int(lines[-1]) if lines[-1].isdigit() else 500
|
||||
if status in (201, 409):
|
||||
label = "uploaded" if status == 201 else "exists"
|
||||
print(f" {label}: {apk.name}")
|
||||
if status == 201:
|
||||
stats["uploaded"] += 1
|
||||
else:
|
||||
stats["skipped"] += 1
|
||||
else:
|
||||
body = lines[0] if len(lines) > 1 else ""
|
||||
print(f" FAILED ({status}): {apk.name} — {body[:120]}")
|
||||
stats["failed"].append(apk.name)
|
||||
subprocess.run(
|
||||
["docker", "exec", "gitea", "rm", "-f", tmp_name],
|
||||
capture_output=True, timeout=10,
|
||||
)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
Reference in New Issue
Block a user