stack/dev/scripts/pkg_mirror_sync.py

"""Download packages from upstream and upload to Gitea package registry.

Reads data/pkg-manifest.json, downloads each package from its upstream
source (PyPI, Debian repos, Alpine repos), then pushes to Gitea's
built-in package registry so all builds pull exclusively from Gitea.

Usage:
    uv run python dev/scripts/pkg_mirror_sync.py              # sync all
    uv run python dev/scripts/pkg_mirror_sync.py --type pypi   # sync pypi only
    uv run python dev/scripts/pkg_mirror_sync.py --dry-run     # show what would change
"""

from __future__ import annotations

import json
import os
import subprocess
import tempfile
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
CACHE_DIR = ROOT / "mirrors" / "cache"

# Gitea registry config — loaded from env or .env file
GITEA_URL = ""
GITEA_TOKEN = ""
GITEA_OWNER = "homelab"
# Docker container name of the Gitea service (compose.yml container_name)
GITEA_CONTAINER = "git"


def _load_env() -> None:
    global GITEA_URL, GITEA_TOKEN, GITEA_OWNER
    GITEA_URL = os.environ.get("GITEA_URL", "")
    GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "")

    if not GITEA_TOKEN:
        env_file = ROOT / ".env"
        if env_file.exists():
            for line in env_file.read_text().splitlines():
                if line.startswith("GITEA_TOKEN="):
                    GITEA_TOKEN = line.split("=", 1)[1].strip().strip('"').strip("'")

    if not GITEA_URL:
        # Read from stack.toml [services]
        toml_file = ROOT / "stack.toml"
        if toml_file.exists():
            for line in toml_file.read_text().splitlines():
                stripped = line.strip()
                if stripped.startswith("git ") or stripped.startswith("git="):
                    parts = line.split("=", 1)
                    if len(parts) == 2:
                        GITEA_URL = parts[1].strip().strip('"').strip("'")
                        break
        if not GITEA_URL:
            GITEA_URL = "http://git:3000"


def _api_via_docker(method: str, path: str, file_path: str = "") -> tuple[int, str]:
    """Call Gitea API via docker exec (handles DNS resolution)."""
    url = f"http://localhost:3000/api/v1/{path}"
    cmd = [
        "docker",
        "exec",
        GITEA_CONTAINER,
        "curl",
        "-s",
        "-w",
        "\n%{http_code}",
        "-H",
        f"Authorization: token {GITEA_TOKEN}",
    ]
    if method == "PUT" and file_path:
        # For file uploads, we need to copy the file into the container first
        tmp_name = f"/tmp/pkg_upload_{os.path.basename(file_path)}"
        subprocess.run(
            ["docker", "cp", file_path, f"{GITEA_CONTAINER}:{tmp_name}"],
            capture_output=True,
            timeout=60,
        )
        cmd += ["-X", "PUT", "--upload-file", tmp_name, url]
    elif method == "GET":
        cmd += [url]
    else:
        cmd += ["-X", method, url]

    result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
    if result.returncode != 0:
        return 500, result.stderr[:200]

    lines = result.stdout.strip().rsplit("\n", 1)
    body = lines[0] if len(lines) > 1 else ""
    status = int(lines[-1]) if lines[-1].isdigit() else 500
    return status, body


def _gitea_pkg_exists(pkg_type: str, name: str) -> bool:
    """Check if a package already exists in Gitea registry."""
    path = f"packages/{GITEA_OWNER}/{pkg_type}?q={name}&limit=1"
    status, body = _api_via_docker("GET", path)
    if status == 200:
        try:
            data = json.loads(body)
            return len(data) > 0
        except json.JSONDecodeError:
            pass
    return False


def _parse_dist_filename(filename: str) -> tuple[str, str]:
    """Extract package name and version from a wheel or sdist filename.

    Examples:
        pyasn1-0.6.3-py3-none-any.whl -> (pyasn1, 0.6.3)
        sqlglot-26.0.0.tar.gz -> (sqlglot, 26.0.0)
    """
    import re

    # Wheel: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl
    m = re.match(r"^(.+?)-(\d+[^-]*)-", filename)
    if m:
        return m.group(1).replace("_", "-").lower(), m.group(2)

    # Sdist: {name}-{version}.tar.gz or {name}-{version}.zip
    m = re.match(r"^(.+?)-(\d+\S+?)\.(?:tar\.gz|zip)$", filename)
    if m:
        return m.group(1).replace("_", "-").lower(), m.group(2)

    return "", ""


def load_manifest() -> dict:
    if not MANIFEST_PATH.exists():
        print(f"ERROR: {MANIFEST_PATH} not found. Run pkg_inventory.py first.")
        raise SystemExit(2)
    return json.loads(MANIFEST_PATH.read_text())


def _flat_apt_packages(manifest: dict) -> list[str]:
    pkgs: set[str] = set()
    for pkg_list in manifest.get("apt", {}).values():
        pkgs.update(pkg_list)
    return sorted(pkgs)


def _flat_pypi_packages(manifest: dict) -> list[dict]:
    seen: set[str] = set()
    pkgs: list[dict] = []
    # Skip the local project — it can't be downloaded from PyPI
    local_project = "stack"
    for section in ("project_prod", "project_dev", "notebook", "dockerfile_adhoc"):
        for pkg in manifest.get("pypi", {}).get(section, []):
            if pkg["name"] == local_project:
                continue
            if pkg["name"] not in seen:
                seen.add(pkg["name"])
                pkgs.append(pkg)
    return sorted(pkgs, key=lambda p: p["name"])


def sync_pypi(manifest: dict, *, dry_run: bool = False) -> dict:
    """Download Python wheels from PyPI and upload to Gitea."""
    pkgs = _flat_pypi_packages(manifest)
    if not pkgs:
        print("  pypi: no packages to mirror")
        return {"uploaded": 0, "skipped": 0, "failed": []}

    stats = {"uploaded": 0, "skipped": 0, "failed": []}

    with tempfile.TemporaryDirectory(prefix="pkg_pypi_") as tmpdir:
        # Build requirements spec
        specs = []
        for pkg in pkgs:
            spec = pkg["name"]
            if pkg.get("extras"):
                spec += f"[{pkg['extras']}]"
            if pkg.get("version"):
                spec += pkg["version"]
            specs.append(spec)

        req_file = Path(tmpdir) / "requirements.txt"
        req_file.write_text("\n".join(specs) + "\n")

        if dry_run:
            print(f"  pypi: would download and upload {len(specs)} packages:")
            for s in specs:
                print(f"    - {s}")
            return stats

        # Download wheels/sdists from upstream PyPI
        print(f"  pypi: downloading {len(specs)} packages from PyPI...")
        dl_dir = Path(tmpdir) / "downloads"
        dl_dir.mkdir()
        result = subprocess.run(
            [
                "uvx",
                "pip",
                "download",
                "--no-deps",
                "--dest",
                str(dl_dir),
                "-r",
                str(req_file),
            ],
            capture_output=True,
            text=True,
            timeout=600,
        )
        if result.returncode != 0:
            print(f"  pypi: download failed: {result.stderr[:500]}")
            stats["failed"].append("pip-download")
            return stats

        # Upload to Gitea PyPI registry (requires name, version, sha256_digest)
        dist_files = list(dl_dir.iterdir())
        print(f"  pypi: uploading {len(dist_files)} files to Gitea...")
        upload_url = f"http://localhost:3000/api/packages/{GITEA_OWNER}/pypi"
        for dist in sorted(dist_files):
            # Parse name and version from filename
            pkg_name, pkg_version = _parse_dist_filename(dist.name)
            if not pkg_name:
                print(f"    SKIP: {dist.name} (can't parse name/version)")
                stats["skipped"] += 1
                continue

            # Compute SHA-256
            import hashlib

            sha256 = hashlib.sha256(dist.read_bytes()).hexdigest()

            # Copy file into gitea container and upload
            tmp_name = f"/tmp/pkg_{dist.name}"
            subprocess.run(
                ["docker", "cp", str(dist), f"{GITEA_CONTAINER}:{tmp_name}"],
                capture_output=True,
                timeout=60,
            )
            result_up = subprocess.run(
                [
                    "docker",
                    "exec",
                    GITEA_CONTAINER,
                    "curl",
                    "-s",
                    "-w",
                    "\n%{http_code}",
                    "-H",
                    f"Authorization: token {GITEA_TOKEN}",
                    "-F",
                    f"content=@{tmp_name}",
                    "-F",
                    f"name={pkg_name}",
                    "-F",
                    f"version={pkg_version}",
                    "-F",
                    f"sha256_digest={sha256}",
                    upload_url,
                ],
                capture_output=True,
                text=True,
                timeout=120,
            )
            lines = result_up.stdout.strip().rsplit("\n", 1)
            status = int(lines[-1]) if lines[-1].isdigit() else 500
            if status in (201, 409):
                label = "uploaded" if status == 201 else "exists"
                print(f"    {label}: {dist.name}")
                if status == 201:
                    stats["uploaded"] += 1
                else:
                    stats["skipped"] += 1
            else:
                body = lines[0] if len(lines) > 1 else ""
                print(f"    FAILED ({status}): {dist.name} — {body[:120]}")
                stats["failed"].append(dist.name)
            # Cleanup
            subprocess.run(
                ["docker", "exec", GITEA_CONTAINER, "rm", "-f", tmp_name],
                capture_output=True,
                timeout=10,
            )

    return stats


def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
    """Download .deb packages and upload to Gitea Debian registry."""
    pkgs = _flat_apt_packages(manifest)
    if not pkgs:
        print("  apt: no packages to mirror")
        return {"uploaded": 0, "skipped": 0, "failed": []}

    stats = {"uploaded": 0, "skipped": 0, "failed": []}

    if dry_run:
        print(f"  apt: would download and upload {len(pkgs)} packages:")
        for p in pkgs:
            print(f"    - {p}")
        return stats

    # Use a subdirectory under mirrors/ for apt downloads (avoids tmpdir
    # permission issues with Docker volume mounts)
    dl_dir = ROOT / "mirrors" / "cache" / "apt"
    dl_dir.mkdir(parents=True, exist_ok=True)
    # Clean previous downloads
    for old in dl_dir.glob("*.deb"):
        old.unlink()

    # Download .debs via Docker — try each package individually to handle
    # third-party packages (e.g. zotero) that aren't in base Debian repos
    print(f"  apt: downloading {len(pkgs)} packages...")
    # Build a script that tries each package, skipping failures
    install_cmds = " && ".join(
        f"(apt-get install --reinstall --download-only -y {p} 2>/dev/null || "
        f"echo 'SKIP: {p} (not in base repos)')"
        for p in pkgs
    )
    subprocess.run(
        [
            "docker",
            "run",
            "--rm",
            "-v",
            f"{dl_dir}:/debs",
            "debian:bookworm-slim",
            "bash",
            "-c",
            f"apt-get update -qq 2>/dev/null && {install_cmds}; "
            f"cp /var/cache/apt/archives/*.deb /debs/ 2>/dev/null || true; "
            f"chmod 644 /debs/*.deb 2>/dev/null || true",
        ],
        capture_output=True,
        text=True,
        timeout=300,
    )

    deb_files = list(dl_dir.glob("*.deb"))
    if not deb_files:
        print("  apt: no .deb files downloaded")
        stats["skipped"] = len(pkgs)
        return stats

    print(f"  apt: uploading {len(deb_files)} .deb files to Gitea...")
    for deb in sorted(deb_files):
        tmp_name = f"/tmp/pkg_{deb.name}"
        subprocess.run(
            ["docker", "cp", str(deb), f"{GITEA_CONTAINER}:{tmp_name}"],
            capture_output=True,
            timeout=60,
        )
        result_up = subprocess.run(
            [
                "docker",
                "exec",
                GITEA_CONTAINER,
                "curl",
                "-s",
                "-w",
                "\n%{http_code}",
                "-H",
                f"Authorization: token {GITEA_TOKEN}",
                "--upload-file",
                tmp_name,
                f"http://localhost:3000/api/packages/{GITEA_OWNER}"
                f"/debian/pool/bookworm/main/upload",
            ],
            capture_output=True,
            text=True,
            timeout=120,
        )
        lines = result_up.stdout.strip().rsplit("\n", 1)
        status = int(lines[-1]) if lines[-1].isdigit() else 500
        if status in (201, 409):
            label = "uploaded" if status == 201 else "exists"
            print(f"    {label}: {deb.name}")
            if status == 201:
                stats["uploaded"] += 1
            else:
                stats["skipped"] += 1
        else:
            body = lines[0] if len(lines) > 1 else ""
            print(f"    FAILED ({status}): {deb.name} — {body[:120]}")
            stats["failed"].append(deb.name)
        subprocess.run(
            ["docker", "exec", GITEA_CONTAINER, "rm", "-f", tmp_name],
            capture_output=True,
            timeout=10,
        )

    return stats


def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
    """Download .apk packages from Alpine base images and upload to Gitea.

    Even when no explicit `apk add` packages exist, `apk upgrade` in
    Dockerfiles pulls updates for every installed package. We mirror
    all installed packages from Alpine-based images so builds can run
    fully offline.
    """
    stats = {"uploaded": 0, "skipped": 0, "failed": []}

    # Find Alpine-based images from the manifest
    alpine_images: list[str] = []
    for rel, images in manifest.get("base_images", {}).items():
        for img in images:
            if "alpine" in img.lower():
                alpine_images.append(img)

    if not alpine_images:
        print("  apk: no Alpine-based images in manifest")
        return stats

    # Get the list of installed packages from each Alpine image
    all_pkgs: set[str] = set()
    for img in alpine_images:
        result = subprocess.run(
            [
                "docker",
                "run",
                "--rm",
                img,
                "sh",
                "-c",
                "apk info 2>/dev/null",
            ],
            capture_output=True,
            text=True,
            timeout=120,
        )
        if result.returncode == 0:
            for line in result.stdout.strip().splitlines():
                line = line.strip()
                if line:
                    all_pkgs.add(line)

    if not all_pkgs:
        print("  apk: could not enumerate packages from Alpine images")
        return stats

    if dry_run:
        print(
            f"  apk: would download and upload {len(all_pkgs)} packages from Alpine images:"
        )
        for p in sorted(all_pkgs):
            print(f"    - {p}")
        return stats

    # Get Alpine version from first image
    ver_result = subprocess.run(
        ["docker", "run", "--rm", alpine_images[0], "cat", "/etc/alpine-release"],
        capture_output=True,
        text=True,
        timeout=30,
    )
    alpine_version = (
        "v" + ".".join(ver_result.stdout.strip().split(".")[:2])
        if ver_result.returncode == 0
        else "v3.23"
    )

    # Download .apk files
    dl_dir = ROOT / "mirrors" / "cache" / "apk"
    dl_dir.mkdir(parents=True, exist_ok=True)
    for old in dl_dir.glob("*.apk"):
        old.unlink()

    # Fetch each package individually — some (e.g. nginx modules) come
    # from repos not in the default Alpine config, so batch fetch fails.
    fetch_cmds = "; ".join(
        f"apk fetch --no-cache -o /out {p} 2>/dev/null || true"
        for p in sorted(all_pkgs)
    )
    print(
        f"  apk: downloading {len(all_pkgs)} packages from Alpine {alpine_version}..."
    )
    subprocess.run(
        [
            "docker",
            "run",
            "--rm",
            "-v",
            f"{dl_dir}:/out",
            alpine_images[0],
            "sh",
            "-c",
            f"{fetch_cmds}; chmod 644 /out/*.apk 2>/dev/null || true",
        ],
        capture_output=True,
        text=True,
        timeout=300,
    )

    apk_files = list(dl_dir.glob("*.apk"))
    if not apk_files:
        print("  apk: no .apk files downloaded")
        stats["skipped"] = len(all_pkgs)
        return stats

    print(f"  apk: uploading {len(apk_files)} .apk files to Gitea...")
    for apk in sorted(apk_files):
        tmp_name = f"/tmp/pkg_{apk.name}"
        subprocess.run(
            ["docker", "cp", str(apk), f"{GITEA_CONTAINER}:{tmp_name}"],
            capture_output=True,
            timeout=60,
        )
        result_up = subprocess.run(
            [
                "docker",
                "exec",
                GITEA_CONTAINER,
                "curl",
                "-s",
                "-w",
                "\n%{http_code}",
                "-H",
                f"Authorization: token {GITEA_TOKEN}",
                "--upload-file",
                tmp_name,
                f"http://localhost:3000/api/packages/{GITEA_OWNER}"
                f"/alpine/{alpine_version}/main",
            ],
            capture_output=True,
            text=True,
            timeout=120,
        )
        lines = result_up.stdout.strip().rsplit("\n", 1)
        status = int(lines[-1]) if lines[-1].isdigit() else 500
        if status in (201, 409):
            label = "uploaded" if status == 201 else "exists"
            print(f"    {label}: {apk.name}")
            if status == 201:
                stats["uploaded"] += 1
            else:
                stats["skipped"] += 1
        else:
            body = lines[0] if len(lines) > 1 else ""
            print(f"    FAILED ({status}): {apk.name} — {body[:120]}")
            stats["failed"].append(apk.name)
        subprocess.run(
            ["docker", "exec", GITEA_CONTAINER, "rm", "-f", tmp_name],
            capture_output=True,
            timeout=10,
        )

    return stats


def main() -> None:
    import argparse

    parser = argparse.ArgumentParser(
        description="Download packages from upstream and upload to Gitea registry"
    )
    parser.add_argument(
        "--type",
        choices=["apt", "apk", "pypi", "all"],
        default="all",
        help="Which package type to sync",
    )
    parser.add_argument("--dry-run", action="store_true")
    args = parser.parse_args()

    _load_env()
    if not GITEA_TOKEN:
        print("ERROR: GITEA_TOKEN not set. Set it in .env or environment.")
        raise SystemExit(2)

    manifest = load_manifest()
    print("Syncing packages to Gitea registry...")

    results: dict[str, dict] = {}
    if args.type in ("pypi", "all"):
        results["pypi"] = sync_pypi(manifest, dry_run=args.dry_run)
    if args.type in ("apt", "all"):
        results["apt"] = sync_apt(manifest, dry_run=args.dry_run)
    if args.type in ("apk", "all"):
        results["apk"] = sync_apk(manifest, dry_run=args.dry_run)

    if not args.dry_run:
        print("\n=== Summary ===")
        total_up = sum(r.get("uploaded", 0) for r in results.values())
        total_skip = sum(r.get("skipped", 0) for r in results.values())
        total_fail = sum(len(r.get("failed", [])) for r in results.values())
        print(f"  Uploaded: {total_up}  Skipped: {total_skip}  Failed: {total_fail}")

        if total_fail:
            raise SystemExit(1)


if __name__ == "__main__":
    main()