add package mirrors, auto-issue scripts, network isolation test, CI workflow

refs #160 #161 #162 #163 #164 #166 #167 #168

- compose.yml: add apt-cacher-ng and devpi services on ci network
- pkg_mirror_sync.py: generates seed files for apt/apk/pypi mirrors
- pkg_issues.py: auto-creates Gitea issues for drift and CVEs,
  deduplicates, auto-closes resolved issues
- test_network_isolation.sh: builds all Dockerfiles with --network=none
- gitea.py: add pkg-supply-chain workflow (daily + on Dockerfile/dep changes)
- mirrors/: seed files for apt (5 pkgs), apk (index), pypi (39 pkgs)

.gitea/workflows/pkg-supply-chain.yml

@@ -0,0 +1,56 @@
+# DO NOT EDIT — generated by gen_config.py from stack.toml
+# Re-generate: uv run python dev/scripts/gen_config.py
+
+name: Package Supply Chain
+
+on:
+  push:
+    paths:
+      - "**/Dockerfile*"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "docs/package.json"
+      - "docs/pnpm-lock.yaml"
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * *"
+
+jobs:
+  pkg-supply-chain:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: https://github.com/actions/checkout@v4
+
+      - name: Set up uv
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+        env:
+          UV_INSTALL_DIR: /usr/local/bin
+
+      - name: Install dependencies
+        run: uv sync --no-dev
+
+      - name: Package inventory
+        run: uv run python dev/scripts/pkg_inventory.py
+
+      - name: Check manifest freshness
+        run: uv run python dev/scripts/pkg_inventory.py --check
+
+      - name: Sync mirrors
+        run: uv run python dev/scripts/pkg_mirror_sync.py
+
+      - name: Drift detection
+        run: uv run python dev/scripts/pkg_drift.py
+
+      - name: Install trivy
+        run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
+
+      - name: Vulnerability scan
+        run: |
+          trivy fs --scanners vuln --format json -o data/pkg-vulns.json uv.lock || true
+          trivy fs --scanners vuln --format json -o data/pkg-vulns-pyproject.json pyproject.toml || true
+
+      - name: Auto-create issues for drift and vulns
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: uv run python dev/scripts/pkg_issues.py

compose.yml

@@ -600,6 +600,32 @@ services:
       - no-new-privileges:true
     restart: unless-stopped
 
+  # Package mirrors — local caches for reproducible builds
+  apt-cache:
+    image: sameersbn/apt-cacher-ng:latest
+    container_name: apt-cache
+    networks:
+      - ci
+    volumes:
+      - apt_cache_data:/var/cache/apt-cacher-ng
+      - ./mirrors/apt/acng.conf:/etc/apt-cacher-ng/acng.conf:ro
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+
+  devpi:
+    image: muccg/devpi:latest
+    container_name: devpi
+    networks:
+      - ci
+    volumes:
+      - devpi_data:/var/lib/devpi
+    environment:
+      - DEVPI_PASSWORD=${DEVPI_PASSWORD:-changeme}
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+
 volumes:
   postgres_data:
   rustfs_data:
@@ -612,3 +638,5 @@ volumes:
   loki_data:
   prometheus_data:
   grafana_data:
+  apt_cache_data:
+  devpi_data:

dev/scripts/backends/gitea.py

@@ -401,6 +401,65 @@ jobs:
     return (".gitea/workflows/infra-ci.yml", content)
 
 
+def _gen_pkg_supply_chain(
+    runner: str,
+    uv_version: str,
+    **_kw: object,
+) -> tuple[str, str]:
+    content = f"""\
+{_HEADER}
+name: Package Supply Chain
+
+on:
+  push:
+    paths:
+      - "**/Dockerfile*"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "docs/package.json"
+      - "docs/pnpm-lock.yaml"
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * *"
+
+jobs:
+  pkg-supply-chain:
+    runs-on: {runner}
+    steps:
+{_checkout_step()}
+
+{_setup_uv_step(uv_version)}
+
+      - name: Install dependencies
+        run: uv sync --no-dev
+
+      - name: Package inventory
+        run: uv run python dev/scripts/pkg_inventory.py
+
+      - name: Check manifest freshness
+        run: uv run python dev/scripts/pkg_inventory.py --check
+
+      - name: Sync mirrors
+        run: uv run python dev/scripts/pkg_mirror_sync.py
+
+      - name: Drift detection
+        run: uv run python dev/scripts/pkg_drift.py
+
+{_install_trivy_step()}
+
+      - name: Vulnerability scan
+        run: |
+          trivy fs --scanners vuln --format json -o data/pkg-vulns.json uv.lock || true
+          trivy fs --scanners vuln --format json -o data/pkg-vulns-pyproject.json pyproject.toml || true
+
+      - name: Auto-create issues for drift and vulns
+        env:
+          GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
+        run: uv run python dev/scripts/pkg_issues.py
+"""
+    return (".gitea/workflows/pkg-supply-chain.yml", content)
+
+
 def _gen_release(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
     content = f"""\
 {_HEADER}
@@ -464,6 +523,7 @@ def emit(
         _gen_rebuild_all,
         _gen_infra_ci,
         _gen_release,
+        _gen_pkg_supply_chain,
     ):
         path, content = gen_fn(**common)  # type: ignore[arg-type]
         files[path] = content

dev/scripts/pkg_issues.py

@@ -0,0 +1,306 @@
+"""Auto-create Gitea issues for missing packages and vulnerabilities.
+
+Reads drift report and vulnerability scan results, creates/closes
+Gitea issues via the API.
+
+Usage:
+    uv run python dev/scripts/pkg_issues.py                    # all checks
+    uv run python dev/scripts/pkg_issues.py --drift-only       # missing/surplus only
+    uv run python dev/scripts/pkg_issues.py --vuln-only        # CVEs only
+    uv run python dev/scripts/pkg_issues.py --dry-run          # show what would happen
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
+VULN_PATH = ROOT / "data" / "pkg-vulns.json"
+
+GITEA_URL = os.environ.get("GITEA_URL", "http://localhost:3000")
+GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "")
+REPO = os.environ.get("GITEA_REPO", "homelab/stack")
+MILESTONE_TITLE = "P21: Package Supply Chain"
+
+# Labels to attach (by name — resolved to IDs at runtime)
+LABEL_CI = "ci"
+LABEL_QUALITY = "quality"
+
+
+def _api(method: str, path: str, data: dict | None = None) -> dict | list | None:
+    """Call Gitea API. Uses docker exec if GITEA_TOKEN not in env."""
+    import subprocess
+
+    token = GITEA_TOKEN
+    if not token:
+        # Try to load from .env
+        env_file = ROOT / ".env"
+        if env_file.exists():
+            for line in env_file.read_text().splitlines():
+                if line.startswith("GITEA_TOKEN="):
+                    token = line.split("=", 1)[1].strip().strip('"').strip("'")
+                    break
+
+    if not token:
+        print("ERROR: GITEA_TOKEN not set and not found in .env")
+        raise SystemExit(2)
+
+    url = f"http://localhost:3000/api/v1/{path}"
+    cmd = [
+        "docker", "exec", "gitea", "curl", "-s",
+        "-H", f"Authorization: token {token}",
+        "-H", "Content-Type: application/json",
+    ]
+    if method == "POST":
+        cmd += ["-X", "POST", "-d", json.dumps(data), url]
+    elif method == "PATCH":
+        cmd += ["-X", "PATCH", "-d", json.dumps(data), url]
+    else:
+        cmd += [url]
+
+    result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+    if result.returncode != 0:
+        print(f"  API error: {result.stderr[:200]}")
+        return None
+    try:
+        return json.loads(result.stdout)
+    except json.JSONDecodeError:
+        return None
+
+
+def _get_milestone_id() -> int | None:
+    """Find the P21 milestone ID."""
+    milestones = _api("GET", f"repos/{REPO}/milestones?limit=50")
+    if not milestones:
+        return None
+    for m in milestones:
+        if MILESTONE_TITLE in m.get("title", ""):
+            return m["id"]
+    return None
+
+
+def _get_label_ids(names: list[str]) -> list[int]:
+    """Resolve label names to IDs."""
+    labels = _api("GET", f"repos/{REPO}/labels?limit=50")
+    if not labels:
+        return []
+    name_to_id = {lb["name"]: lb["id"] for lb in labels}
+    return [name_to_id[n] for n in names if n in name_to_id]
+
+
+def _find_existing_issues(prefix: str) -> dict[str, dict]:
+    """Find open issues with titles starting with prefix."""
+    issues = _api(
+        "GET",
+        f"repos/{REPO}/issues?state=open&type=issues&limit=50"
+        f"&labels={LABEL_CI}",
+    )
+    if not issues:
+        return {}
+    return {
+        i["title"]: i
+        for i in issues
+        if i["title"].startswith(prefix)
+    }
+
+
+def handle_drift(*, dry_run: bool = False) -> int:
+    """Create issues for drift (missing/untracked packages)."""
+    # Run drift detection inline
+    sys.path.insert(0, str(ROOT / "dev" / "scripts"))
+    from pkg_drift import check_drift
+
+    drift = check_drift()
+    if not drift["has_drift"]:
+        print("  drift: no drift detected")
+        return 0
+
+    milestone_id = _get_milestone_id()
+    label_ids = _get_label_ids([LABEL_CI])
+    existing = _find_existing_issues("[pkg-missing]")
+    created = 0
+
+    for pkg_type in ("apt", "apk"):
+        section = drift.get(pkg_type, {})
+        for pkg in section.get("untracked", []):
+            title = f"[pkg-missing] {pkg_type}/{pkg}"
+            if title in existing:
+                continue
+            body = (
+                f"Package `{pkg}` found in source ({pkg_type}) "
+                f"but not in the package manifest.\n\n"
+                f"Run `uv run python dev/scripts/pkg_inventory.py` to update."
+            )
+            if dry_run:
+                print(f"  would create: {title}")
+            else:
+                issue_data = {"title": title, "body": body, "labels": label_ids}
+                if milestone_id:
+                    issue_data["milestone"] = milestone_id
+                result = _api("POST", f"repos/{REPO}/issues", issue_data)
+                if result:
+                    print(f"  created #{result['number']}: {title}")
+                    created += 1
+
+    # PyPI sections
+    pypi_drift = drift.get("pypi", {})
+    for sub, d in pypi_drift.items():
+        for pkg in d.get("untracked", []):
+            title = f"[pkg-missing] pypi/{pkg} ({sub})"
+            if title in existing:
+                continue
+            body = (
+                f"PyPI package `{pkg}` found in source ({sub}) "
+                f"but not in the package manifest.\n\n"
+                f"Run `uv run python dev/scripts/pkg_inventory.py` to update."
+            )
+            if dry_run:
+                print(f"  would create: {title}")
+            else:
+                issue_data = {"title": title, "body": body, "labels": label_ids}
+                if milestone_id:
+                    issue_data["milestone"] = milestone_id
+                result = _api("POST", f"repos/{REPO}/issues", issue_data)
+                if result:
+                    print(f"  created #{result['number']}: {title}")
+                    created += 1
+
+    # Close issues for packages that are no longer missing
+    for title, issue in existing.items():
+        # Extract package name from title
+        m = re.match(r"\[pkg-missing\] (\w+)/(.+?)(?:\s|$)", title)
+        if not m:
+            continue
+        pkg_type, pkg_name = m.group(1), m.group(2)
+        still_missing = False
+        if pkg_type in ("apt", "apk"):
+            still_missing = pkg_name in drift.get(pkg_type, {}).get("untracked", [])
+        elif pkg_type == "pypi":
+            for sub_d in pypi_drift.values():
+                if pkg_name in sub_d.get("untracked", []):
+                    still_missing = True
+                    break
+        if not still_missing:
+            if dry_run:
+                print(f"  would close: {title}")
+            else:
+                _api(
+                    "PATCH",
+                    f"repos/{REPO}/issues/{issue['number']}",
+                    {"state": "closed"},
+                )
+                print(f"  closed #{issue['number']}: {title}")
+
+    return created
+
+
+def handle_vulns(*, dry_run: bool = False) -> int:
+    """Create issues for vulnerabilities found in package scans."""
+    if not VULN_PATH.exists():
+        print("  vulns: no scan results found (data/pkg-vulns.json)")
+        return 0
+
+    data = json.loads(VULN_PATH.read_text())
+
+    # trivy JSON format: {"Results": [{"Vulnerabilities": [...]}]}
+    vulns: list[dict] = []
+    for result in data.get("Results", []):
+        for v in result.get("Vulnerabilities", []):
+            vulns.append(v)
+
+    if not vulns:
+        print("  vulns: no vulnerabilities found")
+        return 0
+
+    milestone_id = _get_milestone_id()
+    label_ids = _get_label_ids([LABEL_CI, LABEL_QUALITY])
+    existing = _find_existing_issues("[pkg-vuln]")
+    created = 0
+
+    for v in vulns:
+        cve = v.get("VulnerabilityID", "UNKNOWN")
+        pkg = v.get("PkgName", "unknown")
+        version = v.get("InstalledVersion", "?")
+        severity = v.get("Severity", "UNKNOWN")
+        fixed = v.get("FixedVersion", "none")
+        desc = v.get("Description", "")[:500]
+        title = f"[pkg-vuln] {cve} in {pkg}@{version}"
+
+        if title in existing:
+            continue
+
+        body = (
+            f"**Severity:** {severity}\n"
+            f"**Package:** `{pkg}` @ `{version}`\n"
+            f"**Fixed in:** `{fixed}`\n\n"
+            f"{desc}\n\n"
+            f"**Reference:** https://nvd.nist.gov/vuln/detail/{cve}"
+        )
+
+        if dry_run:
+            print(f"  would create: {title}")
+        else:
+            issue_data = {"title": title, "body": body, "labels": label_ids}
+            if milestone_id:
+                issue_data["milestone"] = milestone_id
+            result = _api("POST", f"repos/{REPO}/issues", issue_data)
+            if result:
+                print(f"  created #{result['number']}: {title}")
+                created += 1
+
+    # Close issues for CVEs that are no longer present
+    active_cves = {
+        f"[pkg-vuln] {v.get('VulnerabilityID', '')} in "
+        f"{v.get('PkgName', '')}@{v.get('InstalledVersion', '')}"
+        for v in vulns
+    }
+    for title, issue in existing.items():
+        if title not in active_cves:
+            if dry_run:
+                print(f"  would close: {title}")
+            else:
+                _api(
+                    "PATCH",
+                    f"repos/{REPO}/issues/{issue['number']}",
+                    {"state": "closed"},
+                )
+                print(f"  closed #{issue['number']}: {title}")
+
+    return created
+
+
+def main() -> None:
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Auto-create Gitea issues for package drift and vulnerabilities"
+    )
+    parser.add_argument("--drift-only", action="store_true")
+    parser.add_argument("--vuln-only", action="store_true")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    do_drift = not args.vuln_only
+    do_vuln = not args.drift_only
+
+    total = 0
+    if do_drift:
+        print("Checking package drift...")
+        total += handle_drift(dry_run=args.dry_run)
+    if do_vuln:
+        print("Checking vulnerabilities...")
+        total += handle_vulns(dry_run=args.dry_run)
+
+    if total:
+        print(f"\nCreated {total} issue(s).")
+    else:
+        print("\nNo new issues to create.")
+
+
+if __name__ == "__main__":
+    main()

dev/scripts/pkg_mirror_sync.py

@@ -0,0 +1,225 @@
+"""Synchronize local package mirrors from the package manifest.
+
+Reads data/pkg-manifest.json and ensures apt-cache, apk-mirror, and
+devpi mirrors contain exactly the packages we need.
+
+Usage:
+    uv run python dev/scripts/pkg_mirror_sync.py              # sync all
+    uv run python dev/scripts/pkg_mirror_sync.py --type apt    # sync apt only
+    uv run python dev/scripts/pkg_mirror_sync.py --dry-run     # show what would change
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
+
+# Mirror config directories
+APT_MIRROR_DIR = ROOT / "mirrors" / "apt"
+APK_MIRROR_DIR = ROOT / "mirrors" / "apk"
+PYPI_MIRROR_DIR = ROOT / "mirrors" / "pypi"
+
+
+def load_manifest() -> dict:
+    if not MANIFEST_PATH.exists():
+        print(f"ERROR: {MANIFEST_PATH} not found. Run pkg_inventory.py first.")
+        raise SystemExit(2)
+    return json.loads(MANIFEST_PATH.read_text())
+
+
+def _flat_apt_packages(manifest: dict) -> list[str]:
+    pkgs: set[str] = set()
+    for pkg_list in manifest.get("apt", {}).values():
+        pkgs.update(pkg_list)
+    return sorted(pkgs)
+
+
+def _flat_apk_packages(manifest: dict) -> list[str]:
+    pkgs: set[str] = set()
+    for pkg_list in manifest.get("apk", {}).values():
+        pkgs.update(pkg_list)
+    return sorted(pkgs)
+
+
+def _flat_pypi_packages(manifest: dict) -> list[dict]:
+    seen: set[str] = set()
+    pkgs: list[dict] = []
+    for section in ("project_prod", "project_dev", "notebook", "dockerfile_adhoc"):
+        for pkg in manifest.get("pypi", {}).get(section, []):
+            if pkg["name"] not in seen:
+                seen.add(pkg["name"])
+                pkgs.append(pkg)
+    return sorted(pkgs, key=lambda p: p["name"])
+
+
+def sync_apt(manifest: dict, *, dry_run: bool = False) -> None:
+    """Generate apt-cacher-ng pre-seed package list."""
+    pkgs = _flat_apt_packages(manifest)
+    if not pkgs:
+        print("  apt: no packages to mirror")
+        return
+
+    APT_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
+    seed_file = APT_MIRROR_DIR / "package-list.txt"
+    content = "\n".join(pkgs) + "\n"
+
+    if dry_run:
+        print(f"  apt: would write {len(pkgs)} packages to {seed_file}")
+        for p in pkgs:
+            print(f"    - {p}")
+        return
+
+    seed_file.write_text(content)
+    print(f"  apt: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
+
+    # Generate apt-cacher-ng config for pre-seeding
+    config = APT_MIRROR_DIR / "acng.conf"
+    config.write_text(
+        "# apt-cacher-ng config — generated by pkg_mirror_sync.py\n"
+        "CacheDir: /var/cache/apt-cacher-ng\n"
+        "LogDir: /var/log/apt-cacher-ng\n"
+        "Port: 3142\n"
+        "PassThroughPattern: .*\n"
+    )
+    print(f"  apt: wrote config to {config.relative_to(ROOT)}")
+
+
+def sync_apk(manifest: dict, *, dry_run: bool = False) -> None:
+    """Generate apk mirror seed list."""
+    pkgs = _flat_apk_packages(manifest)
+
+    APK_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
+    seed_file = APK_MIRROR_DIR / "package-list.txt"
+
+    # Even if no explicit apk add packages, Alpine base images need
+    # their index for `apk upgrade`
+    content = "# Alpine packages to pre-cache (includes base for apk upgrade)\n"
+    if pkgs:
+        content += "\n".join(pkgs) + "\n"
+    else:
+        content += "# No explicit packages — mirror Alpine index only\n"
+
+    if dry_run:
+        print(f"  apk: would write {len(pkgs)} packages to {seed_file}")
+        return
+
+    seed_file.write_text(content)
+    print(f"  apk: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
+
+
+def sync_pypi(manifest: dict, *, dry_run: bool = False) -> None:
+    """Generate devpi pre-seed requirements file."""
+    pkgs = _flat_pypi_packages(manifest)
+    if not pkgs:
+        print("  pypi: no packages to mirror")
+        return
+
+    PYPI_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
+    seed_file = PYPI_MIRROR_DIR / "requirements.txt"
+
+    lines = [
+        "# PyPI packages to mirror — generated by pkg_mirror_sync.py",
+        "# Feed to: devpi-server --mirror, or pip download -r",
+    ]
+    for pkg in pkgs:
+        spec = pkg["name"]
+        if pkg.get("extras"):
+            spec += f"[{pkg['extras']}]"
+        if pkg.get("version"):
+            spec += pkg["version"]
+        lines.append(spec)
+
+    content = "\n".join(lines) + "\n"
+
+    if dry_run:
+        print(f"  pypi: would write {len(pkgs)} packages to {seed_file}")
+        for p in pkgs:
+            print(f"    - {p['name']}{p.get('version', '')}")
+        return
+
+    seed_file.write_text(content)
+    print(f"  pypi: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
+
+    # Generate devpi config
+    config = PYPI_MIRROR_DIR / "devpi.conf"
+    config.write_text(
+        "# devpi config — generated by pkg_mirror_sync.py\n"
+        "[devpi-server]\n"
+        "serverdir = /var/lib/devpi\n"
+        "port = 3141\n"
+        "host = 0.0.0.0\n"
+        "\n"
+        "[mirror]\n"
+        "type = mirror\n"
+        "url = https://pypi.org/simple/\n"
+    )
+    print(f"  pypi: wrote config to {config.relative_to(ROOT)}")
+
+
+def warm_devpi(manifest: dict, *, dry_run: bool = False) -> None:
+    """Warm devpi cache by downloading packages listed in the manifest.
+
+    Requires devpi to be running. Skips if devpi-server is unreachable.
+    """
+    seed_file = PYPI_MIRROR_DIR / "requirements.txt"
+    if not seed_file.exists():
+        return
+
+    if dry_run:
+        print("  pypi: would warm devpi cache from requirements.txt")
+        return
+
+    try:
+        result = subprocess.run(
+            [
+                sys.executable, "-m", "pip", "download",
+                "--no-deps", "--dest", str(PYPI_MIRROR_DIR / "cache"),
+                "-r", str(seed_file),
+            ],
+            capture_output=True,
+            text=True,
+            timeout=300,
+        )
+        if result.returncode == 0:
+            print("  pypi: warmed cache successfully")
+        else:
+            print(f"  pypi: cache warm failed (non-fatal): {result.stderr[:200]}")
+    except (FileNotFoundError, subprocess.TimeoutExpired) as e:
+        print(f"  pypi: cache warm skipped: {e}")
+
+
+def main() -> None:
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Sync package mirrors from manifest")
+    parser.add_argument(
+        "--type",
+        choices=["apt", "apk", "pypi", "all"],
+        default="all",
+        help="Which mirror type to sync",
+    )
+    parser.add_argument("--dry-run", action="store_true", help="Show what would change")
+    args = parser.parse_args()
+
+    manifest = load_manifest()
+    print("Syncing mirrors from pkg-manifest.json...")
+
+    if args.type in ("apt", "all"):
+        sync_apt(manifest, dry_run=args.dry_run)
+    if args.type in ("apk", "all"):
+        sync_apk(manifest, dry_run=args.dry_run)
+    if args.type in ("pypi", "all"):
+        sync_pypi(manifest, dry_run=args.dry_run)
+
+    if not args.dry_run:
+        print("\nMirror seed files generated. Start mirrors with:")
+        print("  docker compose up -d apt-cache devpi")
+
+
+if __name__ == "__main__":
+    main()

dev/scripts/test_network_isolation.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# test_network_isolation.sh — Verify Docker builds succeed without internet.
+#
+# Builds each Dockerfile with --network=none to prove all packages
+# come from local mirrors. Requires mirrors to be running first.
+#
+# Usage:
+#   bash dev/scripts/test_network_isolation.sh           # test all
+#   bash dev/scripts/test_network_isolation.sh api docs  # test specific images
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+MANIFEST="${ROOT}/data/pkg-manifest.json"
+
+if [ ! -f "$MANIFEST" ]; then
+    echo "ERROR: ${MANIFEST} not found. Run: uv run python dev/scripts/pkg_inventory.py"
+    exit 2
+fi
+
+# Image definitions — matches stack.toml [images]
+declare -A DOCKERFILES=(
+    [api]="api/Dockerfile"
+    [notebooks]="notebooks/Dockerfile"
+    [docs]="docs/Dockerfile"
+    [zotero]="zotero/Dockerfile"
+    [mc]="rustfs/Dockerfile.mc"
+)
+
+declare -A CONTEXTS=(
+    [api]="."
+    [notebooks]="notebooks/"
+    [docs]="."
+    [zotero]="zotero/"
+    [mc]="rustfs/"
+)
+
+# Select images to test
+if [ $# -gt 0 ]; then
+    IMAGES=("$@")
+else
+    IMAGES=("${!DOCKERFILES[@]}")
+fi
+
+PASS=0
+FAIL=0
+SKIP=0
+
+echo "=== Network Isolation Test ==="
+echo "Testing ${#IMAGES[@]} image(s) with --network=none"
+echo ""
+
+for img in "${IMAGES[@]}"; do
+    dockerfile="${DOCKERFILES[$img]:-}"
+    context="${CONTEXTS[$img]:-}"
+
+    if [ -z "$dockerfile" ]; then
+        echo "SKIP  $img (unknown image)"
+        ((SKIP++))
+        continue
+    fi
+
+    echo -n "TEST  $img ... "
+
+    # Build with no network — will fail if any RUN step needs internet
+    if docker build \
+        --network=none \
+        -f "${ROOT}/${dockerfile}" \
+        -t "isolation-test/${img}:test" \
+        "${ROOT}/${context}" \
+        > "/tmp/isolation-${img}.log" 2>&1; then
+        echo "PASS"
+        ((PASS++))
+        # Clean up test image
+        docker rmi "isolation-test/${img}:test" > /dev/null 2>&1 || true
+    else
+        echo "FAIL"
+        echo "  Build log: /tmp/isolation-${img}.log"
+        echo "  Last 5 lines:"
+        tail -5 "/tmp/isolation-${img}.log" | sed 's/^/    /'
+        ((FAIL++))
+    fi
+done
+
+echo ""
+echo "=== Results ==="
+echo "  Pass: ${PASS}  Fail: ${FAIL}  Skip: ${SKIP}"
+
+if [ "$FAIL" -gt 0 ]; then
+    echo ""
+    echo "FAILED — ${FAIL} image(s) require external network access."
+    echo "Ensure mirrors are running: docker compose up -d apt-cache devpi"
+    exit 1
+fi
+
+echo "All images build without external network access."
+exit 0

mirrors/apk/package-list.txt

@@ -0,0 +1,2 @@
+# Alpine packages to pre-cache (includes base for apk upgrade)
+# No explicit packages — mirror Alpine index only

mirrors/apt/acng.conf

@@ -0,0 +1,5 @@
+# apt-cacher-ng config — generated by pkg_mirror_sync.py
+CacheDir: /var/cache/apt-cacher-ng
+LogDir: /var/log/apt-cacher-ng
+Port: 3142
+PassThroughPattern: .*

mirrors/apt/package-list.txt

@@ -0,0 +1,5 @@
+build-essential
+ca-certificates
+curl
+git
+zotero

mirrors/pypi/devpi.conf

@@ -0,0 +1,9 @@
+# devpi config — generated by pkg_mirror_sync.py
+[devpi-server]
+serverdir = /var/lib/devpi
+port = 3141
+host = 0.0.0.0
+
+[mirror]
+type = mirror
+url = https://pypi.org/simple/

mirrors/pypi/requirements.txt

@@ -0,0 +1,41 @@
+# PyPI packages to mirror — generated by pkg_mirror_sync.py
+# Feed to: devpi-server --mirror, or pip download -r
+altair>=6.0.0
+coverage>=7.13.4
+cryptography>=46.0.5
+cudf-polars-cu12
+databricks-cli>=0.18.0
+databricks-sdk>=0.85.0
+dbt-core==1.10.15
+dbt-duckdb>=1.10,<1.11
+duckdb>=1.0.0
+fastapi>=0.135.1
+fastexcel>=0.19.0
+fsspec>=2024.1.0
+griffe
+httpx>=0.28.1
+marimo>=0.20.0
+narwhals>=2.17.0
+numpy
+obstore>=0.9.2
+openpyxl>=3.1.5
+pandas>=3.0.1
+pdfplumber>=0.11.9
+pillow>=12.1.1
+polars>=1.38.1
+pyarrow>=23.0.0
+pyasn1>=0.6.3
+pydantic
+pyiceberg[s3,pyarrow]>=0.7.0
+pyjwt>=2.12.0
+pytest>=9.0.2
+pytest-cov>=7.0.0
+pyzotero
+ruff>=0.11.0
+s3fs>=2026.2.0
+sqlglot>=26.0.0
+trino>=0.328.0
+typer>=0.24.1
+uv-build>=0.7,<1
+uvicorn>=0.41.0
+vega-datasets