add package mirrors, auto-issue scripts, network isolation test, CI workflow
refs #160 #161 #162 #163 #164 #166 #167 #168 - compose.yml: add apt-cacher-ng and devpi services on ci network - pkg_mirror_sync.py: generates seed files for apt/apk/pypi mirrors - pkg_issues.py: auto-creates Gitea issues for drift and CVEs, deduplicates, auto-closes resolved issues - test_network_isolation.sh: builds all Dockerfiles with --network=none - gitea.py: add pkg-supply-chain workflow (daily + on Dockerfile/dep changes) - mirrors/: seed files for apt (5 pkgs), apk (index), pypi (39 pkgs)
This commit is contained in:
kert
56
.gitea/workflows/pkg-supply-chain.yml
Normal file
56
.gitea/workflows/pkg-supply-chain.yml
Normal file
@@ -0,0 +1,56 @@
|
||||
# DO NOT EDIT — generated by gen_config.py from stack.toml
|
||||
# Re-generate: uv run python dev/scripts/gen_config.py
|
||||
|
||||
name: Package Supply Chain
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "**/Dockerfile*"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
- "docs/package.json"
|
||||
- "docs/pnpm-lock.yaml"
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 6 * * *"
|
||||
|
||||
jobs:
|
||||
pkg-supply-chain:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- name: Set up uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
env:
|
||||
UV_INSTALL_DIR: /usr/local/bin
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --no-dev
|
||||
|
||||
- name: Package inventory
|
||||
run: uv run python dev/scripts/pkg_inventory.py
|
||||
|
||||
- name: Check manifest freshness
|
||||
run: uv run python dev/scripts/pkg_inventory.py --check
|
||||
|
||||
- name: Sync mirrors
|
||||
run: uv run python dev/scripts/pkg_mirror_sync.py
|
||||
|
||||
- name: Drift detection
|
||||
run: uv run python dev/scripts/pkg_drift.py
|
||||
|
||||
- name: Install trivy
|
||||
run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
|
||||
|
||||
- name: Vulnerability scan
|
||||
run: |
|
||||
trivy fs --scanners vuln --format json -o data/pkg-vulns.json uv.lock || true
|
||||
trivy fs --scanners vuln --format json -o data/pkg-vulns-pyproject.json pyproject.toml || true
|
||||
|
||||
- name: Auto-create issues for drift and vulns
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
run: uv run python dev/scripts/pkg_issues.py
|
||||
28
compose.yml
28
compose.yml
@@ -600,6 +600,32 @@ services:
|
||||
- no-new-privileges:true
|
||||
restart: unless-stopped
|
||||
|
||||
# Package mirrors — local caches for reproducible builds
|
||||
apt-cache:
|
||||
image: sameersbn/apt-cacher-ng:latest
|
||||
container_name: apt-cache
|
||||
networks:
|
||||
- ci
|
||||
volumes:
|
||||
- apt_cache_data:/var/cache/apt-cacher-ng
|
||||
- ./mirrors/apt/acng.conf:/etc/apt-cacher-ng/acng.conf:ro
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
restart: unless-stopped
|
||||
|
||||
devpi:
|
||||
image: muccg/devpi:latest
|
||||
container_name: devpi
|
||||
networks:
|
||||
- ci
|
||||
volumes:
|
||||
- devpi_data:/var/lib/devpi
|
||||
environment:
|
||||
- DEVPI_PASSWORD=${DEVPI_PASSWORD:-changeme}
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
rustfs_data:
|
||||
@@ -612,3 +638,5 @@ volumes:
|
||||
loki_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
apt_cache_data:
|
||||
devpi_data:
|
||||
|
||||
@@ -401,6 +401,65 @@ jobs:
|
||||
return (".gitea/workflows/infra-ci.yml", content)
|
||||
|
||||
|
||||
def _gen_pkg_supply_chain(
|
||||
runner: str,
|
||||
uv_version: str,
|
||||
**_kw: object,
|
||||
) -> tuple[str, str]:
|
||||
content = f"""\
|
||||
{_HEADER}
|
||||
name: Package Supply Chain
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "**/Dockerfile*"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
- "docs/package.json"
|
||||
- "docs/pnpm-lock.yaml"
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 6 * * *"
|
||||
|
||||
jobs:
|
||||
pkg-supply-chain:
|
||||
runs-on: {runner}
|
||||
steps:
|
||||
{_checkout_step()}
|
||||
|
||||
{_setup_uv_step(uv_version)}
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --no-dev
|
||||
|
||||
- name: Package inventory
|
||||
run: uv run python dev/scripts/pkg_inventory.py
|
||||
|
||||
- name: Check manifest freshness
|
||||
run: uv run python dev/scripts/pkg_inventory.py --check
|
||||
|
||||
- name: Sync mirrors
|
||||
run: uv run python dev/scripts/pkg_mirror_sync.py
|
||||
|
||||
- name: Drift detection
|
||||
run: uv run python dev/scripts/pkg_drift.py
|
||||
|
||||
{_install_trivy_step()}
|
||||
|
||||
- name: Vulnerability scan
|
||||
run: |
|
||||
trivy fs --scanners vuln --format json -o data/pkg-vulns.json uv.lock || true
|
||||
trivy fs --scanners vuln --format json -o data/pkg-vulns-pyproject.json pyproject.toml || true
|
||||
|
||||
- name: Auto-create issues for drift and vulns
|
||||
env:
|
||||
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
|
||||
run: uv run python dev/scripts/pkg_issues.py
|
||||
"""
|
||||
return (".gitea/workflows/pkg-supply-chain.yml", content)
|
||||
|
||||
|
||||
def _gen_release(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
|
||||
content = f"""\
|
||||
{_HEADER}
|
||||
@@ -464,6 +523,7 @@ def emit(
|
||||
_gen_rebuild_all,
|
||||
_gen_infra_ci,
|
||||
_gen_release,
|
||||
_gen_pkg_supply_chain,
|
||||
):
|
||||
path, content = gen_fn(**common) # type: ignore[arg-type]
|
||||
files[path] = content
|
||||
|
||||
306
dev/scripts/pkg_issues.py
Normal file
306
dev/scripts/pkg_issues.py
Normal file
@@ -0,0 +1,306 @@
|
||||
"""Auto-create Gitea issues for missing packages and vulnerabilities.
|
||||
|
||||
Reads drift report and vulnerability scan results, creates/closes
|
||||
Gitea issues via the API.
|
||||
|
||||
Usage:
|
||||
uv run python dev/scripts/pkg_issues.py # all checks
|
||||
uv run python dev/scripts/pkg_issues.py --drift-only # missing/surplus only
|
||||
uv run python dev/scripts/pkg_issues.py --vuln-only # CVEs only
|
||||
uv run python dev/scripts/pkg_issues.py --dry-run # show what would happen
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
|
||||
VULN_PATH = ROOT / "data" / "pkg-vulns.json"
|
||||
|
||||
GITEA_URL = os.environ.get("GITEA_URL", "http://localhost:3000")
|
||||
GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "")
|
||||
REPO = os.environ.get("GITEA_REPO", "homelab/stack")
|
||||
MILESTONE_TITLE = "P21: Package Supply Chain"
|
||||
|
||||
# Labels to attach (by name — resolved to IDs at runtime)
|
||||
LABEL_CI = "ci"
|
||||
LABEL_QUALITY = "quality"
|
||||
|
||||
|
||||
def _api(method: str, path: str, data: dict | None = None) -> dict | list | None:
|
||||
"""Call Gitea API. Uses docker exec if GITEA_TOKEN not in env."""
|
||||
import subprocess
|
||||
|
||||
token = GITEA_TOKEN
|
||||
if not token:
|
||||
# Try to load from .env
|
||||
env_file = ROOT / ".env"
|
||||
if env_file.exists():
|
||||
for line in env_file.read_text().splitlines():
|
||||
if line.startswith("GITEA_TOKEN="):
|
||||
token = line.split("=", 1)[1].strip().strip('"').strip("'")
|
||||
break
|
||||
|
||||
if not token:
|
||||
print("ERROR: GITEA_TOKEN not set and not found in .env")
|
||||
raise SystemExit(2)
|
||||
|
||||
url = f"http://localhost:3000/api/v1/{path}"
|
||||
cmd = [
|
||||
"docker", "exec", "gitea", "curl", "-s",
|
||||
"-H", f"Authorization: token {token}",
|
||||
"-H", "Content-Type: application/json",
|
||||
]
|
||||
if method == "POST":
|
||||
cmd += ["-X", "POST", "-d", json.dumps(data), url]
|
||||
elif method == "PATCH":
|
||||
cmd += ["-X", "PATCH", "-d", json.dumps(data), url]
|
||||
else:
|
||||
cmd += [url]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
if result.returncode != 0:
|
||||
print(f" API error: {result.stderr[:200]}")
|
||||
return None
|
||||
try:
|
||||
return json.loads(result.stdout)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _get_milestone_id() -> int | None:
|
||||
"""Find the P21 milestone ID."""
|
||||
milestones = _api("GET", f"repos/{REPO}/milestones?limit=50")
|
||||
if not milestones:
|
||||
return None
|
||||
for m in milestones:
|
||||
if MILESTONE_TITLE in m.get("title", ""):
|
||||
return m["id"]
|
||||
return None
|
||||
|
||||
|
||||
def _get_label_ids(names: list[str]) -> list[int]:
|
||||
"""Resolve label names to IDs."""
|
||||
labels = _api("GET", f"repos/{REPO}/labels?limit=50")
|
||||
if not labels:
|
||||
return []
|
||||
name_to_id = {lb["name"]: lb["id"] for lb in labels}
|
||||
return [name_to_id[n] for n in names if n in name_to_id]
|
||||
|
||||
|
||||
def _find_existing_issues(prefix: str) -> dict[str, dict]:
|
||||
"""Find open issues with titles starting with prefix."""
|
||||
issues = _api(
|
||||
"GET",
|
||||
f"repos/{REPO}/issues?state=open&type=issues&limit=50"
|
||||
f"&labels={LABEL_CI}",
|
||||
)
|
||||
if not issues:
|
||||
return {}
|
||||
return {
|
||||
i["title"]: i
|
||||
for i in issues
|
||||
if i["title"].startswith(prefix)
|
||||
}
|
||||
|
||||
|
||||
def handle_drift(*, dry_run: bool = False) -> int:
|
||||
"""Create issues for drift (missing/untracked packages)."""
|
||||
# Run drift detection inline
|
||||
sys.path.insert(0, str(ROOT / "dev" / "scripts"))
|
||||
from pkg_drift import check_drift
|
||||
|
||||
drift = check_drift()
|
||||
if not drift["has_drift"]:
|
||||
print(" drift: no drift detected")
|
||||
return 0
|
||||
|
||||
milestone_id = _get_milestone_id()
|
||||
label_ids = _get_label_ids([LABEL_CI])
|
||||
existing = _find_existing_issues("[pkg-missing]")
|
||||
created = 0
|
||||
|
||||
for pkg_type in ("apt", "apk"):
|
||||
section = drift.get(pkg_type, {})
|
||||
for pkg in section.get("untracked", []):
|
||||
title = f"[pkg-missing] {pkg_type}/{pkg}"
|
||||
if title in existing:
|
||||
continue
|
||||
body = (
|
||||
f"Package `{pkg}` found in source ({pkg_type}) "
|
||||
f"but not in the package manifest.\n\n"
|
||||
f"Run `uv run python dev/scripts/pkg_inventory.py` to update."
|
||||
)
|
||||
if dry_run:
|
||||
print(f" would create: {title}")
|
||||
else:
|
||||
issue_data = {"title": title, "body": body, "labels": label_ids}
|
||||
if milestone_id:
|
||||
issue_data["milestone"] = milestone_id
|
||||
result = _api("POST", f"repos/{REPO}/issues", issue_data)
|
||||
if result:
|
||||
print(f" created #{result['number']}: {title}")
|
||||
created += 1
|
||||
|
||||
# PyPI sections
|
||||
pypi_drift = drift.get("pypi", {})
|
||||
for sub, d in pypi_drift.items():
|
||||
for pkg in d.get("untracked", []):
|
||||
title = f"[pkg-missing] pypi/{pkg} ({sub})"
|
||||
if title in existing:
|
||||
continue
|
||||
body = (
|
||||
f"PyPI package `{pkg}` found in source ({sub}) "
|
||||
f"but not in the package manifest.\n\n"
|
||||
f"Run `uv run python dev/scripts/pkg_inventory.py` to update."
|
||||
)
|
||||
if dry_run:
|
||||
print(f" would create: {title}")
|
||||
else:
|
||||
issue_data = {"title": title, "body": body, "labels": label_ids}
|
||||
if milestone_id:
|
||||
issue_data["milestone"] = milestone_id
|
||||
result = _api("POST", f"repos/{REPO}/issues", issue_data)
|
||||
if result:
|
||||
print(f" created #{result['number']}: {title}")
|
||||
created += 1
|
||||
|
||||
# Close issues for packages that are no longer missing
|
||||
for title, issue in existing.items():
|
||||
# Extract package name from title
|
||||
m = re.match(r"\[pkg-missing\] (\w+)/(.+?)(?:\s|$)", title)
|
||||
if not m:
|
||||
continue
|
||||
pkg_type, pkg_name = m.group(1), m.group(2)
|
||||
still_missing = False
|
||||
if pkg_type in ("apt", "apk"):
|
||||
still_missing = pkg_name in drift.get(pkg_type, {}).get("untracked", [])
|
||||
elif pkg_type == "pypi":
|
||||
for sub_d in pypi_drift.values():
|
||||
if pkg_name in sub_d.get("untracked", []):
|
||||
still_missing = True
|
||||
break
|
||||
if not still_missing:
|
||||
if dry_run:
|
||||
print(f" would close: {title}")
|
||||
else:
|
||||
_api(
|
||||
"PATCH",
|
||||
f"repos/{REPO}/issues/{issue['number']}",
|
||||
{"state": "closed"},
|
||||
)
|
||||
print(f" closed #{issue['number']}: {title}")
|
||||
|
||||
return created
|
||||
|
||||
|
||||
def handle_vulns(*, dry_run: bool = False) -> int:
|
||||
"""Create issues for vulnerabilities found in package scans."""
|
||||
if not VULN_PATH.exists():
|
||||
print(" vulns: no scan results found (data/pkg-vulns.json)")
|
||||
return 0
|
||||
|
||||
data = json.loads(VULN_PATH.read_text())
|
||||
|
||||
# trivy JSON format: {"Results": [{"Vulnerabilities": [...]}]}
|
||||
vulns: list[dict] = []
|
||||
for result in data.get("Results", []):
|
||||
for v in result.get("Vulnerabilities", []):
|
||||
vulns.append(v)
|
||||
|
||||
if not vulns:
|
||||
print(" vulns: no vulnerabilities found")
|
||||
return 0
|
||||
|
||||
milestone_id = _get_milestone_id()
|
||||
label_ids = _get_label_ids([LABEL_CI, LABEL_QUALITY])
|
||||
existing = _find_existing_issues("[pkg-vuln]")
|
||||
created = 0
|
||||
|
||||
for v in vulns:
|
||||
cve = v.get("VulnerabilityID", "UNKNOWN")
|
||||
pkg = v.get("PkgName", "unknown")
|
||||
version = v.get("InstalledVersion", "?")
|
||||
severity = v.get("Severity", "UNKNOWN")
|
||||
fixed = v.get("FixedVersion", "none")
|
||||
desc = v.get("Description", "")[:500]
|
||||
title = f"[pkg-vuln] {cve} in {pkg}@{version}"
|
||||
|
||||
if title in existing:
|
||||
continue
|
||||
|
||||
body = (
|
||||
f"**Severity:** {severity}\n"
|
||||
f"**Package:** `{pkg}` @ `{version}`\n"
|
||||
f"**Fixed in:** `{fixed}`\n\n"
|
||||
f"{desc}\n\n"
|
||||
f"**Reference:** https://nvd.nist.gov/vuln/detail/{cve}"
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
print(f" would create: {title}")
|
||||
else:
|
||||
issue_data = {"title": title, "body": body, "labels": label_ids}
|
||||
if milestone_id:
|
||||
issue_data["milestone"] = milestone_id
|
||||
result = _api("POST", f"repos/{REPO}/issues", issue_data)
|
||||
if result:
|
||||
print(f" created #{result['number']}: {title}")
|
||||
created += 1
|
||||
|
||||
# Close issues for CVEs that are no longer present
|
||||
active_cves = {
|
||||
f"[pkg-vuln] {v.get('VulnerabilityID', '')} in "
|
||||
f"{v.get('PkgName', '')}@{v.get('InstalledVersion', '')}"
|
||||
for v in vulns
|
||||
}
|
||||
for title, issue in existing.items():
|
||||
if title not in active_cves:
|
||||
if dry_run:
|
||||
print(f" would close: {title}")
|
||||
else:
|
||||
_api(
|
||||
"PATCH",
|
||||
f"repos/{REPO}/issues/{issue['number']}",
|
||||
{"state": "closed"},
|
||||
)
|
||||
print(f" closed #{issue['number']}: {title}")
|
||||
|
||||
return created
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Auto-create Gitea issues for package drift and vulnerabilities"
|
||||
)
|
||||
parser.add_argument("--drift-only", action="store_true")
|
||||
parser.add_argument("--vuln-only", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
do_drift = not args.vuln_only
|
||||
do_vuln = not args.drift_only
|
||||
|
||||
total = 0
|
||||
if do_drift:
|
||||
print("Checking package drift...")
|
||||
total += handle_drift(dry_run=args.dry_run)
|
||||
if do_vuln:
|
||||
print("Checking vulnerabilities...")
|
||||
total += handle_vulns(dry_run=args.dry_run)
|
||||
|
||||
if total:
|
||||
print(f"\nCreated {total} issue(s).")
|
||||
else:
|
||||
print("\nNo new issues to create.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
225
dev/scripts/pkg_mirror_sync.py
Normal file
225
dev/scripts/pkg_mirror_sync.py
Normal file
@@ -0,0 +1,225 @@
|
||||
"""Synchronize local package mirrors from the package manifest.
|
||||
|
||||
Reads data/pkg-manifest.json and ensures apt-cache, apk-mirror, and
|
||||
devpi mirrors contain exactly the packages we need.
|
||||
|
||||
Usage:
|
||||
uv run python dev/scripts/pkg_mirror_sync.py # sync all
|
||||
uv run python dev/scripts/pkg_mirror_sync.py --type apt # sync apt only
|
||||
uv run python dev/scripts/pkg_mirror_sync.py --dry-run # show what would change
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
|
||||
|
||||
# Mirror config directories
|
||||
APT_MIRROR_DIR = ROOT / "mirrors" / "apt"
|
||||
APK_MIRROR_DIR = ROOT / "mirrors" / "apk"
|
||||
PYPI_MIRROR_DIR = ROOT / "mirrors" / "pypi"
|
||||
|
||||
|
||||
def load_manifest() -> dict:
|
||||
if not MANIFEST_PATH.exists():
|
||||
print(f"ERROR: {MANIFEST_PATH} not found. Run pkg_inventory.py first.")
|
||||
raise SystemExit(2)
|
||||
return json.loads(MANIFEST_PATH.read_text())
|
||||
|
||||
|
||||
def _flat_apt_packages(manifest: dict) -> list[str]:
|
||||
pkgs: set[str] = set()
|
||||
for pkg_list in manifest.get("apt", {}).values():
|
||||
pkgs.update(pkg_list)
|
||||
return sorted(pkgs)
|
||||
|
||||
|
||||
def _flat_apk_packages(manifest: dict) -> list[str]:
|
||||
pkgs: set[str] = set()
|
||||
for pkg_list in manifest.get("apk", {}).values():
|
||||
pkgs.update(pkg_list)
|
||||
return sorted(pkgs)
|
||||
|
||||
|
||||
def _flat_pypi_packages(manifest: dict) -> list[dict]:
|
||||
seen: set[str] = set()
|
||||
pkgs: list[dict] = []
|
||||
for section in ("project_prod", "project_dev", "notebook", "dockerfile_adhoc"):
|
||||
for pkg in manifest.get("pypi", {}).get(section, []):
|
||||
if pkg["name"] not in seen:
|
||||
seen.add(pkg["name"])
|
||||
pkgs.append(pkg)
|
||||
return sorted(pkgs, key=lambda p: p["name"])
|
||||
|
||||
|
||||
def sync_apt(manifest: dict, *, dry_run: bool = False) -> None:
|
||||
"""Generate apt-cacher-ng pre-seed package list."""
|
||||
pkgs = _flat_apt_packages(manifest)
|
||||
if not pkgs:
|
||||
print(" apt: no packages to mirror")
|
||||
return
|
||||
|
||||
APT_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
seed_file = APT_MIRROR_DIR / "package-list.txt"
|
||||
content = "\n".join(pkgs) + "\n"
|
||||
|
||||
if dry_run:
|
||||
print(f" apt: would write {len(pkgs)} packages to {seed_file}")
|
||||
for p in pkgs:
|
||||
print(f" - {p}")
|
||||
return
|
||||
|
||||
seed_file.write_text(content)
|
||||
print(f" apt: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
|
||||
|
||||
# Generate apt-cacher-ng config for pre-seeding
|
||||
config = APT_MIRROR_DIR / "acng.conf"
|
||||
config.write_text(
|
||||
"# apt-cacher-ng config — generated by pkg_mirror_sync.py\n"
|
||||
"CacheDir: /var/cache/apt-cacher-ng\n"
|
||||
"LogDir: /var/log/apt-cacher-ng\n"
|
||||
"Port: 3142\n"
|
||||
"PassThroughPattern: .*\n"
|
||||
)
|
||||
print(f" apt: wrote config to {config.relative_to(ROOT)}")
|
||||
|
||||
|
||||
def sync_apk(manifest: dict, *, dry_run: bool = False) -> None:
|
||||
"""Generate apk mirror seed list."""
|
||||
pkgs = _flat_apk_packages(manifest)
|
||||
|
||||
APK_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
seed_file = APK_MIRROR_DIR / "package-list.txt"
|
||||
|
||||
# Even if no explicit apk add packages, Alpine base images need
|
||||
# their index for `apk upgrade`
|
||||
content = "# Alpine packages to pre-cache (includes base for apk upgrade)\n"
|
||||
if pkgs:
|
||||
content += "\n".join(pkgs) + "\n"
|
||||
else:
|
||||
content += "# No explicit packages — mirror Alpine index only\n"
|
||||
|
||||
if dry_run:
|
||||
print(f" apk: would write {len(pkgs)} packages to {seed_file}")
|
||||
return
|
||||
|
||||
seed_file.write_text(content)
|
||||
print(f" apk: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
|
||||
|
||||
|
||||
def sync_pypi(manifest: dict, *, dry_run: bool = False) -> None:
|
||||
"""Generate devpi pre-seed requirements file."""
|
||||
pkgs = _flat_pypi_packages(manifest)
|
||||
if not pkgs:
|
||||
print(" pypi: no packages to mirror")
|
||||
return
|
||||
|
||||
PYPI_MIRROR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
seed_file = PYPI_MIRROR_DIR / "requirements.txt"
|
||||
|
||||
lines = [
|
||||
"# PyPI packages to mirror — generated by pkg_mirror_sync.py",
|
||||
"# Feed to: devpi-server --mirror, or pip download -r",
|
||||
]
|
||||
for pkg in pkgs:
|
||||
spec = pkg["name"]
|
||||
if pkg.get("extras"):
|
||||
spec += f"[{pkg['extras']}]"
|
||||
if pkg.get("version"):
|
||||
spec += pkg["version"]
|
||||
lines.append(spec)
|
||||
|
||||
content = "\n".join(lines) + "\n"
|
||||
|
||||
if dry_run:
|
||||
print(f" pypi: would write {len(pkgs)} packages to {seed_file}")
|
||||
for p in pkgs:
|
||||
print(f" - {p['name']}{p.get('version', '')}")
|
||||
return
|
||||
|
||||
seed_file.write_text(content)
|
||||
print(f" pypi: wrote {len(pkgs)} packages to {seed_file.relative_to(ROOT)}")
|
||||
|
||||
# Generate devpi config
|
||||
config = PYPI_MIRROR_DIR / "devpi.conf"
|
||||
config.write_text(
|
||||
"# devpi config — generated by pkg_mirror_sync.py\n"
|
||||
"[devpi-server]\n"
|
||||
"serverdir = /var/lib/devpi\n"
|
||||
"port = 3141\n"
|
||||
"host = 0.0.0.0\n"
|
||||
"\n"
|
||||
"[mirror]\n"
|
||||
"type = mirror\n"
|
||||
"url = https://pypi.org/simple/\n"
|
||||
)
|
||||
print(f" pypi: wrote config to {config.relative_to(ROOT)}")
|
||||
|
||||
|
||||
def warm_devpi(manifest: dict, *, dry_run: bool = False) -> None:
|
||||
"""Warm devpi cache by downloading packages listed in the manifest.
|
||||
|
||||
Requires devpi to be running. Skips if devpi-server is unreachable.
|
||||
"""
|
||||
seed_file = PYPI_MIRROR_DIR / "requirements.txt"
|
||||
if not seed_file.exists():
|
||||
return
|
||||
|
||||
if dry_run:
|
||||
print(" pypi: would warm devpi cache from requirements.txt")
|
||||
return
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable, "-m", "pip", "download",
|
||||
"--no-deps", "--dest", str(PYPI_MIRROR_DIR / "cache"),
|
||||
"-r", str(seed_file),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(" pypi: warmed cache successfully")
|
||||
else:
|
||||
print(f" pypi: cache warm failed (non-fatal): {result.stderr[:200]}")
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired) as e:
|
||||
print(f" pypi: cache warm skipped: {e}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Sync package mirrors from manifest")
|
||||
parser.add_argument(
|
||||
"--type",
|
||||
choices=["apt", "apk", "pypi", "all"],
|
||||
default="all",
|
||||
help="Which mirror type to sync",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show what would change")
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest = load_manifest()
|
||||
print("Syncing mirrors from pkg-manifest.json...")
|
||||
|
||||
if args.type in ("apt", "all"):
|
||||
sync_apt(manifest, dry_run=args.dry_run)
|
||||
if args.type in ("apk", "all"):
|
||||
sync_apk(manifest, dry_run=args.dry_run)
|
||||
if args.type in ("pypi", "all"):
|
||||
sync_pypi(manifest, dry_run=args.dry_run)
|
||||
|
||||
if not args.dry_run:
|
||||
print("\nMirror seed files generated. Start mirrors with:")
|
||||
print(" docker compose up -d apt-cache devpi")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
97
dev/scripts/test_network_isolation.sh
Normal file
97
dev/scripts/test_network_isolation.sh
Normal file
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env bash
|
||||
# test_network_isolation.sh — Verify Docker builds succeed without internet.
|
||||
#
|
||||
# Builds each Dockerfile with --network=none to prove all packages
|
||||
# come from local mirrors. Requires mirrors to be running first.
|
||||
#
|
||||
# Usage:
|
||||
# bash dev/scripts/test_network_isolation.sh # test all
|
||||
# bash dev/scripts/test_network_isolation.sh api docs # test specific images
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
MANIFEST="${ROOT}/data/pkg-manifest.json"
|
||||
|
||||
if [ ! -f "$MANIFEST" ]; then
|
||||
echo "ERROR: ${MANIFEST} not found. Run: uv run python dev/scripts/pkg_inventory.py"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Image definitions — matches stack.toml [images]
|
||||
declare -A DOCKERFILES=(
|
||||
[api]="api/Dockerfile"
|
||||
[notebooks]="notebooks/Dockerfile"
|
||||
[docs]="docs/Dockerfile"
|
||||
[zotero]="zotero/Dockerfile"
|
||||
[mc]="rustfs/Dockerfile.mc"
|
||||
)
|
||||
|
||||
declare -A CONTEXTS=(
|
||||
[api]="."
|
||||
[notebooks]="notebooks/"
|
||||
[docs]="."
|
||||
[zotero]="zotero/"
|
||||
[mc]="rustfs/"
|
||||
)
|
||||
|
||||
# Select images to test
|
||||
if [ $# -gt 0 ]; then
|
||||
IMAGES=("$@")
|
||||
else
|
||||
IMAGES=("${!DOCKERFILES[@]}")
|
||||
fi
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
SKIP=0
|
||||
|
||||
echo "=== Network Isolation Test ==="
|
||||
echo "Testing ${#IMAGES[@]} image(s) with --network=none"
|
||||
echo ""
|
||||
|
||||
for img in "${IMAGES[@]}"; do
|
||||
dockerfile="${DOCKERFILES[$img]:-}"
|
||||
context="${CONTEXTS[$img]:-}"
|
||||
|
||||
if [ -z "$dockerfile" ]; then
|
||||
echo "SKIP $img (unknown image)"
|
||||
((SKIP++))
|
||||
continue
|
||||
fi
|
||||
|
||||
echo -n "TEST $img ... "
|
||||
|
||||
# Build with no network — will fail if any RUN step needs internet
|
||||
if docker build \
|
||||
--network=none \
|
||||
-f "${ROOT}/${dockerfile}" \
|
||||
-t "isolation-test/${img}:test" \
|
||||
"${ROOT}/${context}" \
|
||||
> "/tmp/isolation-${img}.log" 2>&1; then
|
||||
echo "PASS"
|
||||
((PASS++))
|
||||
# Clean up test image
|
||||
docker rmi "isolation-test/${img}:test" > /dev/null 2>&1 || true
|
||||
else
|
||||
echo "FAIL"
|
||||
echo " Build log: /tmp/isolation-${img}.log"
|
||||
echo " Last 5 lines:"
|
||||
tail -5 "/tmp/isolation-${img}.log" | sed 's/^/ /'
|
||||
((FAIL++))
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Results ==="
|
||||
echo " Pass: ${PASS} Fail: ${FAIL} Skip: ${SKIP}"
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "FAILED — ${FAIL} image(s) require external network access."
|
||||
echo "Ensure mirrors are running: docker compose up -d apt-cache devpi"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All images build without external network access."
|
||||
exit 0
|
||||
2
mirrors/apk/package-list.txt
Normal file
2
mirrors/apk/package-list.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
# Alpine packages to pre-cache (includes base for apk upgrade)
|
||||
# No explicit packages — mirror Alpine index only
|
||||
5
mirrors/apt/acng.conf
Normal file
5
mirrors/apt/acng.conf
Normal file
@@ -0,0 +1,5 @@
|
||||
# apt-cacher-ng config — generated by pkg_mirror_sync.py
|
||||
CacheDir: /var/cache/apt-cacher-ng
|
||||
LogDir: /var/log/apt-cacher-ng
|
||||
Port: 3142
|
||||
PassThroughPattern: .*
|
||||
5
mirrors/apt/package-list.txt
Normal file
5
mirrors/apt/package-list.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
build-essential
|
||||
ca-certificates
|
||||
curl
|
||||
git
|
||||
zotero
|
||||
9
mirrors/pypi/devpi.conf
Normal file
9
mirrors/pypi/devpi.conf
Normal file
@@ -0,0 +1,9 @@
|
||||
# devpi config — generated by pkg_mirror_sync.py
|
||||
[devpi-server]
|
||||
serverdir = /var/lib/devpi
|
||||
port = 3141
|
||||
host = 0.0.0.0
|
||||
|
||||
[mirror]
|
||||
type = mirror
|
||||
url = https://pypi.org/simple/
|
||||
41
mirrors/pypi/requirements.txt
Normal file
41
mirrors/pypi/requirements.txt
Normal file
@@ -0,0 +1,41 @@
|
||||
# PyPI packages to mirror — generated by pkg_mirror_sync.py
|
||||
# Feed to: devpi-server --mirror, or pip download -r
|
||||
altair>=6.0.0
|
||||
coverage>=7.13.4
|
||||
cryptography>=46.0.5
|
||||
cudf-polars-cu12
|
||||
databricks-cli>=0.18.0
|
||||
databricks-sdk>=0.85.0
|
||||
dbt-core==1.10.15
|
||||
dbt-duckdb>=1.10,<1.11
|
||||
duckdb>=1.0.0
|
||||
fastapi>=0.135.1
|
||||
fastexcel>=0.19.0
|
||||
fsspec>=2024.1.0
|
||||
griffe
|
||||
httpx>=0.28.1
|
||||
marimo>=0.20.0
|
||||
narwhals>=2.17.0
|
||||
numpy
|
||||
obstore>=0.9.2
|
||||
openpyxl>=3.1.5
|
||||
pandas>=3.0.1
|
||||
pdfplumber>=0.11.9
|
||||
pillow>=12.1.1
|
||||
polars>=1.38.1
|
||||
pyarrow>=23.0.0
|
||||
pyasn1>=0.6.3
|
||||
pydantic
|
||||
pyiceberg[s3,pyarrow]>=0.7.0
|
||||
pyjwt>=2.12.0
|
||||
pytest>=9.0.2
|
||||
pytest-cov>=7.0.0
|
||||
pyzotero
|
||||
ruff>=0.11.0
|
||||
s3fs>=2026.2.0
|
||||
sqlglot>=26.0.0
|
||||
trino>=0.328.0
|
||||
typer>=0.24.1
|
||||
uv-build>=0.7,<1
|
||||
uvicorn>=0.41.0
|
||||
vega-datasets
|
||||
Reference in New Issue
Block a user