fix(ci): finish gitea→git container rename in pkg scripts; skip gitignored Dockerfiles in inventory
Some checks failed
CI / lint (push) Successful in 1m2s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / mc (push) Has been skipped
Deploy / report (push) Successful in 17s
CI / test (push) Failing after 17m33s

pkg-supply-chain has failed on every run since the April service rename
(b361cd1 fixed the URLs but not the docker exec/cp container name), and
the script masked the real error: a failed 'docker exec gitea' subprocess
was reported as a bodyless HTTP 500 for all 39 wheel uploads.

- pkg_mirror_sync.py: docker exec/cp target is now GITEA_CONTAINER
  ("git") at every call site; pkg_issues.py: same rename at its one site
- pkg_inventory.py: drop gitignored paths (git check-ignore) from the
  Dockerfile scan so vendored checkouts like infra/marimo/src can't
  contaminate the manifest — that's where the unresolvable
  marimo==${marimo_version} entry in the committed manifest came from
- data/pkg-manifest.json: regenerated; now matches what CI generates

Verified: pkg_mirror_sync.py --type pypi uploads all 39 wheels
(36 uploaded, 3 already present, 0 failed).
This commit is contained in:
kert
2026-07-09 21:27:02 -04:00
parent e740bfac4a
commit eba1b77b8f
4 changed files with 40 additions and 39 deletions

View File

@@ -554,28 +554,7 @@
], ],
"project_dev": [], "project_dev": [],
"notebook": [], "notebook": [],
"dockerfile_adhoc": [ "dockerfile_adhoc": []
{
"name": "altair",
"version": "",
"extras": ""
},
{
"name": "marimo",
"version": "==${marimo_version}",
"extras": ""
},
{
"name": "numpy",
"version": "",
"extras": ""
},
{
"name": "pandas",
"version": "",
"extras": ""
}
]
}, },
"npm": [ "npm": [
{ {
@@ -644,11 +623,6 @@
], ],
"infra/images/selkies/Dockerfile.upstream": [ "infra/images/selkies/Dockerfile.upstream": [
"${DISTRIB_IMAGE}:${DISTRIB_RELEASE}" "${DISTRIB_IMAGE}:${DISTRIB_RELEASE}"
],
"infra/marimo/src/docker/Dockerfile": [
"python:3.13-slim",
"base",
"data"
] ]
} }
} }

View File

@@ -12,6 +12,7 @@ from __future__ import annotations
import json import json
import re import re
import subprocess
import sys import sys
from pathlib import Path from pathlib import Path
@@ -22,6 +23,29 @@ MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
SKIP_DIRS = {"node_modules", ".git", "__pycache__", "plugins"} SKIP_DIRS = {"node_modules", ".git", "__pycache__", "plugins"}
def _drop_gitignored(paths: list[Path]) -> list[Path]:
"""Filter out gitignored paths (e.g. vendored checkouts like
infra/marimo/src) so local manifests match CI, where ignored
files are absent from the checkout."""
if not paths:
return paths
try:
result = subprocess.run(
["git", "-C", str(ROOT), "check-ignore", "--stdin"],
input="\n".join(str(p) for p in paths),
capture_output=True,
text=True,
timeout=30,
)
except (OSError, subprocess.TimeoutExpired):
return paths
# exit 0: some ignored, 1: none ignored, 128: fatal (not a repo)
if result.returncode not in (0, 1):
return paths
ignored = set(result.stdout.splitlines())
return [p for p in paths if str(p) not in ignored]
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Dockerfile parsers # Dockerfile parsers
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -298,6 +322,7 @@ def scan() -> dict:
dockerfiles = [ dockerfiles = [
df for df in dockerfiles if not any(skip in df.parts for skip in SKIP_DIRS) df for df in dockerfiles if not any(skip in df.parts for skip in SKIP_DIRS)
] ]
dockerfiles = _drop_gitignored(dockerfiles)
for df in sorted(dockerfiles): for df in sorted(dockerfiles):
rel = str(df.relative_to(ROOT)) rel = str(df.relative_to(ROOT))
text = df.read_text() text = df.read_text()

View File

@@ -54,7 +54,7 @@ def _api(method: str, path: str, data: dict | None = None) -> dict | list | None
cmd = [ cmd = [
"docker", "docker",
"exec", "exec",
"gitea", "git",
"curl", "curl",
"-s", "-s",
"-H", "-H",

View File

@@ -26,6 +26,8 @@ CACHE_DIR = ROOT / "mirrors" / "cache"
GITEA_URL = "" GITEA_URL = ""
GITEA_TOKEN = "" GITEA_TOKEN = ""
GITEA_OWNER = "homelab" GITEA_OWNER = "homelab"
# Docker container name of the Gitea service (compose.yml container_name)
GITEA_CONTAINER = "git"
def _load_env() -> None: def _load_env() -> None:
@@ -61,7 +63,7 @@ def _api_via_docker(method: str, path: str, file_path: str = "") -> tuple[int, s
cmd = [ cmd = [
"docker", "docker",
"exec", "exec",
"gitea", GITEA_CONTAINER,
"curl", "curl",
"-s", "-s",
"-w", "-w",
@@ -73,7 +75,7 @@ def _api_via_docker(method: str, path: str, file_path: str = "") -> tuple[int, s
# For file uploads, we need to copy the file into the container first # For file uploads, we need to copy the file into the container first
tmp_name = f"/tmp/pkg_upload_{os.path.basename(file_path)}" tmp_name = f"/tmp/pkg_upload_{os.path.basename(file_path)}"
subprocess.run( subprocess.run(
["docker", "cp", file_path, f"gitea:{tmp_name}"], ["docker", "cp", file_path, f"{GITEA_CONTAINER}:{tmp_name}"],
capture_output=True, capture_output=True,
timeout=60, timeout=60,
) )
@@ -230,7 +232,7 @@ def sync_pypi(manifest: dict, *, dry_run: bool = False) -> dict:
# Copy file into gitea container and upload # Copy file into gitea container and upload
tmp_name = f"/tmp/pkg_{dist.name}" tmp_name = f"/tmp/pkg_{dist.name}"
subprocess.run( subprocess.run(
["docker", "cp", str(dist), f"gitea:{tmp_name}"], ["docker", "cp", str(dist), f"{GITEA_CONTAINER}:{tmp_name}"],
capture_output=True, capture_output=True,
timeout=60, timeout=60,
) )
@@ -238,7 +240,7 @@ def sync_pypi(manifest: dict, *, dry_run: bool = False) -> dict:
[ [
"docker", "docker",
"exec", "exec",
"gitea", GITEA_CONTAINER,
"curl", "curl",
"-s", "-s",
"-w", "-w",
@@ -274,7 +276,7 @@ def sync_pypi(manifest: dict, *, dry_run: bool = False) -> dict:
stats["failed"].append(dist.name) stats["failed"].append(dist.name)
# Cleanup # Cleanup
subprocess.run( subprocess.run(
["docker", "exec", "gitea", "rm", "-f", tmp_name], ["docker", "exec", GITEA_CONTAINER, "rm", "-f", tmp_name],
capture_output=True, capture_output=True,
timeout=10, timeout=10,
) )
@@ -343,7 +345,7 @@ def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
for deb in sorted(deb_files): for deb in sorted(deb_files):
tmp_name = f"/tmp/pkg_{deb.name}" tmp_name = f"/tmp/pkg_{deb.name}"
subprocess.run( subprocess.run(
["docker", "cp", str(deb), f"gitea:{tmp_name}"], ["docker", "cp", str(deb), f"{GITEA_CONTAINER}:{tmp_name}"],
capture_output=True, capture_output=True,
timeout=60, timeout=60,
) )
@@ -351,7 +353,7 @@ def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
[ [
"docker", "docker",
"exec", "exec",
"gitea", GITEA_CONTAINER,
"curl", "curl",
"-s", "-s",
"-w", "-w",
@@ -381,7 +383,7 @@ def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
print(f" FAILED ({status}): {deb.name}{body[:120]}") print(f" FAILED ({status}): {deb.name}{body[:120]}")
stats["failed"].append(deb.name) stats["failed"].append(deb.name)
subprocess.run( subprocess.run(
["docker", "exec", "gitea", "rm", "-f", tmp_name], ["docker", "exec", GITEA_CONTAINER, "rm", "-f", tmp_name],
capture_output=True, capture_output=True,
timeout=10, timeout=10,
) )
@@ -500,7 +502,7 @@ def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
for apk in sorted(apk_files): for apk in sorted(apk_files):
tmp_name = f"/tmp/pkg_{apk.name}" tmp_name = f"/tmp/pkg_{apk.name}"
subprocess.run( subprocess.run(
["docker", "cp", str(apk), f"gitea:{tmp_name}"], ["docker", "cp", str(apk), f"{GITEA_CONTAINER}:{tmp_name}"],
capture_output=True, capture_output=True,
timeout=60, timeout=60,
) )
@@ -508,7 +510,7 @@ def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
[ [
"docker", "docker",
"exec", "exec",
"gitea", GITEA_CONTAINER,
"curl", "curl",
"-s", "-s",
"-w", "-w",
@@ -538,7 +540,7 @@ def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
print(f" FAILED ({status}): {apk.name}{body[:120]}") print(f" FAILED ({status}): {apk.name}{body[:120]}")
stats["failed"].append(apk.name) stats["failed"].append(apk.name)
subprocess.run( subprocess.run(
["docker", "exec", "gitea", "rm", "-f", tmp_name], ["docker", "exec", GITEA_CONTAINER, "rm", "-f", tmp_name],
capture_output=True, capture_output=True,
timeout=10, timeout=10,
) )