fix(ci): finish gitea→git container rename in pkg scripts; skip gitignored Dockerfiles in inventory
Some checks failed
CI / lint (push) Successful in 1m2s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / mc (push) Has been skipped
Deploy / report (push) Successful in 17s
CI / test (push) Failing after 17m33s
Some checks failed
CI / lint (push) Successful in 1m2s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / mc (push) Has been skipped
Deploy / report (push) Successful in 17s
CI / test (push) Failing after 17m33s
pkg-supply-chain has failed on every run since the April service rename
(b361cd1 fixed the URLs but not the docker exec/cp container name), and
the script masked the real error: a failed 'docker exec gitea' subprocess
was reported as a bodyless HTTP 500 for all 39 wheel uploads.
- pkg_mirror_sync.py: docker exec/cp target is now GITEA_CONTAINER
("git") at every call site; pkg_issues.py: same rename at its one site
- pkg_inventory.py: drop gitignored paths (git check-ignore) from the
Dockerfile scan so vendored checkouts like infra/marimo/src can't
contaminate the manifest — that's where the unresolvable
marimo==${marimo_version} entry in the committed manifest came from
- data/pkg-manifest.json: regenerated; now matches what CI generates
Verified: pkg_mirror_sync.py --type pypi uploads all 39 wheels
(36 uploaded, 3 already present, 0 failed).
This commit is contained in:
@@ -554,28 +554,7 @@
|
|||||||
],
|
],
|
||||||
"project_dev": [],
|
"project_dev": [],
|
||||||
"notebook": [],
|
"notebook": [],
|
||||||
"dockerfile_adhoc": [
|
"dockerfile_adhoc": []
|
||||||
{
|
|
||||||
"name": "altair",
|
|
||||||
"version": "",
|
|
||||||
"extras": ""
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "marimo",
|
|
||||||
"version": "==${marimo_version}",
|
|
||||||
"extras": ""
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "numpy",
|
|
||||||
"version": "",
|
|
||||||
"extras": ""
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "pandas",
|
|
||||||
"version": "",
|
|
||||||
"extras": ""
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"npm": [
|
"npm": [
|
||||||
{
|
{
|
||||||
@@ -644,11 +623,6 @@
|
|||||||
],
|
],
|
||||||
"infra/images/selkies/Dockerfile.upstream": [
|
"infra/images/selkies/Dockerfile.upstream": [
|
||||||
"${DISTRIB_IMAGE}:${DISTRIB_RELEASE}"
|
"${DISTRIB_IMAGE}:${DISTRIB_RELEASE}"
|
||||||
],
|
|
||||||
"infra/marimo/src/docker/Dockerfile": [
|
|
||||||
"python:3.13-slim",
|
|
||||||
"base",
|
|
||||||
"data"
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -22,6 +23,29 @@ MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
|
|||||||
SKIP_DIRS = {"node_modules", ".git", "__pycache__", "plugins"}
|
SKIP_DIRS = {"node_modules", ".git", "__pycache__", "plugins"}
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_gitignored(paths: list[Path]) -> list[Path]:
|
||||||
|
"""Filter out gitignored paths (e.g. vendored checkouts like
|
||||||
|
infra/marimo/src) so local manifests match CI, where ignored
|
||||||
|
files are absent from the checkout."""
|
||||||
|
if not paths:
|
||||||
|
return paths
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "-C", str(ROOT), "check-ignore", "--stdin"],
|
||||||
|
input="\n".join(str(p) for p in paths),
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
return paths
|
||||||
|
# exit 0: some ignored, 1: none ignored, 128: fatal (not a repo)
|
||||||
|
if result.returncode not in (0, 1):
|
||||||
|
return paths
|
||||||
|
ignored = set(result.stdout.splitlines())
|
||||||
|
return [p for p in paths if str(p) not in ignored]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Dockerfile parsers
|
# Dockerfile parsers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -298,6 +322,7 @@ def scan() -> dict:
|
|||||||
dockerfiles = [
|
dockerfiles = [
|
||||||
df for df in dockerfiles if not any(skip in df.parts for skip in SKIP_DIRS)
|
df for df in dockerfiles if not any(skip in df.parts for skip in SKIP_DIRS)
|
||||||
]
|
]
|
||||||
|
dockerfiles = _drop_gitignored(dockerfiles)
|
||||||
for df in sorted(dockerfiles):
|
for df in sorted(dockerfiles):
|
||||||
rel = str(df.relative_to(ROOT))
|
rel = str(df.relative_to(ROOT))
|
||||||
text = df.read_text()
|
text = df.read_text()
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ def _api(method: str, path: str, data: dict | None = None) -> dict | list | None
|
|||||||
cmd = [
|
cmd = [
|
||||||
"docker",
|
"docker",
|
||||||
"exec",
|
"exec",
|
||||||
"gitea",
|
"git",
|
||||||
"curl",
|
"curl",
|
||||||
"-s",
|
"-s",
|
||||||
"-H",
|
"-H",
|
||||||
|
|||||||
@@ -26,6 +26,8 @@ CACHE_DIR = ROOT / "mirrors" / "cache"
|
|||||||
GITEA_URL = ""
|
GITEA_URL = ""
|
||||||
GITEA_TOKEN = ""
|
GITEA_TOKEN = ""
|
||||||
GITEA_OWNER = "homelab"
|
GITEA_OWNER = "homelab"
|
||||||
|
# Docker container name of the Gitea service (compose.yml container_name)
|
||||||
|
GITEA_CONTAINER = "git"
|
||||||
|
|
||||||
|
|
||||||
def _load_env() -> None:
|
def _load_env() -> None:
|
||||||
@@ -61,7 +63,7 @@ def _api_via_docker(method: str, path: str, file_path: str = "") -> tuple[int, s
|
|||||||
cmd = [
|
cmd = [
|
||||||
"docker",
|
"docker",
|
||||||
"exec",
|
"exec",
|
||||||
"gitea",
|
GITEA_CONTAINER,
|
||||||
"curl",
|
"curl",
|
||||||
"-s",
|
"-s",
|
||||||
"-w",
|
"-w",
|
||||||
@@ -73,7 +75,7 @@ def _api_via_docker(method: str, path: str, file_path: str = "") -> tuple[int, s
|
|||||||
# For file uploads, we need to copy the file into the container first
|
# For file uploads, we need to copy the file into the container first
|
||||||
tmp_name = f"/tmp/pkg_upload_{os.path.basename(file_path)}"
|
tmp_name = f"/tmp/pkg_upload_{os.path.basename(file_path)}"
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "cp", file_path, f"gitea:{tmp_name}"],
|
["docker", "cp", file_path, f"{GITEA_CONTAINER}:{tmp_name}"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
timeout=60,
|
timeout=60,
|
||||||
)
|
)
|
||||||
@@ -230,7 +232,7 @@ def sync_pypi(manifest: dict, *, dry_run: bool = False) -> dict:
|
|||||||
# Copy file into gitea container and upload
|
# Copy file into gitea container and upload
|
||||||
tmp_name = f"/tmp/pkg_{dist.name}"
|
tmp_name = f"/tmp/pkg_{dist.name}"
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "cp", str(dist), f"gitea:{tmp_name}"],
|
["docker", "cp", str(dist), f"{GITEA_CONTAINER}:{tmp_name}"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
timeout=60,
|
timeout=60,
|
||||||
)
|
)
|
||||||
@@ -238,7 +240,7 @@ def sync_pypi(manifest: dict, *, dry_run: bool = False) -> dict:
|
|||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
"exec",
|
"exec",
|
||||||
"gitea",
|
GITEA_CONTAINER,
|
||||||
"curl",
|
"curl",
|
||||||
"-s",
|
"-s",
|
||||||
"-w",
|
"-w",
|
||||||
@@ -274,7 +276,7 @@ def sync_pypi(manifest: dict, *, dry_run: bool = False) -> dict:
|
|||||||
stats["failed"].append(dist.name)
|
stats["failed"].append(dist.name)
|
||||||
# Cleanup
|
# Cleanup
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "exec", "gitea", "rm", "-f", tmp_name],
|
["docker", "exec", GITEA_CONTAINER, "rm", "-f", tmp_name],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
)
|
)
|
||||||
@@ -343,7 +345,7 @@ def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
|
|||||||
for deb in sorted(deb_files):
|
for deb in sorted(deb_files):
|
||||||
tmp_name = f"/tmp/pkg_{deb.name}"
|
tmp_name = f"/tmp/pkg_{deb.name}"
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "cp", str(deb), f"gitea:{tmp_name}"],
|
["docker", "cp", str(deb), f"{GITEA_CONTAINER}:{tmp_name}"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
timeout=60,
|
timeout=60,
|
||||||
)
|
)
|
||||||
@@ -351,7 +353,7 @@ def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
|
|||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
"exec",
|
"exec",
|
||||||
"gitea",
|
GITEA_CONTAINER,
|
||||||
"curl",
|
"curl",
|
||||||
"-s",
|
"-s",
|
||||||
"-w",
|
"-w",
|
||||||
@@ -381,7 +383,7 @@ def sync_apt(manifest: dict, *, dry_run: bool = False) -> dict:
|
|||||||
print(f" FAILED ({status}): {deb.name} — {body[:120]}")
|
print(f" FAILED ({status}): {deb.name} — {body[:120]}")
|
||||||
stats["failed"].append(deb.name)
|
stats["failed"].append(deb.name)
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "exec", "gitea", "rm", "-f", tmp_name],
|
["docker", "exec", GITEA_CONTAINER, "rm", "-f", tmp_name],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
)
|
)
|
||||||
@@ -500,7 +502,7 @@ def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
|
|||||||
for apk in sorted(apk_files):
|
for apk in sorted(apk_files):
|
||||||
tmp_name = f"/tmp/pkg_{apk.name}"
|
tmp_name = f"/tmp/pkg_{apk.name}"
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "cp", str(apk), f"gitea:{tmp_name}"],
|
["docker", "cp", str(apk), f"{GITEA_CONTAINER}:{tmp_name}"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
timeout=60,
|
timeout=60,
|
||||||
)
|
)
|
||||||
@@ -508,7 +510,7 @@ def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
|
|||||||
[
|
[
|
||||||
"docker",
|
"docker",
|
||||||
"exec",
|
"exec",
|
||||||
"gitea",
|
GITEA_CONTAINER,
|
||||||
"curl",
|
"curl",
|
||||||
"-s",
|
"-s",
|
||||||
"-w",
|
"-w",
|
||||||
@@ -538,7 +540,7 @@ def sync_apk(manifest: dict, *, dry_run: bool = False) -> dict:
|
|||||||
print(f" FAILED ({status}): {apk.name} — {body[:120]}")
|
print(f" FAILED ({status}): {apk.name} — {body[:120]}")
|
||||||
stats["failed"].append(apk.name)
|
stats["failed"].append(apk.name)
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "exec", "gitea", "rm", "-f", tmp_name],
|
["docker", "exec", GITEA_CONTAINER, "rm", "-f", tmp_name],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user