fix(ci): pkg_inventory parser drops inline comments cleanly
Some checks failed
CI / lint (push) Failing after 33s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / mc (push) Has been skipped
Deploy / report (push) Successful in 11s
CI / test (push) Successful in 25m55s

The pyproject dep parser did chained `.strip().strip(",").strip('"')`
which left trailing junk on lines with inline comments. The line:
    "pymupdf>=1.24",     # AGPL-3.0 — flag if shipping outside …
became the spec `pymupdf>=1.24",` — pip rejected it and the
pkg-supply-chain workflow failed on every push (#400-403).

Replace the four duplicated chained-strip blocks with a single
`_extract_dep_specs` helper that pulls the first quoted string from
each non-comment line. Same call from prod/dev/optional/build-system.

Manifest pypi count: 62 → 66 (parser was previously dropping a few
specs entirely, not just mangling pymupdf).

Add tests/dev/test_pkg_inventory.py with 4 regression tests covering
inline comments, blank/comment lines, single-quoted strings, and a
full pyproject excerpt with the bug pattern.

Closes the noise from #400-403.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
kert
2026-04-23 18:07:07 -04:00
parent 49f004a466
commit 7445d097fd
4 changed files with 465 additions and 23 deletions

View File

@@ -1,6 +1,234 @@
{ {
"apt": {}, "apt": {
"apk": {}, "infra/images/selkies/Dockerfile.upstream": [
"\"./virtualgl32_${VIRTUALGL_VERSION}_amd64.deb\"",
"\"./virtualgl_${VIRTUALGL_VERSION}_amd64.deb\"",
"./google-chrome-stable.deb",
"./heroic_launcher.deb",
"./kasmvncserver.deb",
"./lutris.deb",
"./rustdesk.deb",
"./selkies-js-interposer.deb",
"./virtualgl_${VIRTUALGL_VERSION}_arm64.deb",
"GStreamer",
"Operating",
"adwaita-icon-theme-full",
"aom-tools",
"appmenu-gtk3-module",
"apt-utils",
"ark",
"aspell",
"aspell-en",
"breeze",
"breeze-cursor-theme",
"breeze-gtk-theme",
"breeze-icon-theme",
"dbus-user-session",
"dbus-x11",
"debconf-kde-helper",
"dependencies",
"desktop-file-utils",
"dolphin",
"dolphin-plugins",
"enchant-2",
"fakeroot",
"fcitx",
"fcitx-frontend-gtk2",
"fcitx-frontend-gtk3",
"fcitx-frontend-qt5",
"fcitx-hangul",
"fcitx-libpinyin",
"fcitx-m17n",
"fcitx-module-dbus",
"fcitx-module-kimpanel",
"fcitx-module-lua",
"fcitx-module-x11",
"fcitx-mozc",
"fcitx-sayura",
"fcitx-tools",
"fcitx-unikey",
"filelight",
"firefox",
"frameworkintegration",
"fuse",
"gir1.2-wp-0.5",
"gstreamer1.0-libcamera",
"gstreamer1.0-pipewire",
"gstreamer1.0-plugins-bad",
"gwenview",
"haveged",
"hunspell",
"i965-va-driver-shaders",
"i965-va-driver-shaders:i386",
"im-config",
"intel-gpu-tools",
"intel-media-va-driver-non-free",
"intel-media-va-driver-non-free:i386",
"kcalc",
"kcharselect",
"kde-baseapps",
"kde-config-fcitx",
"kde-config-gtk-style",
"kde-config-gtk-style-preview",
"kde-spectacle",
"kdeadmin",
"kdeconnect",
"kdegraphics-thumbnailers",
"kdf",
"kdialog",
"kfind",
"kget",
"khotkeys",
"kimageformat-plugins",
"kinfocenter",
"kio",
"kio-extras",
"kmag",
"kmenuedit",
"kmix",
"kmod",
"kmousetool",
"kmouth",
"ksshaskpass",
"ktimer",
"kwin-addons",
"kwin-x11",
"kwrite",
"libc6:i386",
"libdatetime-perl",
"libdbusmenu-glib4",
"libdbusmenu-gtk3-4",
"libdrm2:i386",
"libegl-dev",
"libegl1:i386",
"libffmpeg-nvenc-dev",
"libgail-common",
"libgdk-pixbuf2.0-bin",
"libgl1:i386",
"libgles1:i386",
"libgles2:i386",
"libglu1:i386",
"libglvnd0:i386",
"libglx0:i386",
"libgstreamer-plugins-bad1.0-dev",
"libgtk-3-bin",
"libgtk2.0-bin",
"libkf5baloowidgets-bin",
"libkf5dbusaddons-bin",
"libkf5iconthemes-bin",
"libkf5kdelibs4support5-bin",
"libkf5khtml-bin",
"libkf5parts-plugins",
"libopengl0:i386",
"libopenh264-dev",
"libpipewire-0.3-modules",
"libpipewire-module-x11-bell",
"libqt5multimedia5-plugins",
"libreoffice",
"libreoffice-kf5",
"libreoffice-plasma",
"libreoffice-style-breeze",
"librsvg2-common",
"libsm6:i386",
"libspa-0.2-bluetooth",
"libspa-0.2-jack",
"libspa-0.2-modules",
"libva-dev",
"libva2:i386",
"libvulkan-dev:i386",
"libx11-6:i386",
"libxau6:i386",
"libxcb1:i386",
"libxdmcp6:i386",
"libxext6:i386",
"libxtst6:i386",
"libxv1:i386",
"locales",
"media-player-info",
"mesa-utils-extra",
"mesa-vulkan-drivers:i386",
"meson",
"nvtop",
"okular",
"okular-extra-backends",
"packages",
"pipewire",
"pipewire-alsa",
"pipewire-audio-client-libraries",
"pipewire-jack",
"pipewire-libcamera",
"pipewire-locales",
"pipewire-v4l2",
"pipewire-vulkan",
"plasma-browser-integration",
"plasma-calendar-addons",
"plasma-dataengines-addons",
"plasma-desktop",
"plasma-discover",
"plasma-integration",
"plasma-runners-addons",
"plasma-widgets-addons",
"plasma-workspace",
"playonlinux",
"print-manager",
"q4wine",
"qapt-deb-installer",
"qml-module-org-kde-qqc2desktopstyle",
"qml-module-org-kde-runnermodel",
"qml-module-qt-labs-platform",
"qml-module-qtgraphicaleffects",
"qml-module-qtquick-xmllistmodel",
"qt5-gtk-platformtheme",
"qt5-image-formats-plugins",
"qt5-style-plugins",
"qtspeech5-flite-plugin",
"qtvirtualkeyboard-plugin",
"software-properties-qt",
"sonnet-plugins",
"ssl-cert",
"sudo",
"svt-av1",
"sweeper",
"system",
"systemsettings",
"transmission-qt",
"tzdata",
"ubuntu-drivers-common",
"udev",
"va-driver-all",
"va-driver-all:i386",
"vdpau-driver-all:i386",
"vlc",
"vlc-plugin-access-extra",
"vlc-plugin-notify",
"vlc-plugin-samba",
"vlc-plugin-skins2",
"vlc-plugin-video-splitter",
"vlc-plugin-visualization",
"winehq-${WINE_BRANCH}",
"wireplumber",
"wireplumber-locales",
"xcvt",
"xdg-user-dirs",
"xdg-utils",
"xvfb"
]
},
"apk": {
"infra/gitea/Dockerfile": [
"bash",
"build-base",
"ca-certificates",
"curl",
"dumb-init",
"gettext",
"git",
"gnupg",
"nodejs",
"npm",
"openssh-keygen"
]
},
"pypi": { "pypi": {
"project_prod": [ "project_prod": [
{ {
@@ -8,6 +236,21 @@
"version": ">=2.0.0", "version": ">=2.0.0",
"extras": "" "extras": ""
}, },
{
"name": "pymupdf",
"version": ">=1.24",
"extras": ""
},
{
"name": "python-docx",
"version": ">=1.1",
"extras": ""
},
{
"name": "xlrd",
"version": ">=2.0.2",
"extras": ""
},
{ {
"name": "pydantic", "name": "pydantic",
"version": ">=2.0.0", "version": ">=2.0.0",
@@ -65,7 +308,7 @@
}, },
{ {
"name": "cryptography", "name": "cryptography",
"version": ">=46.0.5", "version": ">=46.0.7",
"extras": "" "extras": ""
}, },
{ {
@@ -108,6 +351,21 @@
"version": ">=0.41.0", "version": ">=0.41.0",
"extras": "" "extras": ""
}, },
{
"name": "httpx",
"version": ">=0.28.1",
"extras": ""
},
{
"name": "pydo",
"version": ">=0.29.0",
"extras": ""
},
{
"name": "resend",
"version": ">=2.0.0",
"extras": ""
},
{ {
"name": "narwhals", "name": "narwhals",
"version": ">=2.17.0", "version": ">=2.17.0",
@@ -128,6 +386,31 @@
"version": ">=2.17.0", "version": ">=2.17.0",
"extras": "" "extras": ""
}, },
{
"name": "duckdb",
"version": ">=1.0.0",
"extras": ""
},
{
"name": "narwhals",
"version": ">=2.17.0",
"extras": ""
},
{
"name": "duckdb",
"version": ">=1.0.0",
"extras": ""
},
{
"name": "narwhals",
"version": ">=2.17.0",
"extras": ""
},
{
"name": "typer",
"version": ">=0.24.1",
"extras": ""
},
{ {
"name": "narwhals", "name": "narwhals",
"version": ">=2.17.0", "version": ">=2.17.0",
@@ -143,6 +426,36 @@
"version": ">=2024.1.0", "version": ">=2024.1.0",
"extras": "" "extras": ""
}, },
{
"name": "anthropic",
"version": ">=0.40.0",
"extras": ""
},
{
"name": "httpx",
"version": ">=0.28.1",
"extras": "socks"
},
{
"name": "pyyaml",
"version": ">=6.0.0",
"extras": ""
},
{
"name": "pydo",
"version": ">=0.29.0",
"extras": ""
},
{
"name": "pdfminer",
"version": ".six>=20221105",
"extras": ""
},
{
"name": "resend",
"version": ">=2.0.0",
"extras": ""
},
{ {
"name": "opentelemetry-api", "name": "opentelemetry-api",
"version": ">=1.25.0", "version": ">=1.25.0",
@@ -179,8 +492,8 @@
"extras": "" "extras": ""
}, },
{ {
"name": "databricks-cli", "name": "coverage",
"version": ">=0.18.0", "version": ">=7.13.4",
"extras": "" "extras": ""
}, },
{ {
@@ -188,6 +501,11 @@
"version": ">=0.85.0", "version": ">=0.85.0",
"extras": "" "extras": ""
}, },
{
"name": "databricks-bundles",
"version": ">=0.295.0",
"extras": ""
},
{ {
"name": "boto3", "name": "boto3",
"version": ">=1.35.0", "version": ">=1.35.0",
@@ -226,7 +544,28 @@
], ],
"project_dev": [], "project_dev": [],
"notebook": [], "notebook": [],
"dockerfile_adhoc": [] "dockerfile_adhoc": [
{
"name": "altair",
"version": "",
"extras": ""
},
{
"name": "marimo",
"version": "==${marimo_version}",
"extras": ""
},
{
"name": "numpy",
"version": "",
"extras": ""
},
{
"name": "pandas",
"version": "",
"extras": ""
}
]
}, },
"npm": [ "npm": [
{ {
@@ -288,5 +627,18 @@
"version": "latest" "version": "latest"
} }
], ],
"base_images": {} "base_images": {
"infra/gitea/Dockerfile": [
"docker.io/library/golang:1.25-alpine3.22",
"docker.io/library/alpine:3.22"
],
"infra/images/selkies/Dockerfile.upstream": [
"${DISTRIB_IMAGE}:${DISTRIB_RELEASE}"
],
"infra/marimo/src/docker/Dockerfile": [
"python:3.13-slim",
"base",
"data"
]
}
} }

View File

@@ -164,6 +164,25 @@ def _is_self_ref(spec: str) -> bool:
return name == "stack" return name == "stack"
def _extract_dep_specs(block: str) -> list[str]:
"""Extract dependency spec strings from a TOML array body.
Pulls the first quoted string from each non-comment line. This
correctly skips inline comments (e.g. ``"pymupdf>=1.24", # AGPL``),
trailing commas, and surrounding whitespace — the previous chained
``.strip()`` approach broke on inline comments.
"""
specs: list[str] = []
for line in block.splitlines():
s = line.strip()
if not s or s.startswith("#"):
continue
m = re.match(r'["\']([^"\']+)["\']', s)
if m:
specs.append(m.group(1).strip())
return specs
def _parse_pyproject_deps(text: str) -> tuple[list[dict], list[dict]]: def _parse_pyproject_deps(text: str) -> tuple[list[dict], list[dict]]:
"""Parse dependencies from pyproject.toml without a TOML library. """Parse dependencies from pyproject.toml without a TOML library.
@@ -181,10 +200,9 @@ def _parse_pyproject_deps(text: str) -> tuple[list[dict], list[dict]]:
# prod deps — [project] dependencies # prod deps — [project] dependencies
m = re.search(r"^dependencies\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL) m = re.search(r"^dependencies\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL)
if m: if m:
for line in m.group(1).splitlines(): for spec in _extract_dep_specs(m.group(1)):
line = line.strip().strip(",").strip('"').strip("'") if not _is_self_ref(spec):
if line and not line.startswith("#") and not _is_self_ref(line): prod.append(_parse_pypi_spec(spec))
prod.append(_parse_pypi_spec(line))
# optional deps — [project.optional-dependencies] all sections # optional deps — [project.optional-dependencies] all sections
# Extract the section content, then parse each "name = [...]" block. # Extract the section content, then parse each "name = [...]" block.
@@ -214,18 +232,16 @@ def _parse_pyproject_deps(text: str) -> tuple[list[dict], list[dict]]:
depth -= 1 depth -= 1
i += 1 i += 1
block = rest[: i - 1] if depth == 0 else rest block = rest[: i - 1] if depth == 0 else rest
for line in block.splitlines(): for spec in _extract_dep_specs(block):
line = line.strip().strip(",").strip('"').strip("'") if not _is_self_ref(spec):
if line and not line.startswith("#") and not _is_self_ref(line): prod.append(_parse_pypi_spec(spec))
prod.append(_parse_pypi_spec(line))
# dev deps — [dependency-groups] dev # dev deps — [dependency-groups] dev
m = re.search(r"dev\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL) m = re.search(r"dev\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL)
if m: if m:
for line in m.group(1).splitlines(): for spec in _extract_dep_specs(m.group(1)):
line = line.strip().strip(",").strip('"').strip("'") if not _is_self_ref(spec):
if line and not line.startswith("#") and not _is_self_ref(line): dev.append(_parse_pypi_spec(spec))
dev.append(_parse_pypi_spec(line))
# build-system requires # build-system requires
m = re.search( m = re.search(
@@ -234,10 +250,9 @@ def _parse_pyproject_deps(text: str) -> tuple[list[dict], list[dict]]:
re.MULTILINE | re.DOTALL, re.MULTILINE | re.DOTALL,
) )
if m: if m:
for line in m.group(1).splitlines(): for spec in _extract_dep_specs(m.group(1)):
line = line.strip().strip(",").strip('"').strip("'") if not _is_self_ref(spec):
if line and not line.startswith("#") and not _is_self_ref(line): prod.append(_parse_pypi_spec(spec))
prod.append(_parse_pypi_spec(line))
return prod, dev return prod, dev

0
tests/dev/__init__.py Normal file
View File

View File

@@ -0,0 +1,75 @@
"""Regression tests for dev/scripts/pkg_inventory.py.
Specifically guards the inline-comment bug that caused pkg-supply-chain
to fail on every push (issue #400-403): the previous parser did
``line.strip().strip(",").strip('"')`` which left trailing junk when a
dep line had an inline ``# comment``, producing invalid pip specs like
``pymupdf>=1.24",`` that broke the downstream PyPI mirror sync.
"""
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
# Load the script as a module — it lives outside the src/ tree.
_SCRIPT = Path(__file__).resolve().parents[2] / "dev" / "scripts" / "pkg_inventory.py"
_spec = importlib.util.spec_from_file_location("_pkg_inventory", _SCRIPT)
assert _spec and _spec.loader
_pkg_inventory = importlib.util.module_from_spec(_spec)
sys.modules["_pkg_inventory"] = _pkg_inventory
_spec.loader.exec_module(_pkg_inventory)
def test_extract_dep_specs_strips_inline_comments():
block = """
"pymupdf>=1.24", # AGPL-3.0 — flag if shipping outside internal use
"python-docx>=1.1",
"ruff>=0.11.0", # linter
"no-comment>=2.0",
"""
specs = _pkg_inventory._extract_dep_specs(block)
assert specs == [
"pymupdf>=1.24",
"python-docx>=1.1",
"ruff>=0.11.0",
"no-comment>=2.0",
]
def test_extract_dep_specs_skips_blank_and_comment_lines():
block = """
# this whole line is a comment
"pkg>=1.0",
# indented comment
"""
assert _pkg_inventory._extract_dep_specs(block) == ["pkg>=1.0"]
def test_extract_dep_specs_handles_single_quotes():
block = """
'singlequoted>=1.0',
"doublequoted>=2.0",
"""
assert _pkg_inventory._extract_dep_specs(block) == [
"singlequoted>=1.0",
"doublequoted>=2.0",
]
def test_parse_pyproject_deps_real_pymupdf_line():
"""Full integration — parse a small pyproject excerpt with the bug pattern."""
text = """[project]
name = "test"
dependencies = [
"pymupdf>=1.24", # AGPL-3.0 — flag if shipping outside internal use
"python-docx>=1.1",
]
"""
prod, _dev = _pkg_inventory._parse_pyproject_deps(text)
names = sorted(p["name"] for p in prod)
assert names == ["pymupdf", "python-docx"]
versions = {p["name"]: p["version"] for p in prod}
assert versions["pymupdf"] == ">=1.24" # NOT '>=1.24",'