fix: pkg inventory parser — filter self-refs, parse optional-dependencies

The package mirror sync failed because:
1. `_parse_pyproject_deps` included `stack[all]` from [dependency-groups]
   as an external package — `pip download` can't resolve a local project
2. Optional-dependency extras (perf, aco, api, etc.) were silently
   skipped because `]` inside `stack[conf]` terminated the regex early

Fixes:
- `_is_self_ref()` filters out any spec named "stack"
- Balanced-bracket parser for optional-dependencies blocks handles
  nested `[extras]` inside dependency specs
- Manifest now correctly captures all 28 unique PyPI deps including
  6 OpenTelemetry packages + psutil from the perf extra

data/pkg-manifest.json

@@ -0,0 +1,292 @@
+{
+  "apt": {},
+  "apk": {},
+  "pypi": {
+    "project_prod": [
+      {
+        "name": "pydantic",
+        "version": ">=2.0.0",
+        "extras": ""
+      },
+      {
+        "name": "pydantic",
+        "version": ">=2.0.0",
+        "extras": ""
+      },
+      {
+        "name": "httpx",
+        "version": ">=0.28.1",
+        "extras": ""
+      },
+      {
+        "name": "duckdb",
+        "version": ">=1.0.0",
+        "extras": ""
+      },
+      {
+        "name": "narwhals",
+        "version": ">=2.17.0",
+        "extras": ""
+      },
+      {
+        "name": "sqlglot",
+        "version": ">=26.0.0",
+        "extras": ""
+      },
+      {
+        "name": "pyarrow",
+        "version": ">=23.0.0",
+        "extras": ""
+      },
+      {
+        "name": "duckdb",
+        "version": ">=1.0.0",
+        "extras": ""
+      },
+      {
+        "name": "fastapi",
+        "version": ">=0.135.1",
+        "extras": ""
+      },
+      {
+        "name": "uvicorn",
+        "version": ">=0.41.0",
+        "extras": ""
+      },
+      {
+        "name": "httpx",
+        "version": ">=0.28.1",
+        "extras": ""
+      },
+      {
+        "name": "pyjwt",
+        "version": ">=2.12.0",
+        "extras": ""
+      },
+      {
+        "name": "cryptography",
+        "version": ">=46.0.5",
+        "extras": ""
+      },
+      {
+        "name": "httpx",
+        "version": ">=0.28.1",
+        "extras": ""
+      },
+      {
+        "name": "fsspec",
+        "version": ">=2024.1.0",
+        "extras": ""
+      },
+      {
+        "name": "narwhals",
+        "version": ">=2.17.0",
+        "extras": ""
+      },
+      {
+        "name": "pyarrow",
+        "version": ">=23.0.0",
+        "extras": ""
+      },
+      {
+        "name": "pydantic",
+        "version": ">=2.0.0",
+        "extras": ""
+      },
+      {
+        "name": "pydantic",
+        "version": ">=2.0.0",
+        "extras": ""
+      },
+      {
+        "name": "typer",
+        "version": ">=0.24.1",
+        "extras": ""
+      },
+      {
+        "name": "uvicorn",
+        "version": ">=0.41.0",
+        "extras": ""
+      },
+      {
+        "name": "narwhals",
+        "version": ">=2.17.0",
+        "extras": ""
+      },
+      {
+        "name": "pydantic",
+        "version": ">=2.0.0",
+        "extras": ""
+      },
+      {
+        "name": "duckdb",
+        "version": ">=1.0.0",
+        "extras": ""
+      },
+      {
+        "name": "narwhals",
+        "version": ">=2.17.0",
+        "extras": ""
+      },
+      {
+        "name": "narwhals",
+        "version": ">=2.17.0",
+        "extras": ""
+      },
+      {
+        "name": "pyarrow",
+        "version": ">=23.0.0",
+        "extras": ""
+      },
+      {
+        "name": "fsspec",
+        "version": ">=2024.1.0",
+        "extras": ""
+      },
+      {
+        "name": "opentelemetry-api",
+        "version": ">=1.25.0",
+        "extras": ""
+      },
+      {
+        "name": "opentelemetry-sdk",
+        "version": ">=1.25.0",
+        "extras": ""
+      },
+      {
+        "name": "opentelemetry-exporter-otlp-proto-grpc",
+        "version": ">=1.25.0",
+        "extras": ""
+      },
+      {
+        "name": "opentelemetry-exporter-prometheus",
+        "version": ">=0.46b0",
+        "extras": ""
+      },
+      {
+        "name": "opentelemetry-instrumentation-fastapi",
+        "version": ">=0.46b0",
+        "extras": ""
+      },
+      {
+        "name": "opentelemetry-semantic-conventions",
+        "version": ">=0.46b0",
+        "extras": ""
+      },
+      {
+        "name": "psutil",
+        "version": ">=5.9.0",
+        "extras": ""
+      },
+      {
+        "name": "databricks-cli",
+        "version": ">=0.18.0",
+        "extras": ""
+      },
+      {
+        "name": "databricks-sdk",
+        "version": ">=0.85.0",
+        "extras": ""
+      },
+      {
+        "name": "boto3",
+        "version": ">=1.35.0",
+        "extras": ""
+      },
+      {
+        "name": "s3fs",
+        "version": ">=2026.2.0",
+        "extras": ""
+      },
+      {
+        "name": "google-cloud-storage",
+        "version": ">=2.18.0",
+        "extras": ""
+      },
+      {
+        "name": "gcsfs",
+        "version": ">=2024.1.0",
+        "extras": ""
+      },
+      {
+        "name": "azure-storage-blob",
+        "version": ">=12.23.0",
+        "extras": ""
+      },
+      {
+        "name": "adlfs",
+        "version": ">=2024.1.0",
+        "extras": ""
+      },
+      {
+        "name": "uv-build",
+        "version": ">=0.7,<1",
+        "extras": ""
+      }
+    ],
+    "project_dev": [],
+    "notebook": [],
+    "dockerfile_adhoc": []
+  },
+  "npm": [
+    {
+      "name": "@docusaurus/core",
+      "version": "^3.7.0"
+    },
+    {
+      "name": "@docusaurus/preset-classic",
+      "version": "^3.7.0"
+    },
+    {
+      "name": "@easyops-cn/docusaurus-search-local",
+      "version": "^0.55.1"
+    },
+    {
+      "name": "@mdx-js/react",
+      "version": "^3.0.0"
+    },
+    {
+      "name": "clsx",
+      "version": "^2.0.0"
+    },
+    {
+      "name": "prism-react-renderer",
+      "version": "^2.3.0"
+    },
+    {
+      "name": "react",
+      "version": "^18.0.0"
+    },
+    {
+      "name": "react-dom",
+      "version": "^18.0.0"
+    },
+    {
+      "name": "@docusaurus/module-type-aliases",
+      "version": "^3.7.0"
+    },
+    {
+      "name": "@docusaurus/types",
+      "version": "^3.7.0"
+    },
+    {
+      "name": "typescript",
+      "version": "~5.6.0"
+    }
+  ],
+  "ci_tools": [
+    {
+      "name": "uv",
+      "version": "latest"
+    },
+    {
+      "name": "crane",
+      "version": "latest"
+    },
+    {
+      "name": "trivy",
+      "version": "latest"
+    }
+  ],
+  "base_images": {}
+}

dev/scripts/pkg_inventory.py

@@ -158,25 +158,73 @@ def _parse_ci_curl_tools(text: str) -> list[dict]:
 # ---------------------------------------------------------------------------
 
 
+def _is_self_ref(spec: str) -> bool:
+    """Check if a dependency spec is a self-reference (e.g. stack[aco])."""
+    name = re.split(r"[\[>=<~!]", spec)[0].strip().lower().replace("_", "-")
+    return name == "stack"
+
+
 def _parse_pyproject_deps(text: str) -> tuple[list[dict], list[dict]]:
-    """Parse dependencies from pyproject.toml without a TOML library."""
+    """Parse dependencies from pyproject.toml without a TOML library.
+
+    Extracts:
+    - [project] dependencies (prod)
+    - [project.optional-dependencies] all extras (prod)
+    - [dependency-groups] dev (dev)
+    - [build-system] requires (prod)
+
+    Filters out self-references (stack[*]) which can't be downloaded.
+    """
     prod: list[dict] = []
     dev: list[dict] = []
 
-    # prod deps
+    # prod deps — [project] dependencies
     m = re.search(r"^dependencies\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL)
     if m:
         for line in m.group(1).splitlines():
             line = line.strip().strip(",").strip('"').strip("'")
-            if line and not line.startswith("#"):
+            if line and not line.startswith("#") and not _is_self_ref(line):
                 prod.append(_parse_pypi_spec(line))
 
-    # dev deps
+    # optional deps — [project.optional-dependencies] all sections
+    # Extract the section content, then parse each "name = [...]" block.
+    # Can't use simple .*? because extras brackets (e.g. stack[conf])
+    # contain ] which terminates the match early.
+    opt_section = re.search(
+        r"\[project\.optional-dependencies\](.*?)(?:\n\[|\Z)",
+        text,
+        re.DOTALL,
+    )
+    if opt_section:
+        # Match each key = [ ... ] block by finding balanced brackets
+        for m in re.finditer(
+            r"^(\w+)\s*=\s*\[",
+            opt_section.group(1),
+            re.MULTILINE,
+        ):
+            start = m.end()
+            rest = opt_section.group(1)[start:]
+            # Find closing ] that isn't inside a string bracket
+            depth = 1
+            i = 0
+            while i < len(rest) and depth > 0:
+                if rest[i] == "[":
+                    depth += 1
+                elif rest[i] == "]":
+                    depth -= 1
+                i += 1
+            block = rest[: i - 1] if depth == 0 else rest
+            for line in block.splitlines():
+                line = line.strip().strip(",").strip('"').strip("'")
+                if line and not line.startswith("#") and not _is_self_ref(line):
+                    prod.append(_parse_pypi_spec(line))
+
+    # dev deps — [dependency-groups] dev
     m = re.search(r"dev\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL)
     if m:
         for line in m.group(1).splitlines():
             line = line.strip().strip(",").strip('"').strip("'")
-            if line and not line.startswith("#"):
+            if line and not line.startswith("#") and not _is_self_ref(line):
                 dev.append(_parse_pypi_spec(line))
 
     # build-system requires
@@ -188,7 +236,7 @@ def _parse_pyproject_deps(text: str) -> tuple[list[dict], list[dict]]:
     if m:
         for line in m.group(1).splitlines():
             line = line.strip().strip(",").strip('"').strip("'")
-            if line and not line.startswith("#"):
+            if line and not line.startswith("#") and not _is_self_ref(line):
                 prod.append(_parse_pypi_spec(line))
 
     return prod, dev