add package supply chain scaffold: inventory scanner, drift detector, pipeline spec

refs #159 #165 #168

- pkg_inventory.py: scans all Dockerfiles, pyproject.toml, CI workflows
  for apt/apk/pypi/npm packages and emits data/pkg-manifest.json
- pkg_drift.py: compares manifest against fresh scan to detect phantom
  or untracked packages
- pkg-supply-chain.yml: CI-agnostic pipeline definition covering
  inventory → mirror-sync → drift-check → vuln-scan → auto-issues

dev/pipelines/pkg-supply-chain.yml

@@ -0,0 +1,61 @@
+# CI-agnostic pipeline definition for package supply chain management.
+# This is the abstract spec — use gen_config.py to emit concrete workflow
+# files for Gitea Actions, Woodpecker, and GitHub Actions.
+#
+# Pipeline: inventory → mirror-sync → drift-check → vuln-scan → issue-create
+
+name: Package Supply Chain
+
+triggers:
+  push:
+    paths:
+      - "**/Dockerfile*"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "docs/package.json"
+      - "docs/pnpm-lock.yaml"
+      - ".gitea/workflows/*.yml"
+  schedule:
+    cron: "0 6 * * *"  # daily at 06:00 UTC
+
+env:
+  GITEA_URL: "http://gitea:3000"
+  DEVPI_URL: "http://devpi:3141/root/pypi/+simple/"
+  APT_MIRROR_URL: "http://apt-cache:3142"
+
+steps:
+  - name: inventory
+    description: Scan repo and regenerate package manifest
+    run: uv run python dev/scripts/pkg_inventory.py
+    outputs:
+      - data/pkg-manifest.json
+
+  - name: mirror-sync
+    description: Update local mirrors to match manifest
+    needs: [inventory]
+    run: uv run python dev/scripts/pkg_mirror_sync.py
+    services:
+      - apt-cache
+      - devpi
+
+  - name: drift-check
+    description: Compare mirror contents against manifest
+    needs: [mirror-sync]
+    run: uv run python dev/scripts/pkg_drift.py
+    fail_on: drift
+
+  - name: vuln-scan
+    description: Scan mirrored packages for known CVEs
+    needs: [mirror-sync]
+    run: |
+      trivy fs --scanners vuln --format json --output data/pkg-vulns.json data/pkg-manifest.json
+      uv run python dev/scripts/pkg_vuln_report.py
+    tools:
+      - trivy
+
+  - name: auto-issues
+    description: Create Gitea issues for missing packages and CVEs
+    needs: [drift-check, vuln-scan]
+    run: uv run python dev/scripts/pkg_issues.py
+    env:
+      GITEA_TOKEN: "${GITEA_TOKEN}"

dev/scripts/pkg_drift.py

@@ -0,0 +1,138 @@
+"""Detect drift between the package manifest and what is actually used.
+
+Compares data/pkg-manifest.json against the repo source to detect:
+- packages in manifest but missing from source (phantom)
+- packages in source but missing from manifest (untracked)
+
+Usage:
+    uv run python dev/scripts/pkg_drift.py          # report + exit code
+    uv run python dev/scripts/pkg_drift.py --json    # JSON output
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
+
+
+def load_manifest() -> dict:
+    if not MANIFEST_PATH.exists():
+        print(f"ERROR: {MANIFEST_PATH} not found. Run pkg_inventory.py first.")
+        raise SystemExit(2)
+    return json.loads(MANIFEST_PATH.read_text())
+
+
+def scan_fresh() -> dict:
+    """Run a fresh inventory scan and return the result."""
+    from pkg_inventory import scan
+
+    return scan()
+
+
+def diff_lists(manifest_items: list, fresh_items: list, key: str = "name") -> dict:
+    """Compare two lists of dicts by a key field."""
+    if manifest_items and isinstance(manifest_items[0], dict):
+        m_set = {item[key] for item in manifest_items}
+        f_set = {item[key] for item in fresh_items}
+    else:
+        m_set = set(manifest_items)
+        f_set = set(fresh_items)
+
+    return {
+        "phantom": sorted(m_set - f_set),
+        "untracked": sorted(f_set - m_set),
+    }
+
+
+def check_drift() -> dict:
+    manifest = load_manifest()
+    fresh = scan_fresh()
+    drift: dict = {"apt": {}, "apk": {}, "pypi": {}, "npm": {}, "has_drift": False}
+
+    # apt
+    all_apt_manifest = set()
+    for pkgs in manifest.get("apt", {}).values():
+        all_apt_manifest.update(pkgs)
+    all_apt_fresh = set()
+    for pkgs in fresh.get("apt", {}).values():
+        all_apt_fresh.update(pkgs)
+    apt_diff = {
+        "phantom": sorted(all_apt_manifest - all_apt_fresh),
+        "untracked": sorted(all_apt_fresh - all_apt_manifest),
+    }
+    if apt_diff["phantom"] or apt_diff["untracked"]:
+        drift["apt"] = apt_diff
+        drift["has_drift"] = True
+
+    # apk
+    all_apk_manifest = set()
+    for pkgs in manifest.get("apk", {}).values():
+        all_apk_manifest.update(pkgs)
+    all_apk_fresh = set()
+    for pkgs in fresh.get("apk", {}).values():
+        all_apk_fresh.update(pkgs)
+    apk_diff = {
+        "phantom": sorted(all_apk_manifest - all_apk_fresh),
+        "untracked": sorted(all_apk_fresh - all_apk_manifest),
+    }
+    if apk_diff["phantom"] or apk_diff["untracked"]:
+        drift["apk"] = apk_diff
+        drift["has_drift"] = True
+
+    # pypi (compare all sections)
+    for section in ("project_prod", "project_dev", "notebook", "dockerfile_adhoc"):
+        d = diff_lists(
+            manifest.get("pypi", {}).get(section, []),
+            fresh.get("pypi", {}).get(section, []),
+        )
+        if d["phantom"] or d["untracked"]:
+            drift["pypi"][section] = d
+            drift["has_drift"] = True
+
+    # npm
+    d = diff_lists(
+        manifest.get("npm", []),
+        fresh.get("npm", []),
+    )
+    if d["phantom"] or d["untracked"]:
+        drift["npm"] = d
+        drift["has_drift"] = True
+
+    return drift
+
+
+def main() -> None:
+    drift = check_drift()
+
+    if "--json" in sys.argv:
+        print(json.dumps(drift, indent=2))
+    else:
+        if not drift["has_drift"]:
+            print("No drift detected — manifest matches repo source.")
+        else:
+            print("DRIFT DETECTED:")
+            for pkg_type in ("apt", "apk", "pypi", "npm"):
+                section = drift[pkg_type]
+                if not section:
+                    continue
+                if pkg_type == "pypi":
+                    for sub, d in section.items():
+                        if d.get("phantom"):
+                            print(f"  pypi/{sub} phantom (in manifest, not in source): {d['phantom']}")
+                        if d.get("untracked"):
+                            print(f"  pypi/{sub} untracked (in source, not in manifest): {d['untracked']}")
+                else:
+                    if section.get("phantom"):
+                        print(f"  {pkg_type} phantom: {section['phantom']}")
+                    if section.get("untracked"):
+                        print(f"  {pkg_type} untracked: {section['untracked']}")
+
+    raise SystemExit(1 if drift["has_drift"] else 0)
+
+
+if __name__ == "__main__":
+    main()

dev/scripts/pkg_inventory.py

@@ -0,0 +1,336 @@
+"""Scan the repo for all apt, apk, and PyPI packages in use.
+
+Parses Dockerfiles, pyproject.toml, CI workflows, and shell scripts to
+produce a canonical JSON manifest at data/pkg-manifest.json.
+
+Usage:
+    uv run python dev/scripts/pkg_inventory.py
+    uv run python dev/scripts/pkg_inventory.py --check  # exit 1 if manifest is stale
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json"
+
+# Directories to skip when scanning Dockerfiles
+SKIP_DIRS = {"node_modules", ".git", "__pycache__", "plugins"}
+
+
+# ---------------------------------------------------------------------------
+# Dockerfile parsers
+# ---------------------------------------------------------------------------
+
+def _join_continuation_lines(text: str) -> str:
+    """Merge backslash-continued lines into single lines."""
+    return re.sub(r"\\\s*\n\s*", " ", text)
+
+
+def _parse_apt_packages(text: str) -> list[str]:
+    """Extract packages from apt-get install commands."""
+    text = _join_continuation_lines(text)
+    pkgs: set[str] = set()
+    for m in re.finditer(
+        r"apt-get\s+install\s+(?:-\S+\s+)*(.+?)(?:&&|;|\n|$)", text
+    ):
+        tokens = m.group(1).split()
+        for tok in tokens:
+            tok = tok.strip()
+            if tok and not tok.startswith("-") and not tok.startswith("#"):
+                # strip version pin like =1.2.3
+                name = re.split(r"[=<>]", tok)[0]
+                if name and not name.startswith("/"):
+                    pkgs.add(name)
+    return sorted(pkgs)
+
+
+def _parse_apk_packages(text: str) -> list[str]:
+    """Extract packages from apk add commands."""
+    text = _join_continuation_lines(text)
+    pkgs: set[str] = set()
+    for m in re.finditer(r"apk\s+add\s+(?:-\S+\s+)*(.+?)(?:&&|;|\n|$)", text):
+        tokens = m.group(1).split()
+        for tok in tokens:
+            tok = tok.strip()
+            if tok and not tok.startswith("-") and not tok.startswith("#"):
+                name = re.split(r"[=<>~]", tok)[0]
+                if name:
+                    pkgs.add(name)
+    return sorted(pkgs)
+
+
+def _parse_uv_add_packages(text: str) -> list[dict]:
+    """Extract packages from 'uv add' commands in Dockerfiles."""
+    text = _join_continuation_lines(text)
+    pkgs: list[dict] = []
+    for m in re.finditer(r"uv\s+add\s+(.+?)(?:&&|;|\n|$)", text):
+        tokens = m.group(1).split()
+        for tok in tokens:
+            tok = tok.strip().strip('"').strip("'")
+            if tok.startswith("-"):
+                continue
+            if tok.startswith("http"):
+                continue
+            if tok:
+                pkgs.append(_parse_pypi_spec(tok))
+    return pkgs
+
+
+def _parse_pip_install_packages(text: str) -> list[dict]:
+    """Extract packages from 'pip install' commands in Dockerfiles."""
+    text = _join_continuation_lines(text)
+    pkgs: list[dict] = []
+    for m in re.finditer(r"pip\s+install\s+(.+?)(?:&&|;|\n|$)", text):
+        tokens = m.group(1).split()
+        for tok in tokens:
+            tok = tok.strip().strip('"').strip("'")
+            if tok.startswith("-"):
+                continue
+            if tok and tok != ".":
+                pkgs.append(_parse_pypi_spec(tok))
+    return pkgs
+
+
+def _parse_uv_run_with_packages(text: str) -> list[dict]:
+    """Extract packages from 'uv run --with pkg' commands."""
+    text = _join_continuation_lines(text)
+    pkgs: list[dict] = []
+    for m in re.finditer(r"uv\s+run\s+--with\s+(\S+)", text):
+        tok = m.group(1).strip('"').strip("'")
+        if tok:
+            pkgs.append(_parse_pypi_spec(tok))
+    return pkgs
+
+
+def _parse_pypi_spec(spec: str) -> dict:
+    """Parse a PEP 508 spec like 'narwhals>=2.17.0' or 'marimo[recommended]'."""
+    # strip extras
+    extras = ""
+    if "[" in spec:
+        base, rest = spec.split("[", 1)
+        extras = rest.split("]")[0]
+        spec = base + rest.split("]")[-1] if "]" in rest else base
+    m = re.match(r"([a-zA-Z0-9_-]+)(.*)", spec)
+    if not m:
+        return {"name": spec, "version": "", "extras": ""}
+    return {
+        "name": m.group(1).lower().replace("_", "-"),
+        "version": m.group(2).strip(),
+        "extras": extras,
+    }
+
+
+def _parse_base_images(text: str) -> list[str]:
+    """Extract FROM base images from Dockerfiles."""
+    images: list[str] = []
+    for m in re.finditer(r"^FROM\s+(\S+)", text, re.MULTILINE):
+        img = m.group(1)
+        if img not in images:
+            images.append(img)
+    return images
+
+
+# ---------------------------------------------------------------------------
+# CI workflow parser
+# ---------------------------------------------------------------------------
+
+def _parse_ci_curl_tools(text: str) -> list[dict]:
+    """Extract tools installed via curl in CI workflows."""
+    tools: list[dict] = []
+    patterns = [
+        (r"astral\.sh/uv/install\.sh", "uv", "latest"),
+        (r"go-containerregistry.*crane", "crane", "latest"),
+        (r"aquasecurity/trivy", "trivy", "latest"),
+    ]
+    for pattern, name, version in patterns:
+        if re.search(pattern, text):
+            tools.append({"name": name, "version": version})
+    return tools
+
+
+# ---------------------------------------------------------------------------
+# pyproject.toml parser (simple, no toml dependency)
+# ---------------------------------------------------------------------------
+
+def _parse_pyproject_deps(text: str) -> tuple[list[dict], list[dict]]:
+    """Parse dependencies from pyproject.toml without a TOML library."""
+    prod: list[dict] = []
+    dev: list[dict] = []
+
+    # prod deps
+    m = re.search(
+        r"^dependencies\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL
+    )
+    if m:
+        for line in m.group(1).splitlines():
+            line = line.strip().strip(",").strip('"').strip("'")
+            if line and not line.startswith("#"):
+                prod.append(_parse_pypi_spec(line))
+
+    # dev deps
+    m = re.search(r"dev\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL)
+    if m:
+        for line in m.group(1).splitlines():
+            line = line.strip().strip(",").strip('"').strip("'")
+            if line and not line.startswith("#"):
+                dev.append(_parse_pypi_spec(line))
+
+    # build-system requires
+    m = re.search(
+        r"\[build-system\].*?requires\s*=\s*\[(.*?)\]",
+        text,
+        re.MULTILINE | re.DOTALL,
+    )
+    if m:
+        for line in m.group(1).splitlines():
+            line = line.strip().strip(",").strip('"').strip("'")
+            if line and not line.startswith("#"):
+                prod.append(_parse_pypi_spec(line))
+
+    return prod, dev
+
+
+# ---------------------------------------------------------------------------
+# pnpm / Node parser
+# ---------------------------------------------------------------------------
+
+def _parse_package_json_deps(text: str) -> list[dict]:
+    """Parse npm dependencies from package.json."""
+    data = json.loads(text)
+    pkgs: list[dict] = []
+    for section in ("dependencies", "devDependencies"):
+        for name, version in data.get(section, {}).items():
+            pkgs.append({"name": name, "version": version})
+    return pkgs
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def scan() -> dict:
+    """Scan repo and return the package manifest."""
+    manifest: dict = {
+        "apt": {},
+        "apk": {},
+        "pypi": {"project_prod": [], "project_dev": [], "notebook": [], "dockerfile_adhoc": []},
+        "npm": [],
+        "ci_tools": [],
+        "base_images": {},
+    }
+
+    # --- Dockerfiles ---
+    dockerfiles = list(ROOT.glob("**/Dockerfile")) + list(
+        ROOT.glob("**/Dockerfile.*")
+    )
+    dockerfiles = [
+        df for df in dockerfiles
+        if not any(skip in df.parts for skip in SKIP_DIRS)
+    ]
+    for df in sorted(dockerfiles):
+        rel = str(df.relative_to(ROOT))
+        text = df.read_text()
+
+        apt = _parse_apt_packages(text)
+        if apt:
+            manifest["apt"][rel] = apt
+
+        apk = _parse_apk_packages(text)
+        if apk:
+            manifest["apk"][rel] = apk
+
+        images = _parse_base_images(text)
+        if images:
+            manifest["base_images"][rel] = images
+
+        # uv add in Dockerfiles (notebook pattern)
+        uv_pkgs = _parse_uv_add_packages(text)
+        if uv_pkgs:
+            manifest["pypi"]["notebook"].extend(uv_pkgs)
+
+        # pip install in Dockerfiles
+        pip_pkgs = _parse_pip_install_packages(text)
+        if pip_pkgs:
+            manifest["pypi"]["dockerfile_adhoc"].extend(pip_pkgs)
+
+        # uv run --with
+        with_pkgs = _parse_uv_run_with_packages(text)
+        if with_pkgs:
+            manifest["pypi"]["dockerfile_adhoc"].extend(with_pkgs)
+
+    # --- pyproject.toml ---
+    pyproject = ROOT / "pyproject.toml"
+    if pyproject.exists():
+        prod, dev = _parse_pyproject_deps(pyproject.read_text())
+        manifest["pypi"]["project_prod"] = prod
+        manifest["pypi"]["project_dev"] = dev
+
+    # --- package.json (docs) ---
+    pkg_json = ROOT / "docs" / "package.json"
+    if pkg_json.exists():
+        manifest["npm"] = _parse_package_json_deps(pkg_json.read_text())
+
+    # --- CI workflows ---
+    ci_tools_seen: set[str] = set()
+    for wf in sorted((ROOT / ".gitea" / "workflows").glob("*.yml")):
+        text = wf.read_text()
+        for tool in _parse_ci_curl_tools(text):
+            if tool["name"] not in ci_tools_seen:
+                ci_tools_seen.add(tool["name"])
+                manifest["ci_tools"].append(tool)
+
+    # --- Deduplicate & sort ---
+    for section in ("notebook", "dockerfile_adhoc"):
+        seen: set[str] = set()
+        deduped: list[dict] = []
+        for pkg in manifest["pypi"][section]:
+            if pkg["name"] not in seen:
+                seen.add(pkg["name"])
+                deduped.append(pkg)
+        manifest["pypi"][section] = sorted(deduped, key=lambda p: p["name"])
+
+    return manifest
+
+
+def main() -> None:
+    manifest = scan()
+    output = json.dumps(manifest, indent=2, sort_keys=False) + "\n"
+
+    if "--check" in sys.argv:
+        if MANIFEST_PATH.exists():
+            existing = MANIFEST_PATH.read_text()
+            if existing == output:
+                print("pkg-manifest.json is up to date.")
+                raise SystemExit(0)
+            else:
+                print("pkg-manifest.json is STALE — re-run without --check.")
+                raise SystemExit(1)
+        else:
+            print("pkg-manifest.json does not exist — run without --check first.")
+            raise SystemExit(1)
+
+    MANIFEST_PATH.write_text(output)
+    # Summary
+    apt_count = sum(len(v) for v in manifest["apt"].values())
+    apk_count = sum(len(v) for v in manifest["apk"].values())
+    pypi_count = sum(
+        len(manifest["pypi"][k])
+        for k in ("project_prod", "project_dev", "notebook", "dockerfile_adhoc")
+    )
+    npm_count = len(manifest["npm"])
+    ci_count = len(manifest["ci_tools"])
+    img_count = sum(len(v) for v in manifest["base_images"].values())
+    print(f"Wrote {MANIFEST_PATH.relative_to(ROOT)}")
+    print(
+        f"  apt: {apt_count}  apk: {apk_count}  pypi: {pypi_count}"
+        f"  npm: {npm_count}  ci_tools: {ci_count}  base_images: {img_count}"
+    )
+
+
+if __name__ == "__main__":
+    main()