chore(observability): remove cadvisor

Rootless Docker hides container names from cgroup-based exporters, so
cadvisor only ever produced anonymous series here (152 of them, none
with a name label). The five dashboard panels that queried it have been
blank since it was added; drop the service, its scrape target, the
tests that pinned it, and those panels.

compose.yml

@@ -787,27 +787,6 @@ services:
       - no-new-privileges:true
     restart: unless-stopped
 
-  cadvisor:
-    image: gcr.io/cadvisor/cadvisor:latest
-    container_name: cadvisor
-    networks:
-      - observability
-    privileged: true
-    volumes:
-      - /:/rootfs:ro
-      - ${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock:ro
-      - /sys:/sys:ro
-      - /dev/disk/:/dev/disk:ro
-    healthcheck:
-      test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/healthz"]
-      interval: 30s
-      timeout: 5s
-      retries: 5
-      start_period: 30s
-    labels:
-      - "promtail=true"
-    restart: unless-stopped
-
   nvidia-exporter:
     image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04
     # Prefixed: corwins.media compose also defines a "nvidia-exporter"

infra/grafana/dashboards/gpu-notebooks.json

@@ -10,8 +10,6 @@
     {"id": 1, "title": "GPU utilization %", "type": "timeseries", "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "DCGM_FI_DEV_GPU_UTIL"}]},
     {"id": 2, "title": "GPU memory used (MiB)", "type": "timeseries", "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "DCGM_FI_DEV_FB_USED"}]},
     {"id": 3, "title": "GPU temp (°C)", "type": "timeseries", "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "DCGM_FI_DEV_GPU_TEMP"}]},
-    {"id": 4, "title": "GPU power (W)", "type": "timeseries", "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "DCGM_FI_DEV_POWER_USAGE"}]},
+    {"id": 4, "title": "GPU power (W)", "type": "timeseries", "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "DCGM_FI_DEV_POWER_USAGE"}]}
-    {"id": 5, "title": "Notebooks/Zotero CPU %", "type": "timeseries", "gridPos": {"x": 0, "y": 16, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "rate(container_cpu_usage_seconds_total{name=~\"notebooks|zotero\"}[1m]) * 100", "legendFormat": "{{name}}"}]},
-    {"id": 6, "title": "Notebooks/Zotero memory", "type": "timeseries", "gridPos": {"x": 12, "y": 16, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "container_memory_working_set_bytes{name=~\"notebooks|zotero\"} / 1024 / 1024", "legendFormat": "{{name}}"}]}
   ]
 }

infra/grafana/dashboards/homelab-overview.json

@@ -7,11 +7,8 @@
   "time": {"from": "now-6h", "to": "now"},
   "refresh": "30s",
   "panels": [
-    {"id": 1, "title": "Containers up", "type": "stat", "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "count(container_last_seen{name!=\"\"})"}]},
+    {"id": 2, "title": "Log volume / 1m", "type": "stat", "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum(rate({container=~\".+\"}[1m]))"}]},
-    {"id": 2, "title": "Log volume / 1m", "type": "stat", "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum(rate({container=~\".+\"}[1m]))"}]},
+    {"id": 3, "title": "Error rate / 1m", "type": "stat", "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum(rate({level=\"error\"}[1m]))"}]},
-    {"id": 3, "title": "Error rate / 1m", "type": "stat", "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum(rate({level=\"error\"}[1m]))"}]},
+    {"id": 6, "title": "Top error sources (1h)", "type": "logs", "gridPos": {"x": 0, "y": 4, "w": 24, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "{level=\"error\"} | json"}]}
-    {"id": 4, "title": "Container CPU %", "type": "timeseries", "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "rate(container_cpu_usage_seconds_total{name!=\"\"}[1m]) * 100", "legendFormat": "{{name}}"}]},
-    {"id": 5, "title": "Container memory (MiB)", "type": "timeseries", "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "container_memory_working_set_bytes{name!=\"\"} / 1024 / 1024", "legendFormat": "{{name}}"}]},
-    {"id": 6, "title": "Top error sources (1h)", "type": "logs", "gridPos": {"x": 0, "y": 12, "w": 24, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "{level=\"error\"} | json"}]}
   ]
 }

infra/prometheus/targets/services.yml

@@ -19,11 +19,6 @@
   labels:
     job: loki
 
-# cAdvisor
-- targets: ['cadvisor:8080']
-  labels:
-    job: cadvisor
-
 # NVIDIA GPU (DCGM)
 - targets: ['nvidia-exporter:9400']
   labels:

tests/test_compose_mounts.py

@@ -136,7 +136,6 @@ class TestSecurityOpts:
             "docs",  # static site
             "promtail",  # needs host log access
             "cloudflared",  # tunnel agent
-            "cadvisor",  # privileged: true for host metrics
             "nvidia-exporter",  # upstream dcgm-exporter, GPU runtime + SYS_ADMIN
         }
         compose = _load_compose()

tests/test_observability.py

@@ -42,26 +42,6 @@ class TestTempo:
         assert any("./infra/tempo/tempo.yml" in v for v in mounts)
 
 
-class TestCadvisor:
-    def test_cadvisor_service_present(self):
-        svcs = _services()
-        assert "cadvisor" in svcs
-
-    def test_cadvisor_privileged(self):
-        # cadvisor needs privileged for cgroup reads
-        assert _services()["cadvisor"].get("privileged") is True
-
-    def test_cadvisor_mounts(self):
-        svc = _services()["cadvisor"]
-        sources = []
-        for v in svc["volumes"]:
-            if isinstance(v, str):
-                sources.append(v.split(":")[0])
-        # rootfs, sys, /dev/disk needed for resource introspection
-        assert any(s == "/" for s in sources)
-        assert any(s == "/sys" for s in sources)
-
-
 class TestNvidiaExporter:
     def test_nvidia_exporter_present(self):
         svcs = _services()
@@ -155,7 +135,7 @@ class TestPrometheusTargets:
         targets = yaml.safe_load(path.read_text())
         jobs = {entry["labels"]["job"] for entry in targets if "labels" in entry}
         assert "tempo" in jobs
-        assert "cadvisor" in jobs
+        assert "cadvisor" not in jobs
         assert "nvidia-gpu" in jobs
         assert "jaeger" not in jobs
 
@@ -195,7 +175,6 @@ PROMTAIL_LABELED = {
     "loki",
     "promtail",
     "tempo",
-    "cadvisor",
     "otel-collector",
     "prometheus",
     "grafana",