feat(observability): label all services for promtail; enable Traefik access logs

compose.yml

@@ -46,6 +46,8 @@ services:
     volumes:
       - ./infra/coredns/Corefile:/etc/coredns/Corefile:ro
       - ./infra/coredns/hosts:/etc/coredns/hosts:ro
+    labels:
+      - "promtail=true"
     restart: unless-stopped
 
   traefik:
@@ -68,6 +70,12 @@ services:
     environment:
       - DOMAIN=${DOMAIN:-fhirworx.io}
       - OTEL_SERVICE_NAME=traefik
+      - TRAEFIK_ACCESSLOG=true
+      - TRAEFIK_ACCESSLOG_FORMAT=json
+      - TRAEFIK_ACCESSLOG_FIELDS_DEFAULTMODE=keep
+      - TRAEFIK_ACCESSLOG_FIELDS_HEADERS_DEFAULTMODE=keep
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -86,6 +94,8 @@ services:
     volumes:
       - rustfs_data:/data
       - rustfs_logs:/logs
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -125,6 +135,8 @@ services:
       - POSTGRESQL_DATABASE=gitea
     volumes:
       - postgres_data:/bitnami/postgresql
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -194,6 +206,8 @@ services:
     depends_on:
       - postgres
       - rustfs
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -216,6 +230,8 @@ services:
       - ./infra/act-runner/config.yaml:/config.yaml:ro
     depends_on:
       - git
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -258,6 +274,8 @@ services:
               count: all
               capabilities: [gpu]
     shm_size: "1g"
+    labels:
+      - "promtail=true"
     restart: unless-stopped
 
   zotero:
@@ -306,6 +324,8 @@ services:
             - driver: nvidia
               count: 1
               capabilities: [gpu]
+    labels:
+      - "promtail=true"
     restart: unless-stopped
 
   webdav:
@@ -331,6 +351,8 @@ services:
       - --vfs-cache-mode=full
     tmpfs:
       - /tmp/rclone-cache
+    labels:
+      - "promtail=true"
     restart: unless-stopped
     depends_on:
       - rustfs
@@ -362,6 +384,8 @@ services:
     depends_on:
       - postgres
       - rustfs
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -386,6 +410,8 @@ services:
     depends_on:
       - nessie
       - rustfs
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -420,6 +446,8 @@ services:
     depends_on:
       - postgres
       - rustfs
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -435,6 +463,8 @@ services:
       - ./assets/css/dashboard.css:/usr/share/nginx/html/dashboard.css:ro
       - ./assets/css/inject.css:/usr/share/nginx/html/fhirworx.css:ro
       - ./assets/icons/fav32.png:/usr/share/nginx/html/fav32.png:ro
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -448,6 +478,8 @@ services:
     container_name: docs
     networks:
       - gateway
+    labels:
+      - "promtail=true"
     restart: unless-stopped
 
   api:
@@ -482,6 +514,8 @@ services:
       - postgres
       - rustfs
       - nessie
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -515,6 +549,8 @@ services:
         uv run --no-sync stack bib ingest-mail || true;
         sleep $${MAIL_POLL_INTERVAL};
       done'
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -557,6 +593,8 @@ services:
       timeout: 10s
       retries: 5
       start_period: 60s
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -576,6 +614,8 @@ services:
       timeout: 5s
       retries: 5
       start_period: 30s
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -597,6 +637,8 @@ services:
       timeout: 5s
       retries: 5
       start_period: 30s
+    labels:
+      - "promtail=true"
     restart: unless-stopped
 
   nvidia-exporter:
@@ -616,6 +658,8 @@ services:
               capabilities: [gpu]
     cap_add:
       - SYS_ADMIN
+    labels:
+      - "promtail=true"
     restart: unless-stopped
 
   promtail:
@@ -631,6 +675,8 @@ services:
       - loki
     group_add:
       - "${DOCKER_GID:-985}"
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -658,6 +704,8 @@ services:
       timeout: 5s
       retries: 3
       start_period: 10s
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -686,6 +734,8 @@ services:
       timeout: 5s
       retries: 5
       start_period: 15s
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -724,6 +774,8 @@ services:
       timeout: 5s
       retries: 5
       start_period: 30s
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -758,6 +810,8 @@ services:
       - ./infra/oauth2-proxy/auth-handler.conf:/etc/nginx/conf.d/default.conf:ro
     depends_on:
       - oauth2-proxy
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -799,6 +853,8 @@ services:
       timeout: 5s
       retries: 5
       start_period: 15s
+    labels:
+      - "promtail=true"
     security_opt:
       - no-new-privileges:true
     restart: unless-stopped
@@ -811,6 +867,8 @@ services:
       - gateway
     volumes:
       - ./infra/cloudflared:/home/nonroot/.cloudflared:ro
+    labels:
+      - "promtail=true"
     restart: unless-stopped
 
 volumes:

tests/test_observability.py

@@ -162,3 +162,68 @@ class TestGrafanaDatasources:
         tempo = ds["Tempo"]
         assert tempo["jsonData"]["tracesToLogsV2"]["datasourceUid"] == "loki"
         assert tempo["jsonData"]["serviceMap"]["datasourceUid"] == "prometheus"
+
+
+PROMTAIL_LABELED = {
+    "coredns",
+    "traefik",
+    "rustfs",
+    "postgres",
+    "git",
+    "act-runner",
+    "notebooks",
+    "zotero",
+    "webdav",
+    "nessie",
+    "trino",
+    "polaris",
+    "dashboard",
+    "docs",
+    "api",
+    "mail-poller",
+    "auth-handler",
+    "oauth2-proxy",
+    "cloudflared",
+    "loki",
+    "promtail",
+    "tempo",
+    "cadvisor",
+    "otel-collector",
+    "prometheus",
+    "grafana",
+    "nvidia-exporter",
+}
+
+PROMTAIL_NOT_LABELED = {"mc", "wire"}
+
+
+class TestPromtailLabels:
+    def test_eligible_services_have_promtail_label(self):
+        svcs = _services()
+        missing = []
+        for name in PROMTAIL_LABELED:
+            if name not in svcs:
+                missing.append(f"  {name}: not in compose")
+                continue
+            labels = svcs[name].get("labels", []) or []
+            if "promtail=true" not in labels:
+                missing.append(f"  {name}: missing promtail=true label")
+        assert not missing, "\n".join(missing)
+
+    def test_excluded_services_do_not_have_label(self):
+        svcs = _services()
+        for name in PROMTAIL_NOT_LABELED:
+            if name not in svcs:
+                continue
+            labels = svcs[name].get("labels", []) or []
+            assert "promtail=true" not in labels, f"{name} should NOT be labeled"
+
+    def test_traefik_access_logs_enabled(self):
+        traefik = _services()["traefik"]
+        cmd = traefik.get("command", [])
+        if isinstance(cmd, str):
+            cmd = cmd.split()
+        flat = " ".join(str(x) for x in cmd)
+        assert "--accesslog" in flat or any(
+            "TRAEFIK_ACCESSLOG" in str(e) for e in traefik.get("environment", [])
+        )