fix(observability): code-review remediation pass

- Trino scrape path: /v1/status (JSON) → /metrics (OpenMetrics)
- Tempo→Loki correlation: add tags mapping service.name → container
  so span clicks build {container=...} | traceID=... instead of an
  unindexed scan
- Promtail api/mail-poller/cloudflared regexes: add \b word boundaries
  so high-frequency words (error/origin/poll) don't set the labels
  on every other line
- data-lake S3 ops panel: drop `| json` (RustFS isn't JSON), use the
  s3_op stream label directly
- Add Quarkus-pair test covering the nessie|polaris combined stage

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

infra/grafana/dashboards/data-lake.json

@@ -11,6 +11,6 @@
     {"id": 2, "title": "Trino query duration p95", "type": "timeseries", "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(trino_query_execution_seconds_bucket[5m])))"}]},
     {"id": 3, "title": "Nessie/Polaris HTTP rate", "type": "timeseries", "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "sum by (job) (rate(http_server_requests_seconds_count{job=~\"nessie|polaris\"}[1m]))"}]},
     {"id": 4, "title": "Nessie/Polaris error rate", "type": "timeseries", "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "sum by (job) (rate(http_server_requests_seconds_count{job=~\"nessie|polaris\",status=~\"5..\"}[1m]))"}]},
-    {"id": 5, "title": "RustFS S3 ops by op", "type": "timeseries", "gridPos": {"x": 0, "y": 16, "w": 24, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum by (s3_op) (rate({container=\"rustfs\"} | json [1m]))"}]}
+    {"id": 5, "title": "RustFS S3 ops by op", "type": "timeseries", "gridPos": {"x": 0, "y": 16, "w": 24, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum by (s3_op) (rate({container=\"rustfs\", s3_op!~\"\"}[1m]))"}]}
   ]
 }

infra/grafana/provisioning/datasources/datasources.yml

@@ -26,5 +26,11 @@ datasources:
       tracesToLogsV2:
         datasourceUid: loki
         filterByTraceID: true
+        # Map span resource attributes to Loki stream labels so a
+        # span click yields {container="<service>"} | traceID="<id>"
+        # instead of an unindexed scan.
+        tags:
+          - key: service.name
+            value: container
       serviceMap:
         datasourceUid: prometheus

infra/loki/promtail-config.yml

@@ -120,11 +120,13 @@ scrape_configs:
                 s3_bucket:
 
       # --- API (FastAPI/stack) ---
+      # Word-boundary anchors keep the label off lines that merely contain
+      # the substrings (e.g. "polling" should not set api_event=poll).
       - match:
           selector: '{container="api"}'
           stages:
             - regex:
-                expression: '(?P<api_event>pipeline|ingest|health|error)'
+                expression: '\b(?P<api_event>pipeline|ingest|health|error)\b'
             - regex:
                 expression: 'pipeline=(?P<pipeline_name>[A-Za-z0-9_.-]+)'
             - labels:
@@ -136,7 +138,7 @@ scrape_configs:
           selector: '{container="mail-poller"}'
           stages:
             - regex:
-                expression: '(?P<mail_event>ingest|poll|error)'
+                expression: '\b(?P<mail_event>ingest|poll|error)\b'
             - regex:
                 expression: 'count=(?P<mail_count>\d+)'
             - labels:
@@ -160,7 +162,7 @@ scrape_configs:
           selector: '{container="cloudflared"}'
           stages:
             - regex:
-                expression: '(?P<cf_event>connection|registered|unregistered|reconnect|quic|error|tunnel|origin)'
+                expression: '\b(?P<cf_event>connection|registered|unregistered|reconnect|quic|error|tunnel|origin)\b'
             - regex:
                 expression: 'status=(?P<cf_origin_status>\d{3})'
             - labels:

infra/prometheus/targets/services.yml

@@ -41,11 +41,11 @@
     job: polaris
     __metrics_path__: /q/metrics
 
-# Trino
+# Trino — /metrics is the OpenMetrics endpoint (435+); /v1/status is JSON
 - targets: ['trino:8080']
   labels:
     job: trino
-    __metrics_path__: /v1/status
+    __metrics_path__: /metrics
 
 # OTel Collector
 - targets: ['otel-collector:8889']

tests/test_observability.py

@@ -129,6 +129,13 @@ class TestPerServiceStages:
             assert f'container="{container}"' in text, f"no match stage for {container}"
             assert label in text, f"no {label} extraction for {container}"
 
+    def test_quarkus_pair_has_combined_stage(self):
+        # nessie + polaris share one stage via regex selector.
+        cfg = yaml.safe_load((ROOT / "infra/loki/promtail-config.yml").read_text())
+        text = yaml.safe_dump(cfg["scrape_configs"][0]["pipeline_stages"])
+        assert 'container=~"nessie|polaris"' in text
+        assert "quarkus_route" in text
+
 
 class TestOtelTempo:
     def test_otel_exports_traces_to_tempo(self):