stack/infra/grafana/dashboards/pipeline-performance.json

{
  "annotations": {
    "list": []
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "links": [],
  "panels": [
    {
      "title": "Pipeline Run Duration",
      "type": "timeseries",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "color": { "mode": "palette-classic" }
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(stack_stack_pipeline_duration_seconds_bucket{pipeline=~\"$pipeline\"}[$__rate_interval])) by (le, pipeline))",
          "legendFormat": "p95 {{pipeline}}",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.50, sum(rate(stack_stack_pipeline_duration_seconds_bucket{pipeline=~\"$pipeline\"}[$__rate_interval])) by (le, pipeline))",
          "legendFormat": "p50 {{pipeline}}",
          "refId": "B"
        }
      ]
    },
    {
      "title": "Step Duration Heatmap",
      "type": "heatmap",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [
        {
          "expr": "sum(rate(stack_stack_step_duration_seconds_bucket{pipeline=~\"$pipeline\"}[$__rate_interval])) by (le, step)",
          "legendFormat": "{{step}}",
          "refId": "A",
          "format": "heatmap"
        }
      ],
      "options": {
        "calculate": false,
        "yAxis": { "unit": "s" }
      }
    },
    {
      "title": "Cache Hit Rate",
      "type": "gauge",
      "gridPos": { "h": 6, "w": 6, "x": 0, "y": 8 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "min": 0,
          "max": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "red", "value": null },
              { "color": "yellow", "value": 0.3 },
              { "color": "green", "value": 0.7 }
            ]
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "sum(rate(stack_stack_cache_hits_total{pipeline=~\"$pipeline\"}[$__rate_interval])) / (sum(rate(stack_stack_cache_hits_total{pipeline=~\"$pipeline\"}[$__rate_interval])) + sum(rate(stack_stack_cache_misses_total{pipeline=~\"$pipeline\"}[$__rate_interval])))",
          "legendFormat": "hit rate",
          "refId": "A"
        }
      ]
    },
    {
      "title": "Memory RSS",
      "type": "timeseries",
      "gridPos": { "h": 6, "w": 6, "x": 6, "y": 8 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": {
          "unit": "decmbytes",
          "color": { "mode": "palette-classic" }
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "stack_stack_system_rss_mb{service=~\"$pipeline\"}",
          "legendFormat": "{{service}}",
          "refId": "A"
        }
      ]
    },
    {
      "title": "CPU Utilization",
      "type": "timeseries",
      "gridPos": { "h": 6, "w": 6, "x": 12, "y": 8 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "color": { "mode": "palette-classic" }
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "stack_stack_system_cpu_percent{service=~\"$pipeline\"}",
          "legendFormat": "{{service}}",
          "refId": "A"
        }
      ]
    },
    {
      "title": "Row Throughput per Step",
      "type": "bargauge",
      "gridPos": { "h": 6, "w": 6, "x": 18, "y": 8 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "color": { "mode": "palette-classic" }
        },
        "overrides": []
      },
      "targets": [
        {
          "expr": "sum(rate(stack_stack_step_rows_out_sum{pipeline=~\"$pipeline\"}[$__rate_interval])) by (step)",
          "legendFormat": "{{step}}",
          "refId": "A"
        }
      ],
      "options": {
        "orientation": "horizontal",
        "displayMode": "gradient",
        "showUnfilled": true
      }
    },
    {
      "title": "Recent Pipeline Traces",
      "type": "table",
      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
      "datasource": { "type": "tempo", "uid": "tempo" },
      "targets": [
        {
          "query": "pipeline.run",
          "queryType": "search",
          "refId": "A",
          "limit": 20,
          "service": "stack"
        }
      ],
      "options": {
        "showHeader": true,
        "sortBy": [{ "displayName": "Start Time", "desc": true }]
      }
    },
    {
      "title": "Pipeline Logs",
      "type": "logs",
      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 22 },
      "datasource": { "type": "loki", "uid": "loki" },
      "targets": [
        {
          "expr": "{container_name=\"api\"} |= \"trace_id\"",
          "refId": "A"
        }
      ],
      "options": {
        "showTime": true,
        "showLabels": false,
        "showCommonLabels": false,
        "wrapLogMessage": true,
        "prettifyLogMessage": false,
        "enableLogDetails": true,
        "sortOrder": "Descending"
      }
    }
  ],
  "schemaVersion": 39,
  "tags": ["pipeline", "telemetry", "otel"],
  "templating": {
    "list": [
      {
        "name": "pipeline",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "prometheus" },
        "query": "label_values(stack_stack_step_duration_seconds_count, pipeline)",
        "refresh": 2,
        "includeAll": true,
        "multi": true,
        "current": {
          "selected": true,
          "text": "All",
          "value": "$__all"
        }
      }
    ]
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "Pipeline Performance",
  "uid": "stack-pipeline-perf",
  "version": 1
}