fix(perf,infra): actually export the metrics — /metrics route, perf extra in the api/llm images, scrape targets, telemetry on (refs #579)

The P26 telemetry path never reached Prometheus: setup_meter_provider
only installs an in-process PrometheusMetricReader, nothing served the
registry (api and llm answered 404 on /metrics), the images never
installed the perf extra, STACK_TELEMETRY was off, and no scrape target
existed — so the data-pipelines request-rate panel was empty from the
day it was written. Now: perf.middleware.instrument mounts GET /metrics
(prometheus_client registry), the api and llm images install --extra
perf, compose sets STACK_TELEMETRY=true on both, services.yml scrapes
api:8000 and llm:8000, and both dashboards' request-rate panels query
http_server_duration_milliseconds_count (what the FastAPI instrumentor
emits; stack_http_server_requests_total never existed). Verified live:
stack_llm_dispatch_total is queryable in Prometheus with job=llm.

compose.yml

@@ -649,7 +649,9 @@ services:
       - S3_ENDPOINT=${S3_ENDPOINT:-http://rustfs:9000}
       - S3_REGION=${S3_REGION:-us-east-1}
       - S3_WAREHOUSE=${S3_WAREHOUSE:-s3://lakehouse/}
-      # OpenTelemetry
+      # OpenTelemetry — traces go to the collector; metrics are served
+      # at /metrics and scraped directly (#579).
+      - STACK_TELEMETRY=true
       - OTEL_SERVICE_NAME=api
       - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
     volumes:
@@ -694,6 +696,7 @@ services:
       - LLM_PG_HOST=postgres
       - LLM_DB_PASSWORD=${LLM_DB_PASSWORD}
       - LLM_DUCKDB_REPLICA=/app/data/replica/aco.ro.duckdb
+      - STACK_TELEMETRY=true   # /metrics scraped by prometheus (#579)
       - OTEL_SERVICE_NAME=llm
       - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
     depends_on:

infra/grafana/dashboards/data-pipelines.json

@@ -7,7 +7,7 @@
   "time": {"from": "now-6h", "to": "now"},
   "refresh": "30s",
   "panels": [
-    {"id": 1, "title": "API request rate (status)", "type": "timeseries", "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "sum by (status) (rate(stack_http_server_requests_total[1m]))"}]},
+    {"id": 1, "title": "API request rate (status)", "type": "timeseries", "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "sum by (http_status_code) (rate(http_server_duration_milliseconds_count{job=\"api\"}[1m]))"}]},
     {"id": 2, "title": "Mail-poller poll cycles", "type": "timeseries", "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum(rate({container=\"mail-poller\", mail_event=\"poll\"}[5m]))"}]},
     {"id": 3, "title": "Mail-poller ingest count", "type": "timeseries", "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum(rate({container=\"mail-poller\", mail_event=\"ingest\"}[5m]))"}]},
     {"id": 4, "title": "Pipeline errors", "type": "timeseries", "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum by (pipeline_name) (rate({container=\"api\", api_event=\"error\"}[5m]))"}]},

infra/grafana/dashboards/llm.json

@@ -28,8 +28,8 @@
       },
       "targets": [
         {
-          "expr": "sum by (status) (rate(stack_http_server_requests_total{service_name=\"llm\"}[1m]))",
-          "legendFormat": "{{status}}"
+          "expr": "sum by (http_status_code) (rate(http_server_duration_milliseconds_count{job=\"llm\"}[1m]))",
+          "legendFormat": "{{http_status_code}}"
         }
       ]
     },

infra/images/api.Dockerfile

@@ -26,7 +26,7 @@ ENV UV_PYTHON_PREFERENCE=only-system \
     UV_LINK_MODE=copy \
     UV_PROJECT_ENVIRONMENT=.venv \
     UV_INDEX_URL=${PYPI_INDEX_URL}
-RUN uv sync --no-dev --extra cli && uv pip install -e .
+RUN uv sync --no-dev --extra cli --extra perf && uv pip install -e .
 
 # Config
 COPY stack.toml ./

infra/images/llm.Dockerfile

@@ -17,7 +17,7 @@ ENV UV_PYTHON_PREFERENCE=only-system \
     UV_LINK_MODE=copy \
     UV_PROJECT_ENVIRONMENT=.venv \
     UV_INDEX_URL=${PYPI_INDEX_URL}
-RUN uv sync --no-dev --extra llm && uv pip install -e .
+RUN uv sync --no-dev --extra llm --extra perf && uv pip install -e .
 
 # Config (conf reads stack.toml by walking up from CWD)
 COPY stack.toml ./

infra/prometheus/targets/services.yml

@@ -45,3 +45,16 @@
 - targets: ['otel-collector:8889']
   labels:
     job: otel-collector
+
+# Stack API + LLM service — perf.middleware serves the OTel Prometheus
+# registry at /metrics (#579): http_server_* from the FastAPI
+# instrumentor, stack_llm_* dispatch/embedding/chat/index instruments.
+- targets: ['api:8000']
+  labels:
+    job: api
+    __metrics_path__: /metrics
+
+- targets: ['llm:8000']
+  labels:
+    job: llm
+    __metrics_path__: /metrics

src/perf/middleware.py

@@ -29,3 +29,32 @@ def instrument(app: Any) -> None:
         FastAPIInstrumentor.instrument_app(app)
     except Exception:
         pass
+    _mount_metrics(app)
+
+
+def _mount_metrics(app: Any) -> None:
+    """Serve the process's Prometheus registry at ``GET /metrics``.
+
+    ``perf._meter.setup_meter_provider`` exports through a
+    ``PrometheusMetricReader``, which only fills the in-process
+    ``prometheus_client`` registry — nothing leaves the process unless
+    something serves it. Until #579 nothing did (the api and llm
+    containers answered 404 on /metrics and no ``stack_*`` series ever
+    reached Prometheus), so the dashboards' request-rate panels were
+    empty from the day they were written. Prometheus scrapes this route
+    directly (infra/prometheus/targets/services.yml), bypassing the
+    otel-collector's ``namespace: stack`` re-prefixing.
+    """
+    try:
+        from fastapi import Response
+        from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, generate_latest
+    except Exception:
+        return
+
+    def metrics() -> Any:
+        return Response(generate_latest(REGISTRY), media_type=CONTENT_TYPE_LATEST)
+
+    try:
+        app.add_api_route("/metrics", metrics, methods=["GET"], include_in_schema=False)
+    except Exception:
+        pass

tests/perf/test_integration.py

@@ -176,3 +176,62 @@ class TestMiddleware:
 
         assert app is not None
         assert app.title == "stack"
+
+
+class TestMetricsRoute:
+    """#579: instrument() serves the in-process Prometheus registry at
+    /metrics when telemetry is enabled — the only way stack_* series
+    reach Prometheus (the meter's PrometheusMetricReader never pushes)."""
+
+    def test_metrics_route_served_when_enabled(self, monkeypatch):
+        monkeypatch.setenv("STACK_TELEMETRY", "true")
+        from fastapi import FastAPI
+        from fastapi.testclient import TestClient
+
+        from perf.middleware import instrument
+
+        app = FastAPI()
+        instrument(app)
+        r = TestClient(app).get("/metrics")
+        assert r.status_code == 200
+        assert r.headers["content-type"].startswith("text/plain")
+        assert "# HELP" in r.text or "# TYPE" in r.text
+
+    def test_metrics_route_absent_when_disabled(self, monkeypatch):
+        monkeypatch.setenv("STACK_TELEMETRY", "false")
+        from fastapi import FastAPI
+        from fastapi.testclient import TestClient
+
+        from perf.middleware import instrument
+
+        app = FastAPI()
+        instrument(app)
+        assert TestClient(app).get("/metrics").status_code == 404
+
+    def test_mount_metrics_tolerates_missing_prometheus_client(self, monkeypatch):
+        import builtins
+
+        real = builtins.__import__
+
+        def fake(name, *a, **k):
+            if name.startswith("prometheus_client"):
+                raise ImportError(name)
+            return real(name, *a, **k)
+
+        monkeypatch.setattr(builtins, "__import__", fake)
+        from unittest.mock import MagicMock
+
+        from perf.middleware import _mount_metrics
+
+        app = MagicMock()
+        _mount_metrics(app)  # no route added, no exception
+        app.add_api_route.assert_not_called()
+
+    def test_mount_metrics_tolerates_route_failure(self):
+        from unittest.mock import MagicMock
+
+        from perf.middleware import _mount_metrics
+
+        app = MagicMock()
+        app.add_api_route.side_effect = RuntimeError("boom")
+        _mount_metrics(app)  # swallowed

tests/test_observability.py

@@ -307,7 +307,7 @@ class TestLlmDashboard:
             t["expr"] for p in self._dash()["panels"] for t in p.get("targets", [])
         )
         for metric in (
-            "stack_http_server_requests_total",
+            "http_server_duration_milliseconds_count",  # what the FastAPI instrumentor emits
             "stack_llm_dispatch_total",
             "stack_llm_inflight",
             "stack_llm_dispatch_seconds_bucket",