diff --git a/compose.yml b/compose.yml index 51eb787..8d6fce0 100644 --- a/compose.yml +++ b/compose.yml @@ -649,7 +649,9 @@ services: - S3_ENDPOINT=${S3_ENDPOINT:-http://rustfs:9000} - S3_REGION=${S3_REGION:-us-east-1} - S3_WAREHOUSE=${S3_WAREHOUSE:-s3://lakehouse/} - # OpenTelemetry + # OpenTelemetry — traces go to the collector; metrics are served + # at /metrics and scraped directly (#579). + - STACK_TELEMETRY=true - OTEL_SERVICE_NAME=api - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 volumes: @@ -694,6 +696,7 @@ services: - LLM_PG_HOST=postgres - LLM_DB_PASSWORD=${LLM_DB_PASSWORD} - LLM_DUCKDB_REPLICA=/app/data/replica/aco.ro.duckdb + - STACK_TELEMETRY=true # /metrics scraped by prometheus (#579) - OTEL_SERVICE_NAME=llm - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 depends_on: diff --git a/infra/grafana/dashboards/data-pipelines.json b/infra/grafana/dashboards/data-pipelines.json index 1773d09..4b8fb8b 100644 --- a/infra/grafana/dashboards/data-pipelines.json +++ b/infra/grafana/dashboards/data-pipelines.json @@ -7,7 +7,7 @@ "time": {"from": "now-6h", "to": "now"}, "refresh": "30s", "panels": [ - {"id": 1, "title": "API request rate (status)", "type": "timeseries", "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "sum by (status) (rate(stack_http_server_requests_total[1m]))"}]}, + {"id": 1, "title": "API request rate (status)", "type": "timeseries", "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, "datasource": {"uid": "prometheus"}, "targets": [{"expr": "sum by (http_status_code) (rate(http_server_duration_milliseconds_count{job=\"api\"}[1m]))"}]}, {"id": 2, "title": "Mail-poller poll cycles", "type": "timeseries", "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum(rate({container=\"mail-poller\", mail_event=\"poll\"}[5m]))"}]}, {"id": 3, "title": "Mail-poller ingest count", "type": "timeseries", "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum(rate({container=\"mail-poller\", mail_event=\"ingest\"}[5m]))"}]}, {"id": 4, "title": "Pipeline errors", "type": "timeseries", "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, "datasource": {"uid": "loki"}, "targets": [{"expr": "sum by (pipeline_name) (rate({container=\"api\", api_event=\"error\"}[5m]))"}]}, diff --git a/infra/grafana/dashboards/llm.json b/infra/grafana/dashboards/llm.json index b96c971..a2e8e20 100644 --- a/infra/grafana/dashboards/llm.json +++ b/infra/grafana/dashboards/llm.json @@ -28,8 +28,8 @@ }, "targets": [ { - "expr": "sum by (status) (rate(stack_http_server_requests_total{service_name=\"llm\"}[1m]))", - "legendFormat": "{{status}}" + "expr": "sum by (http_status_code) (rate(http_server_duration_milliseconds_count{job=\"llm\"}[1m]))", + "legendFormat": "{{http_status_code}}" } ] }, diff --git a/infra/images/api.Dockerfile b/infra/images/api.Dockerfile index d26ca6a..1336063 100644 --- a/infra/images/api.Dockerfile +++ b/infra/images/api.Dockerfile @@ -26,7 +26,7 @@ ENV UV_PYTHON_PREFERENCE=only-system \ UV_LINK_MODE=copy \ UV_PROJECT_ENVIRONMENT=.venv \ UV_INDEX_URL=${PYPI_INDEX_URL} -RUN uv sync --no-dev --extra cli && uv pip install -e . +RUN uv sync --no-dev --extra cli --extra perf && uv pip install -e . # Config COPY stack.toml ./ diff --git a/infra/images/llm.Dockerfile b/infra/images/llm.Dockerfile index 1cf0eca..f074730 100644 --- a/infra/images/llm.Dockerfile +++ b/infra/images/llm.Dockerfile @@ -17,7 +17,7 @@ ENV UV_PYTHON_PREFERENCE=only-system \ UV_LINK_MODE=copy \ UV_PROJECT_ENVIRONMENT=.venv \ UV_INDEX_URL=${PYPI_INDEX_URL} -RUN uv sync --no-dev --extra llm && uv pip install -e . +RUN uv sync --no-dev --extra llm --extra perf && uv pip install -e . # Config (conf reads stack.toml by walking up from CWD) COPY stack.toml ./ diff --git a/infra/prometheus/targets/services.yml b/infra/prometheus/targets/services.yml index ec513ea..53308aa 100644 --- a/infra/prometheus/targets/services.yml +++ b/infra/prometheus/targets/services.yml @@ -45,3 +45,16 @@ - targets: ['otel-collector:8889'] labels: job: otel-collector + +# Stack API + LLM service — perf.middleware serves the OTel Prometheus +# registry at /metrics (#579): http_server_* from the FastAPI +# instrumentor, stack_llm_* dispatch/embedding/chat/index instruments. +- targets: ['api:8000'] + labels: + job: api + __metrics_path__: /metrics + +- targets: ['llm:8000'] + labels: + job: llm + __metrics_path__: /metrics diff --git a/src/perf/middleware.py b/src/perf/middleware.py index 045af2a..fd8e81e 100644 --- a/src/perf/middleware.py +++ b/src/perf/middleware.py @@ -29,3 +29,32 @@ def instrument(app: Any) -> None: FastAPIInstrumentor.instrument_app(app) except Exception: pass + _mount_metrics(app) + + +def _mount_metrics(app: Any) -> None: + """Serve the process's Prometheus registry at ``GET /metrics``. + + ``perf._meter.setup_meter_provider`` exports through a + ``PrometheusMetricReader``, which only fills the in-process + ``prometheus_client`` registry — nothing leaves the process unless + something serves it. Until #579 nothing did (the api and llm + containers answered 404 on /metrics and no ``stack_*`` series ever + reached Prometheus), so the dashboards' request-rate panels were + empty from the day they were written. Prometheus scrapes this route + directly (infra/prometheus/targets/services.yml), bypassing the + otel-collector's ``namespace: stack`` re-prefixing. + """ + try: + from fastapi import Response + from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, generate_latest + except Exception: + return + + def metrics() -> Any: + return Response(generate_latest(REGISTRY), media_type=CONTENT_TYPE_LATEST) + + try: + app.add_api_route("/metrics", metrics, methods=["GET"], include_in_schema=False) + except Exception: + pass diff --git a/tests/perf/test_integration.py b/tests/perf/test_integration.py index e4398be..2096621 100644 --- a/tests/perf/test_integration.py +++ b/tests/perf/test_integration.py @@ -176,3 +176,62 @@ class TestMiddleware: assert app is not None assert app.title == "stack" + + +class TestMetricsRoute: + """#579: instrument() serves the in-process Prometheus registry at + /metrics when telemetry is enabled — the only way stack_* series + reach Prometheus (the meter's PrometheusMetricReader never pushes).""" + + def test_metrics_route_served_when_enabled(self, monkeypatch): + monkeypatch.setenv("STACK_TELEMETRY", "true") + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from perf.middleware import instrument + + app = FastAPI() + instrument(app) + r = TestClient(app).get("/metrics") + assert r.status_code == 200 + assert r.headers["content-type"].startswith("text/plain") + assert "# HELP" in r.text or "# TYPE" in r.text + + def test_metrics_route_absent_when_disabled(self, monkeypatch): + monkeypatch.setenv("STACK_TELEMETRY", "false") + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from perf.middleware import instrument + + app = FastAPI() + instrument(app) + assert TestClient(app).get("/metrics").status_code == 404 + + def test_mount_metrics_tolerates_missing_prometheus_client(self, monkeypatch): + import builtins + + real = builtins.__import__ + + def fake(name, *a, **k): + if name.startswith("prometheus_client"): + raise ImportError(name) + return real(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", fake) + from unittest.mock import MagicMock + + from perf.middleware import _mount_metrics + + app = MagicMock() + _mount_metrics(app) # no route added, no exception + app.add_api_route.assert_not_called() + + def test_mount_metrics_tolerates_route_failure(self): + from unittest.mock import MagicMock + + from perf.middleware import _mount_metrics + + app = MagicMock() + app.add_api_route.side_effect = RuntimeError("boom") + _mount_metrics(app) # swallowed diff --git a/tests/test_observability.py b/tests/test_observability.py index f91a976..00d626a 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -307,7 +307,7 @@ class TestLlmDashboard: t["expr"] for p in self._dash()["panels"] for t in p.get("targets", []) ) for metric in ( - "stack_http_server_requests_total", + "http_server_duration_milliseconds_count", # what the FastAPI instrumentor emits "stack_llm_dispatch_total", "stack_llm_inflight", "stack_llm_dispatch_seconds_bucket",