stack/tests/test_observability.py

"""Compose-level invariants for the observability port.

refs docs/superpowers/specs/2026-05-01-observability-port-design.md
"""

from __future__ import annotations

import json
from pathlib import Path

import yaml

ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ROOT / "compose.yml"


def _load() -> dict:
    return yaml.safe_load(COMPOSE.read_text())


def _services() -> dict:
    return _load().get("services", {})


def test_compose_has_services():
    assert _services(), "compose.yml has no services"


class TestTempo:
    def test_tempo_service_present(self):
        svcs = _services()
        assert "tempo" in svcs
        assert svcs["tempo"]["networks"] == ["observability"]

    def test_tempo_volume_declared(self):
        vols = _load().get("volumes", {})
        assert "tempo_data" in vols

    def test_tempo_config_mounted(self):
        svc = _services()["tempo"]
        mounts = [v for v in svc["volumes"] if "/etc/tempo" in v]
        assert any("./infra/tempo/tempo.yml" in v for v in mounts)


class TestNvidiaExporter:
    def test_nvidia_exporter_present(self):
        svcs = _services()
        assert "nvidia-exporter" in svcs

    def test_nvidia_runtime(self):
        svc = _services()["nvidia-exporter"]
        assert svc.get("runtime") == "nvidia"

    def test_nvidia_gpu_reservation(self):
        svc = _services()["nvidia-exporter"]
        devices = svc["deploy"]["resources"]["reservations"]["devices"]
        assert any(d.get("driver") == "nvidia" for d in devices)


class TestLokiRetention:
    def test_loki_compactor_configured(self):
        cfg = yaml.safe_load((ROOT / "infra/loki/loki-config.yml").read_text())
        assert cfg["compactor"]["retention_enabled"] is True
        assert cfg["limits_config"]["retention_period"] == "168h"


class TestPromtailRewire:
    def test_promtail_uses_docker_sock(self):
        svc = _services()["promtail"]
        vols = [v for v in svc["volumes"] if isinstance(v, str)]
        # New: docker.sock; old filesystem mount must be gone.
        assert any("docker.sock" in v for v in vols)
        assert not any("/var/lib/docker/containers" in v for v in vols)

    def test_promtail_has_docker_gid(self):
        svc = _services()["promtail"]
        assert "group_add" in svc

    def test_promtail_config_is_docker_sd(self):
        cfg = yaml.safe_load((ROOT / "infra/loki/promtail-config.yml").read_text())
        jobs = cfg["scrape_configs"]
        docker_jobs = [j for j in jobs if j.get("job_name") == "docker"]
        assert docker_jobs, "expected job_name=docker"
        assert "docker_sd_configs" in docker_jobs[0]


class TestPerServiceStages:
    """Each named container must have a matching pipeline stage that emits at
    least one service-specific label."""

    EXPECTED = {
        "traefik": "tf_event",
        "git": "gitea_event",
        "oauth2-proxy": "oauth_event",
        "postgres": "pg_event",
        "trino": "trino_event",
        "rustfs": "s3_op",
        "api": "api_event",
        "mail-poller": "mail_event",
        "act-runner": "ci_event",
        "cloudflared": "cf_event",
    }

    def test_each_service_has_a_match_stage(self):
        cfg = yaml.safe_load((ROOT / "infra/loki/promtail-config.yml").read_text())
        stages = cfg["scrape_configs"][0]["pipeline_stages"]
        text = yaml.safe_dump(stages)
        for container, label in self.EXPECTED.items():
            assert f'container="{container}"' in text, f"no match stage for {container}"
            assert label in text, f"no {label} extraction for {container}"

    def test_quarkus_pair_has_combined_stage(self):
        # nessie + polaris share one stage via regex selector.
        cfg = yaml.safe_load((ROOT / "infra/loki/promtail-config.yml").read_text())
        text = yaml.safe_dump(cfg["scrape_configs"][0]["pipeline_stages"])
        assert 'container=~"nessie|polaris"' in text
        assert "quarkus_route" in text


class TestOtelTempo:
    def test_otel_exports_traces_to_tempo(self):
        cfg = yaml.safe_load((ROOT / "infra/otel/otel-collector.yml").read_text())
        exporters = cfg["exporters"]
        assert "otlp_http/tempo" in exporters
        assert exporters["otlp_http/tempo"]["endpoint"] == "http://tempo:4318"
        assert "otlp_grpc/jaeger" not in exporters
        traces = cfg["service"]["pipelines"]["traces"]
        assert "otlp_http/tempo" in traces["exporters"]
        assert "memory_limiter" in traces["processors"]


class TestPrometheusTargets:
    def test_targets_swap_jaeger_for_tempo_and_add_exporters(self):
        path = ROOT / "infra/prometheus/targets/services.yml"
        targets = yaml.safe_load(path.read_text())
        jobs = {entry["labels"]["job"] for entry in targets if "labels" in entry}
        assert "tempo" in jobs
        assert "nvidia-gpu" in jobs
        assert "jaeger" not in jobs


class TestGrafanaDatasources:
    def test_datasources_have_tempo_loki_default_correlation(self):
        path = ROOT / "infra/grafana/provisioning/datasources/datasources.yml"
        cfg = yaml.safe_load(path.read_text())
        ds = {d["name"]: d for d in cfg["datasources"]}
        assert "Tempo" in ds and "Jaeger" not in ds
        assert ds["Loki"].get("isDefault") is True
        tempo = ds["Tempo"]
        assert tempo["jsonData"]["tracesToLogsV2"]["datasourceUid"] == "loki"
        assert tempo["jsonData"]["serviceMap"]["datasourceUid"] == "prometheus"


PROMTAIL_LABELED = {
    "coredns",
    "traefik",
    "rustfs",
    "postgres",
    "git",
    "act-runner",
    "notebooks",
    "zotero",
    "webdav",
    "nessie",
    "trino",
    "polaris",
    "dashboard",
    "docs",
    "api",
    "mail-poller",
    "auth-handler",
    "oauth2-proxy",
    "cloudflared",
    "loki",
    "promtail",
    "tempo",
    "otel-collector",
    "prometheus",
    "grafana",
    "nvidia-exporter",
}

PROMTAIL_NOT_LABELED = {"mc", "wire"}


class TestPromtailLabels:
    def test_eligible_services_have_promtail_label(self):
        svcs = _services()
        missing = []
        for name in PROMTAIL_LABELED:
            if name not in svcs:
                missing.append(f"  {name}: not in compose")
                continue
            labels = svcs[name].get("labels", []) or []
            if "promtail=true" not in labels:
                missing.append(f"  {name}: missing promtail=true label")
        assert not missing, "\n".join(missing)

    def test_excluded_services_do_not_have_label(self):
        svcs = _services()
        for name in PROMTAIL_NOT_LABELED:
            if name not in svcs:
                continue
            labels = svcs[name].get("labels", []) or []
            assert "promtail=true" not in labels, f"{name} should NOT be labeled"

    def test_traefik_access_logs_enabled(self):
        traefik = _services()["traefik"]
        cmd = traefik.get("command", [])
        if isinstance(cmd, str):
            cmd = cmd.split()
        flat = " ".join(str(x) for x in cmd)
        assert "--accesslog" in flat or any(
            "TRAEFIK_ACCESSLOG" in str(e) for e in traefik.get("environment", [])
        )


class TestGrafanaOAuth:
    def test_grafana_loads_state_env_file(self):
        env_files = _services()["grafana"].get("env_file", [])
        # accepts either a list of dicts or a list of strings
        joined = yaml.safe_dump(env_files)
        assert ".state/gitea/grafana.env" in joined

    def test_grafana_oauth_env_vars(self):
        envs = _services()["grafana"].get("environment", [])
        text = "\n".join(envs)
        assert "GF_AUTH_GENERIC_OAUTH_ENABLED=true" in text
        assert "GF_AUTH_GENERIC_OAUTH_NAME=Gitea" in text
        assert (
            "GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=is_admin && 'GrafanaAdmin'"
            in text
        )

    def test_bootstrap_sso_provisions_grafana(self):
        src = (ROOT / "dev/scripts/bootstrap_sso.py").read_text()
        assert "GRAFANA_ENV" in src
        assert 'name="grafana"' in src
        assert "/login/generic_oauth" in src


class TestSubdomainSwap:
    def test_stack_toml_subdomains(self):
        import tomllib

        cfg = tomllib.loads((ROOT / "stack.toml").read_text())
        subs = cfg["platform"]["subdomains"]
        assert "tempo" in subs
        assert "jaeger" not in subs

    def test_bootstrap_sso_subdomains(self):
        src = (ROOT / "dev/scripts/bootstrap_sso.py").read_text()
        assert '"tempo"' in src
        assert '"jaeger"' not in src

    def test_traefik_route_swap(self):
        src = (ROOT / "infra/traefik/dynamic/services.yml").read_text()
        assert '"tempo"' in src
        assert '"jaeger"' not in src


class TestJaegerRemoved:
    def test_jaeger_service_gone(self):
        assert "jaeger" not in _services()


EXPECTED_DASHBOARDS = {
    "homelab-overview",
    "gateway-auth",
    "data-lake",
    "data-pipelines",
    "ci",
    "gpu-notebooks",
    "llm",
}


class TestDashboards:
    def test_all_dashboards_exist_and_parse(self):
        dash_dir = ROOT / "infra/grafana/dashboards"
        files = {p.stem for p in dash_dir.glob("*.json")}
        missing = EXPECTED_DASHBOARDS - files
        assert not missing, f"missing dashboards: {missing}"
        for name in EXPECTED_DASHBOARDS:
            with (dash_dir / f"{name}.json").open() as f:
                data = json.load(f)
            assert "panels" in data, f"{name}: no panels[]"
            assert data["panels"], f"{name}: empty panels[]"


class TestLlmDashboard:
    """#579: the LLM service dashboard — panels for the metrics
    ``llm.metrics`` emits, on the provisioned prometheus datasource."""

    def _dash(self) -> dict:
        return json.loads((ROOT / "infra/grafana/dashboards/llm.json").read_text())

    def test_identity_matches_the_other_dashboards(self):
        d = self._dash()
        assert d["uid"] == "llm"
        assert d["title"] == "LLM service"
        assert d["schemaVersion"] == 39
        assert set(d["tags"]) == {"homelab", "stack"}

    def test_panels_cover_every_instrument(self):
        exprs = " ".join(
            t["expr"] for p in self._dash()["panels"] for t in p.get("targets", [])
        )
        for metric in (
            "http_server_duration_milliseconds_count",  # what the FastAPI instrumentor emits
            "stack_llm_dispatch_total",
            "stack_llm_inflight",
            "stack_llm_dispatch_seconds_bucket",
            "stack_llm_embedded_texts_total",
            "stack_llm_chat_seconds_bucket",
            "stack_llm_chat_total",
            "stack_llm_chat_tokens_total",
            "stack_llm_indexed_chunks_total",
            "stack_llm_indexed_items_total",
        ):
            assert metric in exprs, f"no panel queries {metric}"

    def test_panel_ids_unique_and_datasources_provisioned(self):
        panels = self._dash()["panels"]
        assert len({p["id"] for p in panels}) == len(panels)
        assert {p["datasource"]["uid"] for p in panels} <= {"prometheus", "loki"}

    def test_latency_panels_use_quantiles(self):
        exprs = [
            t["expr"] for p in self._dash()["panels"] for t in p.get("targets", [])
        ]
        quantiles = [e for e in exprs if e.startswith("histogram_quantile")]
        assert len(quantiles) == 4  # p50 + p95 for dispatch and for chat