fix(zotero): heal kasmxproxy display-bridge race + add real healthcheck/observability

zotero.fhirworx.io rendered a blank desktop: KDE+Zotero draw on Xvfb :20, but kasmvnc serves :21, and the kasmxproxy mirror that bridges them never started. The stock kasmvnc-entrypoint.sh blocks on `until [ -S /tmp/.X11-unix/X21 ]` (a filesystem socket path) while entrypoint.sh's concurrent `rm -rf /tmp/.X*` unlinks it — Xvnc keeps its fd + abstract socket open and serves a blank :21 forever. A startup race, hence the intermittent "doesn't open" with every process reporting healthy. - kasmxproxy-guard.sh: supervised watchdog that probes :20/:21 via xdpyinfo (not the racy socket file) and (re)launches kasmxproxy, self-healing if it dies. Emits structured `zotero_event=` logs to container stdout + a heartbeat file. - zotero-healthcheck.sh + compose healthcheck: detect the real failure mode (bridge down / :21 unreachable / KDE dead / stale heartbeat) that container/nginx/Xvnc liveness all miss. - promtail: extract zotero_event into a Loki label. - Grafana: Zotero Desktop dashboard + alert on sustained bridge failure. - otel-cli baked in; OTLP tracing dormant by default (enabling it requires attaching zotero to the observability network — documented opt-in, left off to preserve gateway-only isolation). Verified live: guard self-heals on kasmxproxy kill; healthcheck passes healthy / fails on stale heartbeat.
2026-06-17 19:49:25 -04:00
parent c0493a0b8f
commit 225ce01d6b
8 changed files with 371 additions and 0 deletions
--- a/compose.yml
+++ b/compose.yml
@@ -317,6 +317,13 @@ services:
      # the UI in a FATAL restart loop. "auto" tells kasmvnc to figure it
      # out itself.
      - TURN_EXTERNAL_IP=auto
+      # OTLP tracing of the desktop-bridge lifecycle is OFF by default. To
+      # enable: add `observability` to `networks` below and uncomment the
+      # endpoint. Left off deliberately — zotero is a GPU desktop kept on
+      # `gateway` only; attaching it to `observability` widens its blast
+      # radius. The structured Loki logs + Grafana alert cover the same
+      # failure without crossing that isolation boundary.
+      # - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
    deploy:
      resources:
        reservations:
@@ -324,6 +331,16 @@ services:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
+    # Detects the blank-desktop failure (dead :20->:21 kasmxproxy bridge) that
+    # container/nginx/Xvnc liveness all miss. The image bakes an equivalent
+    # HEALTHCHECK; this override tunes timing for the GPU desktop's slow boot
+    # (first start may install the NVIDIA userspace driver).
+    healthcheck:
+      test: ["CMD", "/usr/local/bin/zotero-healthcheck"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 180s
    labels:
      - "promtail=true"
    restart: unless-stopped
--- a/infra/grafana/dashboards/zotero-desktop.json
+++ b/infra/grafana/dashboards/zotero-desktop.json
@@ -0,0 +1,88 @@
+{
+  "title": "Zotero Desktop",
+  "uid": "zotero-desktop",
+  "tags": ["zotero", "desktop", "kasmvnc"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "time": { "from": "now-6h", "to": "now" },
+  "refresh": "30s",
+  "panels": [
+    {
+      "id": 1,
+      "title": "Bridge down events (1h)",
+      "type": "stat",
+      "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 },
+      "datasource": { "uid": "loki" },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"zotero\", zotero_event=\"bridge_down\"} [1h]))",
+          "queryType": "instant"
+        }
+      ],
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 1 }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": 2,
+      "title": "Bridge start failures (1h)",
+      "type": "stat",
+      "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 },
+      "datasource": { "uid": "loki" },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"zotero\", zotero_event=\"bridge_start_failed\"} [1h]))",
+          "queryType": "instant"
+        }
+      ],
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 1 }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": 3,
+      "title": "Desktop-bridge events",
+      "type": "timeseries",
+      "gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
+      "datasource": { "uid": "loki" },
+      "targets": [
+        {
+          "expr": "sum by (zotero_event) (count_over_time({container=\"zotero\"} | zotero_event=~\".+\" [5m]))",
+          "legendFormat": "{{zotero_event}}"
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "title": "Bridge + watchdog log tail",
+      "type": "logs",
+      "gridPos": { "x": 0, "y": 4, "w": 12, "h": 12 },
+      "datasource": { "uid": "loki" },
+      "targets": [
+        {
+          "expr": "{container=\"zotero\"} | zotero_event=~\".+\"",
+          "queryType": "range"
+        }
+      ],
+      "options": { "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true }
+    }
+  ]
+}
--- a/infra/grafana/provisioning/alerting/zotero.yml
+++ b/infra/grafana/provisioning/alerting/zotero.yml
@@ -0,0 +1,85 @@
+# Grafana unified-alerting provisioning for the Zotero desktop bridge.
+#
+# Fires when the kasmxproxy-guard watchdog reports the :20->:21 mirror as
+# genuinely failing (kasmxproxy won't start, or a display is down) for a
+# sustained window — i.e. the browser would see a blank desktop. The
+# watchdog's self-healing bridge_down -> bridge_up blips are intentionally
+# NOT alerted (they recover within seconds); we key off bridge_start_failed
+# and *_display_down, gated by `for: 5m`.
+#
+# NOTE: alert-rule provisioning JSON shifts between Grafana versions (this
+# stack tracks grafana:latest). Verify in the UI after deploy:
+#   Alerting -> Alert rules -> "Zotero desktop bridge failing".
+# Routing to an actual channel additionally needs a contact point /
+# notification policy; unrouted, the rule still shows as Firing in the UI.
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: zotero
+    folder: Zotero
+    interval: 1m
+    rules:
+      - uid: zotero-bridge-failing
+        title: Zotero desktop bridge failing
+        condition: C
+        for: 5m
+        noDataState: OK
+        execErrState: OK
+        labels:
+          severity: warning
+          service: zotero
+        annotations:
+          summary: >-
+            Zotero KDE desktop bridge (:20->:21) is failing — the browser sees a
+            blank screen at zotero.fhirworx.io.
+          description: >-
+            kasmxproxy-guard reported bridge_start_failed or a display down for
+            5m+. The KasmVNC display the browser connects to is not being
+            driven. Check `docker logs zotero` and the "Zotero Desktop"
+            dashboard.
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: loki
+            model:
+              refId: A
+              datasource:
+                type: loki
+                uid: loki
+              queryType: instant
+              expr: >-
+                sum(count_over_time({container="zotero",
+                zotero_event=~"bridge_start_failed|dst_display_down|src_display_down"}
+                [5m]))
+          - refId: B
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: B
+              type: reduce
+              reducer: last
+              expression: A
+              datasource:
+                type: __expr__
+                uid: __expr__
+          - refId: C
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: B
+              datasource:
+                type: __expr__
+                uid: __expr__
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [0]
--- a/infra/images/zotero.Dockerfile
+++ b/infra/images/zotero.Dockerfile
@@ -32,4 +32,36 @@ RUN mkdir -p /home/ubuntu/Desktop \
    && chmod +x /home/ubuntu/Desktop/zotero.desktop \
    && chown -R ubuntu:ubuntu /home/ubuntu/Desktop

+# Desktop-bridge watchdog + healthcheck deps.
+#   x11-utils → xdpyinfo (probe X reachability of :20/:21)
+#   procps    → pgrep    (detect kasmxproxy / plasmashell)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        x11-utils \
+        procps \
+    && rm -rf /var/lib/apt/lists/*
+
+# otel-cli: enables OTLP span emission from the bridge watchdog. Dormant by
+# default (the guard only emits when OTEL_EXPORTER_OTLP_ENDPOINT is set and
+# this container is attached to the `observability` network — see compose.yml).
+# Best-effort install so a release-asset hiccup never breaks the image build.
+ARG OTEL_CLI_VERSION=0.4.5
+RUN arch="$(dpkg --print-architecture | sed -e 's/amd64/amd64/' -e 's/arm64/arm64/')" \
+    && curl -fsSL "https://github.com/equinix-labs/otel-cli/releases/download/v${OTEL_CLI_VERSION}/otel-cli_${OTEL_CLI_VERSION}_linux_${arch}.tar.gz" \
+        -o /tmp/otel-cli.tgz \
+    && tar -xzf /tmp/otel-cli.tgz -C /usr/local/bin otel-cli \
+    && chmod +x /usr/local/bin/otel-cli \
+    && rm -f /tmp/otel-cli.tgz \
+    || echo 'WARN: otel-cli install failed; OTLP tracing will be unavailable'
+
+# Desktop-bridge watchdog (fixes the kasmxproxy startup race) + healthcheck.
+COPY infra/images/zotero/kasmxproxy-guard.sh  /usr/local/bin/kasmxproxy-guard.sh
+COPY infra/images/zotero/zotero-healthcheck.sh /usr/local/bin/zotero-healthcheck
+COPY infra/images/zotero/kasmxproxy-guard.conf /etc/supervisor/conf.d/kasmxproxy-guard.conf
+RUN chmod +x /usr/local/bin/kasmxproxy-guard.sh /usr/local/bin/zotero-healthcheck
+
+# Detect a blank desktop (dead :20->:21 bridge), which container/nginx/Xvnc
+# liveness all miss. Compose overrides interval/start_period as needed.
+HEALTHCHECK --interval=30s --timeout=10s --start-period=180s --retries=5 \
+    CMD /usr/local/bin/zotero-healthcheck
+
 USER ubuntu
--- a/infra/images/zotero/kasmxproxy-guard.conf
+++ b/infra/images/zotero/kasmxproxy-guard.conf
@@ -0,0 +1,20 @@
+; Supervised watchdog that keeps the KDE desktop (:20) mirrored onto the
+; KasmVNC display (:21). Fixes the startup race where the stock
+; kasmvnc-entrypoint.sh never launches kasmxproxy. See kasmxproxy-guard.sh.
+;
+; priority 25 > kasmvnc's 20 so we start after Xvnc; autorestart keeps the
+; watchdog itself alive. Logs go to /dev/fd/1 (the container's stdout) so
+; promtail -> Loki picks them up — supervisord child logs default to files
+; under /tmp that never reach `docker logs`.
+[program:kasmxproxy-guard]
+command=/usr/local/bin/kasmxproxy-guard.sh
+autostart=true
+autorestart=true
+startsecs=3
+startretries=9999
+stopasgroup=true
+killasgroup=true
+priority=25
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
--- a/infra/images/zotero/kasmxproxy-guard.sh
+++ b/infra/images/zotero/kasmxproxy-guard.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+# kasmxproxy-guard — keep the KDE desktop (Xvfb :20) mirrored onto the
+# KasmVNC display (:21) that the browser actually connects to.
+#
+# Why this exists
+# ---------------
+# The stock kasmvnc-entrypoint.sh launches kasmxproxy (the :20->:21 mirror)
+# only after `until [ -S /tmp/.X11-unix/X21 ]` — a guard that tests the
+# *filesystem* socket path. entrypoint.sh runs `rm -rf /tmp/.X*` concurrently
+# (both are supervisord programs with no ordering barrier between them) and
+# unlinks /tmp/.X11-unix/X21 while Xvnc keeps its fd + abstract socket
+# (@/tmp/.X11-unix/X21) open and fully serving. The stock guard then loops
+# forever, kasmxproxy never starts, and :21 stays blank — the desktop
+# "doesn't open" even though every process reports healthy. It's a startup
+# race, so it's intermittent.
+#
+# This watchdog ignores the socket *file* and probes X reachability directly
+# (xdpyinfo), (re)launching kasmxproxy whenever both displays are up. It also
+# emits structured, single-line logs to stdout (scraped by promtail -> Loki)
+# and maintains a heartbeat file consumed by the container healthcheck.
+
+set -u
+
+SRC_DISPLAY="${DISPLAY:-:20}"             # where KDE actually renders (Xvfb)
+DST_DISPLAY="${KASMVNC_DISPLAY:-:21}"     # what KasmVNC serves to the browser
+FPS="${DISPLAY_REFRESH:-60}"
+HEARTBEAT="${ZOTERO_HEARTBEAT:-/tmp/kasmxproxy-guard.heartbeat}"
+POLL="${ZOTERO_GUARD_POLL:-5}"
+OTEL_ENDPOINT="${OTEL_EXPORTER_OTLP_ENDPOINT:-}"
+
+# Structured, promtail-friendly: `level=` and `zotero_event=` are extracted
+# into Loki labels by infra/loki/promtail-config.yml.
+log() {
+  local level="$1" event="$2"; shift 2
+  printf 'level=%s zotero_event=%s %s\n' "$level" "$event" "$*"
+}
+
+# Dormant unless an OTLP endpoint is configured AND otel-cli is installed.
+# Enabling tracing also requires attaching this container to the
+# `observability` network — see compose.yml. Best-effort; never fatal.
+emit_span() {
+  [ -n "$OTEL_ENDPOINT" ] || return 0
+  command -v otel-cli >/dev/null 2>&1 || return 0
+  otel-cli span \
+    --service zotero \
+    --name "$1" \
+    --kind internal \
+    --attrs "zotero.event=$1,zotero.src_display=${SRC_DISPLAY},zotero.dst_display=${DST_DISPLAY}" \
+    --endpoint "$OTEL_ENDPOINT" --protocol grpc --timeout 2s >/dev/null 2>&1 || true
+}
+
+x_reachable() { DISPLAY="$1" xdpyinfo >/dev/null 2>&1; }
+
+# A live KDE session means something is actually drawing on :20. Without this
+# the bridge could be "up" while mirroring a blank screen (KDE crash).
+session_alive() {
+  pgrep -x plasmashell >/dev/null 2>&1 || pgrep -x kwin_x11 >/dev/null 2>&1
+}
+
+start_bridge() {
+  log error bridge_down "msg=\"kasmxproxy not running; starting it\" src=${SRC_DISPLAY} dst=${DST_DISPLAY}"
+  emit_span bridge_down
+  DISPLAY="$SRC_DISPLAY" kasmxproxy -a "$SRC_DISPLAY" -v "$DST_DISPLAY" -f "$FPS" \
+    >/tmp/kasmxproxy.log 2>&1 &
+  sleep 2
+  if pgrep -x kasmxproxy >/dev/null 2>&1; then
+    log info bridge_up "pid=$(pgrep -x kasmxproxy | head -n1)"
+    emit_span bridge_up
+  else
+    log error bridge_start_failed "msg=\"kasmxproxy exited immediately; see /tmp/kasmxproxy.log\""
+  fi
+}
+
+log info guard_start "msg=\"watching ${SRC_DISPLAY}->${DST_DISPLAY}\" poll=${POLL}s"
+
+while :; do
+  if ! x_reachable "$SRC_DISPLAY"; then
+    log warn src_display_down "display=${SRC_DISPLAY}"
+  elif ! x_reachable "$DST_DISPLAY"; then
+    log warn dst_display_down "display=${DST_DISPLAY}"
+  elif ! pgrep -x kasmxproxy >/dev/null 2>&1; then
+    start_bridge
+  fi
+
+  # Heartbeat reflects a *fully* healthy bridge: proxy running, the display
+  # the browser sees is reachable, and a real session is drawing to it.
+  if pgrep -x kasmxproxy >/dev/null 2>&1 && x_reachable "$DST_DISPLAY" && session_alive; then
+    : > "$HEARTBEAT"
+  fi
+
+  sleep "$POLL"
+done
--- a/infra/images/zotero/zotero-healthcheck.sh
+++ b/infra/images/zotero/zotero-healthcheck.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Zotero container healthcheck.
+#
+# The container is only "healthy" if the desktop the browser actually sees
+# (KasmVNC display :21) is being driven. This probes the real failure mode the
+# stock checks miss: a blank :21 because the :20->:21 kasmxproxy bridge died
+# (see kasmxproxy-guard.sh). Container-up / nginx-up / Xvnc-up are all true in
+# that failure, which is exactly why it went unnoticed.
+set -u
+
+DST_DISPLAY="${KASMVNC_DISPLAY:-:21}"
+HEARTBEAT="${ZOTERO_HEARTBEAT:-/tmp/kasmxproxy-guard.heartbeat}"
+MAX_AGE="${ZOTERO_HEARTBEAT_MAX_AGE:-30}"
+
+fail() { echo "unhealthy: $1" >&2; exit 1; }
+
+pgrep -x kasmxproxy >/dev/null 2>&1 || fail "kasmxproxy (desktop bridge) not running"
+DISPLAY="$DST_DISPLAY" xdpyinfo >/dev/null 2>&1 || fail "KasmVNC display ${DST_DISPLAY} unreachable"
+pgrep -x plasmashell >/dev/null 2>&1 || fail "KDE session (plasmashell) not running"
+[ -f "$HEARTBEAT" ] || fail "guard heartbeat missing"
+
+age=$(( $(date +%s) - $(stat -c %Y "$HEARTBEAT") ))
+[ "$age" -lt "$MAX_AGE" ] || fail "guard heartbeat stale (${age}s >= ${MAX_AGE}s)"
+
+echo "healthy: bridge up, ${DST_DISPLAY} reachable, session alive (heartbeat ${age}s)"
--- a/infra/loki/promtail-config.yml
+++ b/infra/loki/promtail-config.yml
@@ -151,6 +151,18 @@ scrape_configs:
                mail_event:
                mail_count:

+      # --- zotero (KDE desktop / kasmxproxy bridge watchdog) ---
+      - match:
+          selector: '{container="zotero"}'
+          stages:
+            # kasmxproxy-guard.sh emits `zotero_event=<state>` on each bridge
+            # transition (bridge_up / bridge_down / bridge_start_failed /
+            # src_display_down / dst_display_down / guard_start).
+            - regex:
+                expression: 'zotero_event=(?P<zotero_event>\w+)'
+            - labels:
+                zotero_event:
+
      # --- act-runner ---
      - match:
          selector: '{container="act-runner"}'