225ce01d6b
zotero.fhirworx.io rendered a blank desktop: KDE+Zotero draw on Xvfb :20, but kasmvnc serves :21, and the kasmxproxy mirror that bridges them never started. The stock kasmvnc-entrypoint.sh blocks on `until [ -S /tmp/.X11-unix/X21 ]` (a filesystem socket path) while entrypoint.sh's concurrent `rm -rf /tmp/.X*` unlinks it — Xvnc keeps its fd + abstract socket open and serves a blank :21 forever. A startup race, hence the intermittent "doesn't open" with every process reporting healthy. - kasmxproxy-guard.sh: supervised watchdog that probes :20/:21 via xdpyinfo (not the racy socket file) and (re)launches kasmxproxy, self-healing if it dies. Emits structured `zotero_event=` logs to container stdout + a heartbeat file. - zotero-healthcheck.sh + compose healthcheck: detect the real failure mode (bridge down / :21 unreachable / KDE dead / stale heartbeat) that container/nginx/Xvnc liveness all miss. - promtail: extract zotero_event into a Loki label. - Grafana: Zotero Desktop dashboard + alert on sustained bridge failure. - otel-cli baked in; OTLP tracing dormant by default (enabling it requires attaching zotero to the observability network — documented opt-in, left off to preserve gateway-only isolation). Verified live: guard self-heals on kasmxproxy kill; healthcheck passes healthy / fails on stale heartbeat.
93 lines
3.7 KiB
Bash
Executable File
93 lines
3.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# kasmxproxy-guard — keep the KDE desktop (Xvfb :20) mirrored onto the
|
|
# KasmVNC display (:21) that the browser actually connects to.
|
|
#
|
|
# Why this exists
|
|
# ---------------
|
|
# The stock kasmvnc-entrypoint.sh launches kasmxproxy (the :20->:21 mirror)
|
|
# only after `until [ -S /tmp/.X11-unix/X21 ]` — a guard that tests the
|
|
# *filesystem* socket path. entrypoint.sh runs `rm -rf /tmp/.X*` concurrently
|
|
# (both are supervisord programs with no ordering barrier between them) and
|
|
# unlinks /tmp/.X11-unix/X21 while Xvnc keeps its fd + abstract socket
|
|
# (@/tmp/.X11-unix/X21) open and fully serving. The stock guard then loops
|
|
# forever, kasmxproxy never starts, and :21 stays blank — the desktop
|
|
# "doesn't open" even though every process reports healthy. It's a startup
|
|
# race, so it's intermittent.
|
|
#
|
|
# This watchdog ignores the socket *file* and probes X reachability directly
|
|
# (xdpyinfo), (re)launching kasmxproxy whenever both displays are up. It also
|
|
# emits structured, single-line logs to stdout (scraped by promtail -> Loki)
|
|
# and maintains a heartbeat file consumed by the container healthcheck.
|
|
|
|
set -u
|
|
|
|
SRC_DISPLAY="${DISPLAY:-:20}" # where KDE actually renders (Xvfb)
|
|
DST_DISPLAY="${KASMVNC_DISPLAY:-:21}" # what KasmVNC serves to the browser
|
|
FPS="${DISPLAY_REFRESH:-60}"
|
|
HEARTBEAT="${ZOTERO_HEARTBEAT:-/tmp/kasmxproxy-guard.heartbeat}"
|
|
POLL="${ZOTERO_GUARD_POLL:-5}"
|
|
OTEL_ENDPOINT="${OTEL_EXPORTER_OTLP_ENDPOINT:-}"
|
|
|
|
# Structured, promtail-friendly: `level=` and `zotero_event=` are extracted
|
|
# into Loki labels by infra/loki/promtail-config.yml.
|
|
log() {
|
|
local level="$1" event="$2"; shift 2
|
|
printf 'level=%s zotero_event=%s %s\n' "$level" "$event" "$*"
|
|
}
|
|
|
|
# Dormant unless an OTLP endpoint is configured AND otel-cli is installed.
|
|
# Enabling tracing also requires attaching this container to the
|
|
# `observability` network — see compose.yml. Best-effort; never fatal.
|
|
emit_span() {
|
|
[ -n "$OTEL_ENDPOINT" ] || return 0
|
|
command -v otel-cli >/dev/null 2>&1 || return 0
|
|
otel-cli span \
|
|
--service zotero \
|
|
--name "$1" \
|
|
--kind internal \
|
|
--attrs "zotero.event=$1,zotero.src_display=${SRC_DISPLAY},zotero.dst_display=${DST_DISPLAY}" \
|
|
--endpoint "$OTEL_ENDPOINT" --protocol grpc --timeout 2s >/dev/null 2>&1 || true
|
|
}
|
|
|
|
x_reachable() { DISPLAY="$1" xdpyinfo >/dev/null 2>&1; }
|
|
|
|
# A live KDE session means something is actually drawing on :20. Without this
|
|
# the bridge could be "up" while mirroring a blank screen (KDE crash).
|
|
session_alive() {
|
|
pgrep -x plasmashell >/dev/null 2>&1 || pgrep -x kwin_x11 >/dev/null 2>&1
|
|
}
|
|
|
|
start_bridge() {
|
|
log error bridge_down "msg=\"kasmxproxy not running; starting it\" src=${SRC_DISPLAY} dst=${DST_DISPLAY}"
|
|
emit_span bridge_down
|
|
DISPLAY="$SRC_DISPLAY" kasmxproxy -a "$SRC_DISPLAY" -v "$DST_DISPLAY" -f "$FPS" \
|
|
>/tmp/kasmxproxy.log 2>&1 &
|
|
sleep 2
|
|
if pgrep -x kasmxproxy >/dev/null 2>&1; then
|
|
log info bridge_up "pid=$(pgrep -x kasmxproxy | head -n1)"
|
|
emit_span bridge_up
|
|
else
|
|
log error bridge_start_failed "msg=\"kasmxproxy exited immediately; see /tmp/kasmxproxy.log\""
|
|
fi
|
|
}
|
|
|
|
log info guard_start "msg=\"watching ${SRC_DISPLAY}->${DST_DISPLAY}\" poll=${POLL}s"
|
|
|
|
while :; do
|
|
if ! x_reachable "$SRC_DISPLAY"; then
|
|
log warn src_display_down "display=${SRC_DISPLAY}"
|
|
elif ! x_reachable "$DST_DISPLAY"; then
|
|
log warn dst_display_down "display=${DST_DISPLAY}"
|
|
elif ! pgrep -x kasmxproxy >/dev/null 2>&1; then
|
|
start_bridge
|
|
fi
|
|
|
|
# Heartbeat reflects a *fully* healthy bridge: proxy running, the display
|
|
# the browser sees is reachable, and a real session is drawing to it.
|
|
if pgrep -x kasmxproxy >/dev/null 2>&1 && x_reachable "$DST_DISPLAY" && session_alive; then
|
|
: > "$HEARTBEAT"
|
|
fi
|
|
|
|
sleep "$POLL"
|
|
done
|