Files
stack/infra/loki/promtail-config.yml
kert 225ce01d6b fix(zotero): heal kasmxproxy display-bridge race + add real healthcheck/observability
zotero.fhirworx.io rendered a blank desktop: KDE+Zotero draw on Xvfb :20,
but kasmvnc serves :21, and the kasmxproxy mirror that bridges them never
started. The stock kasmvnc-entrypoint.sh blocks on `until [ -S
/tmp/.X11-unix/X21 ]` (a filesystem socket path) while entrypoint.sh's
concurrent `rm -rf /tmp/.X*` unlinks it — Xvnc keeps its fd + abstract
socket open and serves a blank :21 forever. A startup race, hence the
intermittent "doesn't open" with every process reporting healthy.

- kasmxproxy-guard.sh: supervised watchdog that probes :20/:21 via xdpyinfo
  (not the racy socket file) and (re)launches kasmxproxy, self-healing if it
  dies. Emits structured `zotero_event=` logs to container stdout + a
  heartbeat file.
- zotero-healthcheck.sh + compose healthcheck: detect the real failure mode
  (bridge down / :21 unreachable / KDE dead / stale heartbeat) that
  container/nginx/Xvnc liveness all miss.
- promtail: extract zotero_event into a Loki label.
- Grafana: Zotero Desktop dashboard + alert on sustained bridge failure.
- otel-cli baked in; OTLP tracing dormant by default (enabling it requires
  attaching zotero to the observability network — documented opt-in, left
  off to preserve gateway-only isolation).

Verified live: guard self-heals on kasmxproxy kill; healthcheck passes
healthy / fails on stale heartbeat.
2026-06-17 19:49:25 -04:00

189 lines
6.4 KiB
YAML

server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
scrape_configs:
- job_name: docker
docker_sd_configs:
- host: unix:///var/run/docker.sock
refresh_interval: 15s
relabel_configs:
# Only stack containers (docker.sock exposes every container on the
# host; without this, corwins.media's labelled containers also get
# ingested into our Loki).
- source_labels: ['__meta_docker_container_label_com_docker_compose_project']
regex: 'stack'
action: keep
- source_labels: ['__meta_docker_container_label_promtail']
regex: 'true'
action: keep
- source_labels: ['__meta_docker_container_name']
regex: '/?(.*)'
target_label: container
- source_labels: ['__meta_docker_container_name']
regex: '/?(.*)'
target_label: job
- source_labels: ['__meta_docker_container_label_com_docker_compose_service']
target_label: service
- source_labels: ['__meta_docker_container_label_com_docker_compose_project']
target_label: project
pipeline_stages:
- docker: {}
- regex:
expression: '(?i)(?P<level>error|warn|info|debug|fatal|critical|trace)'
- labels:
level:
- regex:
expression: '(?P<remote_ip>\d+\.\d+\.\d+\.\d+) .* "(?P<http_method>GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS) (?P<http_path>[^ ]*) [^"]*" (?P<http_status>\d{3}) (?P<http_bytes>\d+)'
- labels:
http_method:
http_status:
# --- Traefik ---
- match:
selector: '{container="traefik"}'
stages:
- json:
expressions:
RouterName: RouterName
ServiceName: ServiceName
DownstreamStatus: DownstreamStatus
RequestMethod: RequestMethod
- labels:
tf_router: RouterName
tf_service: ServiceName
http_status: DownstreamStatus
http_method: RequestMethod
- regex:
expression: '(?P<tf_event>TLS handshake|EntryPoint|router|certResolver|middleware|tls)'
- labels:
tf_event:
# --- Gitea ---
- match:
selector: '{container="git"}'
stages:
- regex:
expression: '(?P<gitea_event>login|logout|register|push|pull|branch|tag|issue|webhook|oauth|created|deleted|merged)'
- labels:
gitea_event:
# --- oauth2-proxy ---
- match:
selector: '{container="oauth2-proxy"}'
stages:
- regex:
expression: '(?P<oauth_event>AuthSuccess|AuthFailure|OAuthStart|OAuthCallback|OAuthError|Invalid|Expired|Authenticated|Forbidden)'
- labels:
oauth_event:
# --- Postgres ---
- match:
selector: '{container="postgres"}'
stages:
- regex:
expression: '(?P<pg_event>FATAL|ERROR|WARNING|checkpoint|autovacuum|connection authorized|connection received|statement)'
- labels:
pg_event:
# --- Trino ---
- match:
selector: '{container="trino"}'
stages:
- regex:
expression: '(?P<trino_event>QUERY_CREATED|QUERY_STARTED|QUERY_COMPLETED|QUERY_FAILED|SPLIT_COMPLETED)'
- regex:
expression: '\b(?P<query_id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b'
- labels:
trino_event:
query_id:
# --- Nessie / Polaris (Quarkus) ---
- match:
selector: '{container=~"nessie|polaris"}'
stages:
- regex:
expression: '"(?P<http_method>GET|POST|PUT|DELETE) (?P<quarkus_route>/[^ ?]+)'
- labels:
quarkus_route:
# --- RustFS ---
- match:
selector: '{container="rustfs"}'
stages:
- regex:
expression: '(?P<s3_op>PutObject|GetObject|DeleteObject|ListObjectsV2|HeadObject|CompleteMultipart)'
- regex:
expression: 'bucket=(?P<s3_bucket>[A-Za-z0-9._-]+)'
- labels:
s3_op:
s3_bucket:
# --- API (FastAPI/stack) ---
# Word-boundary anchors keep the label off lines that merely contain
# the substrings (e.g. "polling" should not set api_event=poll).
- match:
selector: '{container="api"}'
stages:
- regex:
expression: '\b(?P<api_event>pipeline|ingest|health|error)\b'
- regex:
expression: 'pipeline=(?P<pipeline_name>[A-Za-z0-9_.-]+)'
- labels:
api_event:
pipeline_name:
# --- mail-poller ---
- match:
selector: '{container="mail-poller"}'
stages:
- regex:
expression: '\b(?P<mail_event>ingest|poll|error)\b'
- regex:
expression: 'count=(?P<mail_count>\d+)'
- labels:
mail_event:
mail_count:
# --- zotero (KDE desktop / kasmxproxy bridge watchdog) ---
- match:
selector: '{container="zotero"}'
stages:
# kasmxproxy-guard.sh emits `zotero_event=<state>` on each bridge
# transition (bridge_up / bridge_down / bridge_start_failed /
# src_display_down / dst_display_down / guard_start).
- regex:
expression: 'zotero_event=(?P<zotero_event>\w+)'
- labels:
zotero_event:
# --- act-runner ---
- match:
selector: '{container="act-runner"}'
stages:
- regex:
expression: '(?P<ci_event>job started|job finished|job failed|runner registered)'
- regex:
expression: 'job_id=(?P<job_id>\S+)'
- labels:
ci_event:
job_id:
# --- cloudflared ---
- match:
selector: '{container="cloudflared"}'
stages:
- regex:
expression: '\b(?P<cf_event>connection|registered|unregistered|reconnect|quic|error|tunnel|origin)\b'
- regex:
expression: 'status=(?P<cf_origin_status>\d{3})'
- labels:
cf_event:
cf_origin_status: