zotero.fhirworx.io rendered a blank desktop: KDE+Zotero draw on Xvfb :20, but kasmvnc serves :21, and the kasmxproxy mirror that bridges them never started. The stock kasmvnc-entrypoint.sh blocks on `until [ -S /tmp/.X11-unix/X21 ]` (a filesystem socket path) while entrypoint.sh's concurrent `rm -rf /tmp/.X*` unlinks it — Xvnc keeps its fd + abstract socket open and serves a blank :21 forever. A startup race, hence the intermittent "doesn't open" with every process reporting healthy. - kasmxproxy-guard.sh: supervised watchdog that probes :20/:21 via xdpyinfo (not the racy socket file) and (re)launches kasmxproxy, self-healing if it dies. Emits structured `zotero_event=` logs to container stdout + a heartbeat file. - zotero-healthcheck.sh + compose healthcheck: detect the real failure mode (bridge down / :21 unreachable / KDE dead / stale heartbeat) that container/nginx/Xvnc liveness all miss. - promtail: extract zotero_event into a Loki label. - Grafana: Zotero Desktop dashboard + alert on sustained bridge failure. - otel-cli baked in; OTLP tracing dormant by default (enabling it requires attaching zotero to the observability network — documented opt-in, left off to preserve gateway-only isolation). Verified live: guard self-heals on kasmxproxy kill; healthcheck passes healthy / fails on stale heartbeat.
189 lines
6.4 KiB
YAML
189 lines
6.4 KiB
YAML
server:
|
|
http_listen_port: 9080
|
|
grpc_listen_port: 0
|
|
|
|
positions:
|
|
filename: /tmp/positions.yaml
|
|
|
|
clients:
|
|
- url: http://loki:3100/loki/api/v1/push
|
|
|
|
scrape_configs:
|
|
- job_name: docker
|
|
docker_sd_configs:
|
|
- host: unix:///var/run/docker.sock
|
|
refresh_interval: 15s
|
|
relabel_configs:
|
|
# Only stack containers (docker.sock exposes every container on the
|
|
# host; without this, corwins.media's labelled containers also get
|
|
# ingested into our Loki).
|
|
- source_labels: ['__meta_docker_container_label_com_docker_compose_project']
|
|
regex: 'stack'
|
|
action: keep
|
|
- source_labels: ['__meta_docker_container_label_promtail']
|
|
regex: 'true'
|
|
action: keep
|
|
- source_labels: ['__meta_docker_container_name']
|
|
regex: '/?(.*)'
|
|
target_label: container
|
|
- source_labels: ['__meta_docker_container_name']
|
|
regex: '/?(.*)'
|
|
target_label: job
|
|
- source_labels: ['__meta_docker_container_label_com_docker_compose_service']
|
|
target_label: service
|
|
- source_labels: ['__meta_docker_container_label_com_docker_compose_project']
|
|
target_label: project
|
|
pipeline_stages:
|
|
- docker: {}
|
|
- regex:
|
|
expression: '(?i)(?P<level>error|warn|info|debug|fatal|critical|trace)'
|
|
- labels:
|
|
level:
|
|
- regex:
|
|
expression: '(?P<remote_ip>\d+\.\d+\.\d+\.\d+) .* "(?P<http_method>GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS) (?P<http_path>[^ ]*) [^"]*" (?P<http_status>\d{3}) (?P<http_bytes>\d+)'
|
|
- labels:
|
|
http_method:
|
|
http_status:
|
|
|
|
# --- Traefik ---
|
|
- match:
|
|
selector: '{container="traefik"}'
|
|
stages:
|
|
- json:
|
|
expressions:
|
|
RouterName: RouterName
|
|
ServiceName: ServiceName
|
|
DownstreamStatus: DownstreamStatus
|
|
RequestMethod: RequestMethod
|
|
- labels:
|
|
tf_router: RouterName
|
|
tf_service: ServiceName
|
|
http_status: DownstreamStatus
|
|
http_method: RequestMethod
|
|
- regex:
|
|
expression: '(?P<tf_event>TLS handshake|EntryPoint|router|certResolver|middleware|tls)'
|
|
- labels:
|
|
tf_event:
|
|
|
|
# --- Gitea ---
|
|
- match:
|
|
selector: '{container="git"}'
|
|
stages:
|
|
- regex:
|
|
expression: '(?P<gitea_event>login|logout|register|push|pull|branch|tag|issue|webhook|oauth|created|deleted|merged)'
|
|
- labels:
|
|
gitea_event:
|
|
|
|
# --- oauth2-proxy ---
|
|
- match:
|
|
selector: '{container="oauth2-proxy"}'
|
|
stages:
|
|
- regex:
|
|
expression: '(?P<oauth_event>AuthSuccess|AuthFailure|OAuthStart|OAuthCallback|OAuthError|Invalid|Expired|Authenticated|Forbidden)'
|
|
- labels:
|
|
oauth_event:
|
|
|
|
# --- Postgres ---
|
|
- match:
|
|
selector: '{container="postgres"}'
|
|
stages:
|
|
- regex:
|
|
expression: '(?P<pg_event>FATAL|ERROR|WARNING|checkpoint|autovacuum|connection authorized|connection received|statement)'
|
|
- labels:
|
|
pg_event:
|
|
|
|
# --- Trino ---
|
|
- match:
|
|
selector: '{container="trino"}'
|
|
stages:
|
|
- regex:
|
|
expression: '(?P<trino_event>QUERY_CREATED|QUERY_STARTED|QUERY_COMPLETED|QUERY_FAILED|SPLIT_COMPLETED)'
|
|
- regex:
|
|
expression: '\b(?P<query_id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b'
|
|
- labels:
|
|
trino_event:
|
|
query_id:
|
|
|
|
# --- Nessie / Polaris (Quarkus) ---
|
|
- match:
|
|
selector: '{container=~"nessie|polaris"}'
|
|
stages:
|
|
- regex:
|
|
expression: '"(?P<http_method>GET|POST|PUT|DELETE) (?P<quarkus_route>/[^ ?]+)'
|
|
- labels:
|
|
quarkus_route:
|
|
|
|
# --- RustFS ---
|
|
- match:
|
|
selector: '{container="rustfs"}'
|
|
stages:
|
|
- regex:
|
|
expression: '(?P<s3_op>PutObject|GetObject|DeleteObject|ListObjectsV2|HeadObject|CompleteMultipart)'
|
|
- regex:
|
|
expression: 'bucket=(?P<s3_bucket>[A-Za-z0-9._-]+)'
|
|
- labels:
|
|
s3_op:
|
|
s3_bucket:
|
|
|
|
# --- API (FastAPI/stack) ---
|
|
# Word-boundary anchors keep the label off lines that merely contain
|
|
# the substrings (e.g. "polling" should not set api_event=poll).
|
|
- match:
|
|
selector: '{container="api"}'
|
|
stages:
|
|
- regex:
|
|
expression: '\b(?P<api_event>pipeline|ingest|health|error)\b'
|
|
- regex:
|
|
expression: 'pipeline=(?P<pipeline_name>[A-Za-z0-9_.-]+)'
|
|
- labels:
|
|
api_event:
|
|
pipeline_name:
|
|
|
|
# --- mail-poller ---
|
|
- match:
|
|
selector: '{container="mail-poller"}'
|
|
stages:
|
|
- regex:
|
|
expression: '\b(?P<mail_event>ingest|poll|error)\b'
|
|
- regex:
|
|
expression: 'count=(?P<mail_count>\d+)'
|
|
- labels:
|
|
mail_event:
|
|
mail_count:
|
|
|
|
# --- zotero (KDE desktop / kasmxproxy bridge watchdog) ---
|
|
- match:
|
|
selector: '{container="zotero"}'
|
|
stages:
|
|
# kasmxproxy-guard.sh emits `zotero_event=<state>` on each bridge
|
|
# transition (bridge_up / bridge_down / bridge_start_failed /
|
|
# src_display_down / dst_display_down / guard_start).
|
|
- regex:
|
|
expression: 'zotero_event=(?P<zotero_event>\w+)'
|
|
- labels:
|
|
zotero_event:
|
|
|
|
# --- act-runner ---
|
|
- match:
|
|
selector: '{container="act-runner"}'
|
|
stages:
|
|
- regex:
|
|
expression: '(?P<ci_event>job started|job finished|job failed|runner registered)'
|
|
- regex:
|
|
expression: 'job_id=(?P<job_id>\S+)'
|
|
- labels:
|
|
ci_event:
|
|
job_id:
|
|
|
|
# --- cloudflared ---
|
|
- match:
|
|
selector: '{container="cloudflared"}'
|
|
stages:
|
|
- regex:
|
|
expression: '\b(?P<cf_event>connection|registered|unregistered|reconnect|quic|error|tunnel|origin)\b'
|
|
- regex:
|
|
expression: 'status=(?P<cf_origin_status>\d{3})'
|
|
- labels:
|
|
cf_event:
|
|
cf_origin_status:
|