fix loki healthcheck, label issues by pipeline/step/image
Some checks failed
ci/woodpecker/push/infra-ci Pipeline was successful
coverage 99% coverage
ci/woodpecker/push/ci Pipeline was successful
ci/woodpecker/push/deploy Pipeline failed

- loki: CMD-SHELL → CMD (distroless has no /bin/sh)
- deploy.yml: drop sha- prefix from image tags
- diag reporter: resolve labels from pipeline name, step
  name, and image name (pipeline:*, step:*, image:*)
- created 20 labels in Gitea for structured issue tagging
This commit is contained in:
kert
2026-03-23 19:28:28 -04:00
parent e703c88bae
commit 9f6683ab10
3 changed files with 101 additions and 22 deletions

View File

@@ -5,7 +5,7 @@
# Only runs on pushes to main (i.e. after PR merge). # Only runs on pushes to main (i.e. after PR merge).
# #
# Image naming: # Image naming:
# gitea.homelab.fhirworx.io/homelab/<service>:sha-<8chars> # gitea.homelab.fhirworx.io/homelab/<service>:<8chars>
# #
# Buildkit pushes via HTTP through Traefik (TLS terminated at edge). # Buildkit pushes via HTTP through Traefik (TLS terminated at edge).
# DNS resolved by CoreDNS via Traefik UDP on the CI network. # DNS resolved by CoreDNS via Traefik UDP on the CI network.
@@ -65,7 +65,7 @@ steps:
dockerfile: notebooks/Dockerfile dockerfile: notebooks/Dockerfile
context: notebooks/ context: notebooks/
tags: tags:
- "sha-${CI_COMMIT_SHA:0:8}" - "${CI_COMMIT_SHA:0:8}"
- latest - latest
- name: scan-notebooks - name: scan-notebooks
@@ -86,7 +86,7 @@ steps:
dockerfile: zotero/Dockerfile dockerfile: zotero/Dockerfile
context: zotero/ context: zotero/
tags: tags:
- "sha-${CI_COMMIT_SHA:0:8}" - "${CI_COMMIT_SHA:0:8}"
- latest - latest
- name: scan-zotero - name: scan-zotero
@@ -115,7 +115,7 @@ steps:
dockerfile: docs/Dockerfile dockerfile: docs/Dockerfile
context: . context: .
tags: tags:
- "sha-${CI_COMMIT_SHA:0:8}" - "${CI_COMMIT_SHA:0:8}"
- latest - latest
depends_on: depends_on:
- prep-docs-context - prep-docs-context
@@ -138,7 +138,7 @@ steps:
dockerfile: api/Dockerfile dockerfile: api/Dockerfile
context: . context: .
tags: tags:
- "sha-${CI_COMMIT_SHA:0:8}" - "${CI_COMMIT_SHA:0:8}"
- latest - latest
- name: scan-api - name: scan-api
@@ -159,7 +159,7 @@ steps:
dockerfile: rustfs/Dockerfile.mc dockerfile: rustfs/Dockerfile.mc
context: rustfs/ context: rustfs/
tags: tags:
- "sha-${CI_COMMIT_SHA:0:8}" - "${CI_COMMIT_SHA:0:8}"
- latest - latest
# ── Upload scan results ───────────────────────────────────── # ── Upload scan results ─────────────────────────────────────

View File

@@ -489,11 +489,7 @@ services:
- loki_data:/loki - loki_data:/loki
command: -config.file=/etc/loki/local-config.yaml command: -config.file=/etc/loki/local-config.yaml
healthcheck: healthcheck:
test: test: ["CMD", "/usr/bin/loki", "--version"]
[
"CMD-SHELL",
"wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1",
]
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 5 retries: 5

View File

@@ -6,10 +6,8 @@ Usage (in a Woodpecker ``failure`` step)::
Reads CI environment variables to locate the failed pipeline, pulls Reads CI environment variables to locate the failed pipeline, pulls
step logs via the Woodpecker API, parses any Python tracebacks, runs step logs via the Woodpecker API, parses any Python tracebacks, runs
``git blame`` on the offending lines, and files a Gitea issue. ``git blame`` on the offending lines, and files a Gitea issue with
labels for the pipeline name, step category, and image (if applicable).
For non-Python failures (docker build errors, shell script crashes),
it still files an issue with the raw log output.
""" """
from __future__ import annotations from __future__ import annotations
@@ -20,6 +18,39 @@ import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
# ── Label ID mapping ──────────────────────────────────────────────
# These IDs correspond to labels created in the Gitea repo.
PIPELINE_LABELS: dict[str, int] = {
"ci": 14,
"deploy": 15,
"harden": 16,
"infra-ci": 17,
"rebuild-all": 18,
"release": 19,
}
STEP_LABELS: dict[str, int] = {
"lint": 20,
"test": 21,
"build": 22,
"scan": 23,
"deploy": 24,
"publish": 25,
"hadolint": 26,
"validate": 27,
}
IMAGE_LABELS: dict[str, int] = {
"api": 28,
"notebooks": 29,
"zotero": 30,
"docs": 31,
"mc": 32,
}
INFRA_LABEL = 33
CI_LABEL = 13 # existing "ci" label
def _get_env(name: str) -> str: def _get_env(name: str) -> str:
val = os.environ.get(name, "") val = os.environ.get(name, "")
@@ -28,6 +59,49 @@ def _get_env(name: str) -> str:
return val return val
def _resolve_labels(pipeline_name: str, step_name: str) -> list[int]:
"""Resolve label IDs from pipeline and step names."""
labels: list[int] = [CI_LABEL]
# Pipeline label
for key, lid in PIPELINE_LABELS.items():
if key in pipeline_name:
labels.append(lid)
break
# Step category label — match the broadest category
step_lower = step_name.lower()
for key, lid in STEP_LABELS.items():
if key in step_lower:
labels.append(lid)
break
# Image label — detect which image the step relates to
for key, lid in IMAGE_LABELS.items():
if key in step_lower:
labels.append(lid)
break
return labels
def _pipeline_name_from_env() -> str:
"""Best-effort pipeline name from CI env vars."""
event = os.environ.get("CI_PIPELINE_EVENT", "push")
branch = os.environ.get("CI_COMMIT_BRANCH", "")
if event == "tag":
return "release"
if event in ("manual", "cron"):
return "harden"
if event == "pull_request":
return "ci"
# push to main = deploy, push to other = ci
if branch == "main":
return "deploy"
return "ci"
def main() -> int: def main() -> int:
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
@@ -54,6 +128,8 @@ def main() -> int:
else: else:
owner, repo = "homelab", "stack" owner, repo = "homelab", "stack"
pipeline_name = _pipeline_name_from_env()
# Fetch pipeline steps and find failures # Fetch pipeline steps and find failures
from api.clients.woodpecker import WoodpeckerClient from api.clients.woodpecker import WoodpeckerClient
@@ -91,6 +167,8 @@ def main() -> int:
log.info("Processing failed step: %s (id=%d)", step_name, step_id) log.info("Processing failed step: %s (id=%d)", step_name, step_id)
labels = _resolve_labels(pipeline_name, step_name)
# Fetch logs — Woodpecker returns a list of log line dicts # Fetch logs — Woodpecker returns a list of log line dicts
try: try:
log_entries = wp.get_logs(rid, pnum, step_id) log_entries = wp.get_logs(rid, pnum, step_id)
@@ -110,6 +188,8 @@ def main() -> int:
# Try to parse Python tracebacks # Try to parse Python tracebacks
reports = parse_traceback_text(log_text) reports = parse_traceback_text(log_text)
sha_short = (commit_sha or "unknown")[:8]
if reports: if reports:
# File one issue per traceback found # File one issue per traceback found
for report in reports: for report in reports:
@@ -121,11 +201,13 @@ def main() -> int:
commit_sha or "unknown", commit_sha or "unknown",
) )
body = ( body = (
f"**Pipeline:** #{pnum} step `{step_name}`\n" f"**Pipeline:** `{pipeline_name}` #{pnum} "
f"step `{step_name}`\n"
f"**Commit:** `{sha_short}`\n"
f"**Status:** `failure`\n\n" + body f"**Status:** `failure`\n\n" + body
) )
title = ( title = (
f"ci: {step_name} — {report.exc_type}: " f"{pipeline_name}/{step_name}: {report.exc_type}: "
f"{_truncate(report.exc_value, 50)}" f"{_truncate(report.exc_value, 50)}"
) )
result = gitea.create_issue( result = gitea.create_issue(
@@ -134,16 +216,17 @@ def main() -> int:
{ {
"title": title, "title": title,
"body": body, "body": body,
"labels": [13], "labels": labels,
}, },
) )
log.info("Filed issue #%s: %s", result.get("number"), title) log.info("Filed issue #%s: %s", result.get("number"), title)
else: else:
# Non-Python failure — file with raw log # Non-Python failure — file with raw log
title = f"ci: {step_name} failed (pipeline #{pnum})" title = f"{pipeline_name}/{step_name} failed (#{pnum} @ {sha_short})"
body = ( body = (
f"**Pipeline:** #{pnum} step `{step_name}`\n" f"**Pipeline:** `{pipeline_name}` #{pnum} "
f"**Commit:** `{commit_sha}`\n" f"step `{step_name}`\n"
f"**Commit:** `{sha_short}`\n"
f"**Status:** `failure`\n\n" f"**Status:** `failure`\n\n"
f"## Step Log\n\n" f"## Step Log\n\n"
f"<details>\n" f"<details>\n"
@@ -157,7 +240,7 @@ def main() -> int:
{ {
"title": title, "title": title,
"body": body, "body": body,
"labels": [13], "labels": labels,
}, },
) )
log.info("Filed issue #%s: %s", result.get("number"), title) log.info("Filed issue #%s: %s", result.get("number"), title)