fix loki healthcheck, label issues by pipeline/step/image
Some checks failed
ci/woodpecker/push/infra-ci Pipeline was successful
coverage 99% coverage
ci/woodpecker/push/ci Pipeline was successful
ci/woodpecker/push/deploy Pipeline failed

- loki: CMD-SHELL → CMD (distroless has no /bin/sh)
- deploy.yml: drop sha- prefix from image tags
- diag reporter: resolve labels from pipeline name, step
  name, and image name (pipeline:*, step:*, image:*)
- created 20 labels in Gitea for structured issue tagging
This commit is contained in:
kert
2026-03-23 19:28:28 -04:00
parent e703c88bae
commit 9f6683ab10
3 changed files with 101 additions and 22 deletions

View File

@@ -5,7 +5,7 @@
# Only runs on pushes to main (i.e. after PR merge).
#
# Image naming:
# gitea.homelab.fhirworx.io/homelab/<service>:sha-<8chars>
# gitea.homelab.fhirworx.io/homelab/<service>:<8chars>
#
# Buildkit pushes via HTTP through Traefik (TLS terminated at edge).
# DNS resolved by CoreDNS via Traefik UDP on the CI network.
@@ -65,7 +65,7 @@ steps:
dockerfile: notebooks/Dockerfile
context: notebooks/
tags:
- "sha-${CI_COMMIT_SHA:0:8}"
- "${CI_COMMIT_SHA:0:8}"
- latest
- name: scan-notebooks
@@ -86,7 +86,7 @@ steps:
dockerfile: zotero/Dockerfile
context: zotero/
tags:
- "sha-${CI_COMMIT_SHA:0:8}"
- "${CI_COMMIT_SHA:0:8}"
- latest
- name: scan-zotero
@@ -115,7 +115,7 @@ steps:
dockerfile: docs/Dockerfile
context: .
tags:
- "sha-${CI_COMMIT_SHA:0:8}"
- "${CI_COMMIT_SHA:0:8}"
- latest
depends_on:
- prep-docs-context
@@ -138,7 +138,7 @@ steps:
dockerfile: api/Dockerfile
context: .
tags:
- "sha-${CI_COMMIT_SHA:0:8}"
- "${CI_COMMIT_SHA:0:8}"
- latest
- name: scan-api
@@ -159,7 +159,7 @@ steps:
dockerfile: rustfs/Dockerfile.mc
context: rustfs/
tags:
- "sha-${CI_COMMIT_SHA:0:8}"
- "${CI_COMMIT_SHA:0:8}"
- latest
# ── Upload scan results ─────────────────────────────────────

View File

@@ -489,11 +489,7 @@ services:
- loki_data:/loki
command: -config.file=/etc/loki/local-config.yaml
healthcheck:
test:
[
"CMD-SHELL",
"wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1",
]
test: ["CMD", "/usr/bin/loki", "--version"]
interval: 30s
timeout: 10s
retries: 5

View File

@@ -6,10 +6,8 @@ Usage (in a Woodpecker ``failure`` step)::
Reads CI environment variables to locate the failed pipeline, pulls
step logs via the Woodpecker API, parses any Python tracebacks, runs
``git blame`` on the offending lines, and files a Gitea issue.
For non-Python failures (docker build errors, shell script crashes),
it still files an issue with the raw log output.
``git blame`` on the offending lines, and files a Gitea issue with
labels for the pipeline name, step category, and image (if applicable).
"""
from __future__ import annotations
@@ -20,6 +18,39 @@ import sys
log = logging.getLogger(__name__)
# ── Label ID mapping ──────────────────────────────────────────────
# These IDs correspond to labels created in the Gitea repo.
PIPELINE_LABELS: dict[str, int] = {
"ci": 14,
"deploy": 15,
"harden": 16,
"infra-ci": 17,
"rebuild-all": 18,
"release": 19,
}
STEP_LABELS: dict[str, int] = {
"lint": 20,
"test": 21,
"build": 22,
"scan": 23,
"deploy": 24,
"publish": 25,
"hadolint": 26,
"validate": 27,
}
IMAGE_LABELS: dict[str, int] = {
"api": 28,
"notebooks": 29,
"zotero": 30,
"docs": 31,
"mc": 32,
}
INFRA_LABEL = 33
CI_LABEL = 13 # existing "ci" label
def _get_env(name: str) -> str:
val = os.environ.get(name, "")
@@ -28,6 +59,49 @@ def _get_env(name: str) -> str:
return val
def _resolve_labels(pipeline_name: str, step_name: str) -> list[int]:
"""Resolve label IDs from pipeline and step names."""
labels: list[int] = [CI_LABEL]
# Pipeline label
for key, lid in PIPELINE_LABELS.items():
if key in pipeline_name:
labels.append(lid)
break
# Step category label — match the broadest category
step_lower = step_name.lower()
for key, lid in STEP_LABELS.items():
if key in step_lower:
labels.append(lid)
break
# Image label — detect which image the step relates to
for key, lid in IMAGE_LABELS.items():
if key in step_lower:
labels.append(lid)
break
return labels
def _pipeline_name_from_env() -> str:
"""Best-effort pipeline name from CI env vars."""
event = os.environ.get("CI_PIPELINE_EVENT", "push")
branch = os.environ.get("CI_COMMIT_BRANCH", "")
if event == "tag":
return "release"
if event in ("manual", "cron"):
return "harden"
if event == "pull_request":
return "ci"
# push to main = deploy, push to other = ci
if branch == "main":
return "deploy"
return "ci"
def main() -> int:
logging.basicConfig(
level=logging.INFO,
@@ -54,6 +128,8 @@ def main() -> int:
else:
owner, repo = "homelab", "stack"
pipeline_name = _pipeline_name_from_env()
# Fetch pipeline steps and find failures
from api.clients.woodpecker import WoodpeckerClient
@@ -91,6 +167,8 @@ def main() -> int:
log.info("Processing failed step: %s (id=%d)", step_name, step_id)
labels = _resolve_labels(pipeline_name, step_name)
# Fetch logs — Woodpecker returns a list of log line dicts
try:
log_entries = wp.get_logs(rid, pnum, step_id)
@@ -110,6 +188,8 @@ def main() -> int:
# Try to parse Python tracebacks
reports = parse_traceback_text(log_text)
sha_short = (commit_sha or "unknown")[:8]
if reports:
# File one issue per traceback found
for report in reports:
@@ -121,11 +201,13 @@ def main() -> int:
commit_sha or "unknown",
)
body = (
f"**Pipeline:** #{pnum} step `{step_name}`\n"
f"**Pipeline:** `{pipeline_name}` #{pnum} "
f"step `{step_name}`\n"
f"**Commit:** `{sha_short}`\n"
f"**Status:** `failure`\n\n" + body
)
title = (
f"ci: {step_name} — {report.exc_type}: "
f"{pipeline_name}/{step_name}: {report.exc_type}: "
f"{_truncate(report.exc_value, 50)}"
)
result = gitea.create_issue(
@@ -134,16 +216,17 @@ def main() -> int:
{
"title": title,
"body": body,
"labels": [13],
"labels": labels,
},
)
log.info("Filed issue #%s: %s", result.get("number"), title)
else:
# Non-Python failure — file with raw log
title = f"ci: {step_name} failed (pipeline #{pnum})"
title = f"{pipeline_name}/{step_name} failed (#{pnum} @ {sha_short})"
body = (
f"**Pipeline:** #{pnum} step `{step_name}`\n"
f"**Commit:** `{commit_sha}`\n"
f"**Pipeline:** `{pipeline_name}` #{pnum} "
f"step `{step_name}`\n"
f"**Commit:** `{sha_short}`\n"
f"**Status:** `failure`\n\n"
f"## Step Log\n\n"
f"<details>\n"
@@ -157,7 +240,7 @@ def main() -> int:
{
"title": title,
"body": body,
"labels": [13],
"labels": labels,
},
)
log.info("Filed issue #%s: %s", result.get("number"), title)