fix loki healthcheck, label issues by pipeline/step/image
- loki: CMD-SHELL → CMD (distroless has no /bin/sh) - deploy.yml: drop sha- prefix from image tags - diag reporter: resolve labels from pipeline name, step name, and image name (pipeline:*, step:*, image:*) - created 20 labels in Gitea for structured issue tagging
This commit is contained in:
@@ -5,7 +5,7 @@
|
||||
# Only runs on pushes to main (i.e. after PR merge).
|
||||
#
|
||||
# Image naming:
|
||||
# gitea.homelab.fhirworx.io/homelab/<service>:sha-<8chars>
|
||||
# gitea.homelab.fhirworx.io/homelab/<service>:<8chars>
|
||||
#
|
||||
# Buildkit pushes via HTTP through Traefik (TLS terminated at edge).
|
||||
# DNS resolved by CoreDNS via Traefik UDP on the CI network.
|
||||
@@ -65,7 +65,7 @@ steps:
|
||||
dockerfile: notebooks/Dockerfile
|
||||
context: notebooks/
|
||||
tags:
|
||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: scan-notebooks
|
||||
@@ -86,7 +86,7 @@ steps:
|
||||
dockerfile: zotero/Dockerfile
|
||||
context: zotero/
|
||||
tags:
|
||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: scan-zotero
|
||||
@@ -115,7 +115,7 @@ steps:
|
||||
dockerfile: docs/Dockerfile
|
||||
context: .
|
||||
tags:
|
||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
depends_on:
|
||||
- prep-docs-context
|
||||
@@ -138,7 +138,7 @@ steps:
|
||||
dockerfile: api/Dockerfile
|
||||
context: .
|
||||
tags:
|
||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
- name: scan-api
|
||||
@@ -159,7 +159,7 @@ steps:
|
||||
dockerfile: rustfs/Dockerfile.mc
|
||||
context: rustfs/
|
||||
tags:
|
||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
||||
- "${CI_COMMIT_SHA:0:8}"
|
||||
- latest
|
||||
|
||||
# ── Upload scan results ─────────────────────────────────────
|
||||
|
||||
@@ -489,11 +489,7 @@ services:
|
||||
- loki_data:/loki
|
||||
command: -config.file=/etc/loki/local-config.yaml
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1",
|
||||
]
|
||||
test: ["CMD", "/usr/bin/loki", "--version"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
@@ -6,10 +6,8 @@ Usage (in a Woodpecker ``failure`` step)::
|
||||
|
||||
Reads CI environment variables to locate the failed pipeline, pulls
|
||||
step logs via the Woodpecker API, parses any Python tracebacks, runs
|
||||
``git blame`` on the offending lines, and files a Gitea issue.
|
||||
|
||||
For non-Python failures (docker build errors, shell script crashes),
|
||||
it still files an issue with the raw log output.
|
||||
``git blame`` on the offending lines, and files a Gitea issue with
|
||||
labels for the pipeline name, step category, and image (if applicable).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -20,6 +18,39 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# ── Label ID mapping ──────────────────────────────────────────────
|
||||
# These IDs correspond to labels created in the Gitea repo.
|
||||
PIPELINE_LABELS: dict[str, int] = {
|
||||
"ci": 14,
|
||||
"deploy": 15,
|
||||
"harden": 16,
|
||||
"infra-ci": 17,
|
||||
"rebuild-all": 18,
|
||||
"release": 19,
|
||||
}
|
||||
|
||||
STEP_LABELS: dict[str, int] = {
|
||||
"lint": 20,
|
||||
"test": 21,
|
||||
"build": 22,
|
||||
"scan": 23,
|
||||
"deploy": 24,
|
||||
"publish": 25,
|
||||
"hadolint": 26,
|
||||
"validate": 27,
|
||||
}
|
||||
|
||||
IMAGE_LABELS: dict[str, int] = {
|
||||
"api": 28,
|
||||
"notebooks": 29,
|
||||
"zotero": 30,
|
||||
"docs": 31,
|
||||
"mc": 32,
|
||||
}
|
||||
|
||||
INFRA_LABEL = 33
|
||||
CI_LABEL = 13 # existing "ci" label
|
||||
|
||||
|
||||
def _get_env(name: str) -> str:
|
||||
val = os.environ.get(name, "")
|
||||
@@ -28,6 +59,49 @@ def _get_env(name: str) -> str:
|
||||
return val
|
||||
|
||||
|
||||
def _resolve_labels(pipeline_name: str, step_name: str) -> list[int]:
|
||||
"""Resolve label IDs from pipeline and step names."""
|
||||
labels: list[int] = [CI_LABEL]
|
||||
|
||||
# Pipeline label
|
||||
for key, lid in PIPELINE_LABELS.items():
|
||||
if key in pipeline_name:
|
||||
labels.append(lid)
|
||||
break
|
||||
|
||||
# Step category label — match the broadest category
|
||||
step_lower = step_name.lower()
|
||||
for key, lid in STEP_LABELS.items():
|
||||
if key in step_lower:
|
||||
labels.append(lid)
|
||||
break
|
||||
|
||||
# Image label — detect which image the step relates to
|
||||
for key, lid in IMAGE_LABELS.items():
|
||||
if key in step_lower:
|
||||
labels.append(lid)
|
||||
break
|
||||
|
||||
return labels
|
||||
|
||||
|
||||
def _pipeline_name_from_env() -> str:
|
||||
"""Best-effort pipeline name from CI env vars."""
|
||||
event = os.environ.get("CI_PIPELINE_EVENT", "push")
|
||||
branch = os.environ.get("CI_COMMIT_BRANCH", "")
|
||||
|
||||
if event == "tag":
|
||||
return "release"
|
||||
if event in ("manual", "cron"):
|
||||
return "harden"
|
||||
if event == "pull_request":
|
||||
return "ci"
|
||||
# push to main = deploy, push to other = ci
|
||||
if branch == "main":
|
||||
return "deploy"
|
||||
return "ci"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -54,6 +128,8 @@ def main() -> int:
|
||||
else:
|
||||
owner, repo = "homelab", "stack"
|
||||
|
||||
pipeline_name = _pipeline_name_from_env()
|
||||
|
||||
# Fetch pipeline steps and find failures
|
||||
from api.clients.woodpecker import WoodpeckerClient
|
||||
|
||||
@@ -91,6 +167,8 @@ def main() -> int:
|
||||
|
||||
log.info("Processing failed step: %s (id=%d)", step_name, step_id)
|
||||
|
||||
labels = _resolve_labels(pipeline_name, step_name)
|
||||
|
||||
# Fetch logs — Woodpecker returns a list of log line dicts
|
||||
try:
|
||||
log_entries = wp.get_logs(rid, pnum, step_id)
|
||||
@@ -110,6 +188,8 @@ def main() -> int:
|
||||
# Try to parse Python tracebacks
|
||||
reports = parse_traceback_text(log_text)
|
||||
|
||||
sha_short = (commit_sha or "unknown")[:8]
|
||||
|
||||
if reports:
|
||||
# File one issue per traceback found
|
||||
for report in reports:
|
||||
@@ -121,11 +201,13 @@ def main() -> int:
|
||||
commit_sha or "unknown",
|
||||
)
|
||||
body = (
|
||||
f"**Pipeline:** #{pnum} step `{step_name}`\n"
|
||||
f"**Pipeline:** `{pipeline_name}` #{pnum} "
|
||||
f"step `{step_name}`\n"
|
||||
f"**Commit:** `{sha_short}`\n"
|
||||
f"**Status:** `failure`\n\n" + body
|
||||
)
|
||||
title = (
|
||||
f"ci: {step_name} — {report.exc_type}: "
|
||||
f"{pipeline_name}/{step_name}: {report.exc_type}: "
|
||||
f"{_truncate(report.exc_value, 50)}"
|
||||
)
|
||||
result = gitea.create_issue(
|
||||
@@ -134,16 +216,17 @@ def main() -> int:
|
||||
{
|
||||
"title": title,
|
||||
"body": body,
|
||||
"labels": [13],
|
||||
"labels": labels,
|
||||
},
|
||||
)
|
||||
log.info("Filed issue #%s: %s", result.get("number"), title)
|
||||
else:
|
||||
# Non-Python failure — file with raw log
|
||||
title = f"ci: {step_name} failed (pipeline #{pnum})"
|
||||
title = f"{pipeline_name}/{step_name} failed (#{pnum} @ {sha_short})"
|
||||
body = (
|
||||
f"**Pipeline:** #{pnum} step `{step_name}`\n"
|
||||
f"**Commit:** `{commit_sha}`\n"
|
||||
f"**Pipeline:** `{pipeline_name}` #{pnum} "
|
||||
f"step `{step_name}`\n"
|
||||
f"**Commit:** `{sha_short}`\n"
|
||||
f"**Status:** `failure`\n\n"
|
||||
f"## Step Log\n\n"
|
||||
f"<details>\n"
|
||||
@@ -157,7 +240,7 @@ def main() -> int:
|
||||
{
|
||||
"title": title,
|
||||
"body": body,
|
||||
"labels": [13],
|
||||
"labels": labels,
|
||||
},
|
||||
)
|
||||
log.info("Filed issue #%s: %s", result.get("number"), title)
|
||||
|
||||
Reference in New Issue
Block a user