fix loki healthcheck, label issues by pipeline/step/image
- loki: CMD-SHELL → CMD (distroless has no /bin/sh) - deploy.yml: drop sha- prefix from image tags - diag reporter: resolve labels from pipeline name, step name, and image name (pipeline:*, step:*, image:*) - created 20 labels in Gitea for structured issue tagging
This commit is contained in:
@@ -5,7 +5,7 @@
|
|||||||
# Only runs on pushes to main (i.e. after PR merge).
|
# Only runs on pushes to main (i.e. after PR merge).
|
||||||
#
|
#
|
||||||
# Image naming:
|
# Image naming:
|
||||||
# gitea.homelab.fhirworx.io/homelab/<service>:sha-<8chars>
|
# gitea.homelab.fhirworx.io/homelab/<service>:<8chars>
|
||||||
#
|
#
|
||||||
# Buildkit pushes via HTTP through Traefik (TLS terminated at edge).
|
# Buildkit pushes via HTTP through Traefik (TLS terminated at edge).
|
||||||
# DNS resolved by CoreDNS via Traefik UDP on the CI network.
|
# DNS resolved by CoreDNS via Traefik UDP on the CI network.
|
||||||
@@ -65,7 +65,7 @@ steps:
|
|||||||
dockerfile: notebooks/Dockerfile
|
dockerfile: notebooks/Dockerfile
|
||||||
context: notebooks/
|
context: notebooks/
|
||||||
tags:
|
tags:
|
||||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
- "${CI_COMMIT_SHA:0:8}"
|
||||||
- latest
|
- latest
|
||||||
|
|
||||||
- name: scan-notebooks
|
- name: scan-notebooks
|
||||||
@@ -86,7 +86,7 @@ steps:
|
|||||||
dockerfile: zotero/Dockerfile
|
dockerfile: zotero/Dockerfile
|
||||||
context: zotero/
|
context: zotero/
|
||||||
tags:
|
tags:
|
||||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
- "${CI_COMMIT_SHA:0:8}"
|
||||||
- latest
|
- latest
|
||||||
|
|
||||||
- name: scan-zotero
|
- name: scan-zotero
|
||||||
@@ -115,7 +115,7 @@ steps:
|
|||||||
dockerfile: docs/Dockerfile
|
dockerfile: docs/Dockerfile
|
||||||
context: .
|
context: .
|
||||||
tags:
|
tags:
|
||||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
- "${CI_COMMIT_SHA:0:8}"
|
||||||
- latest
|
- latest
|
||||||
depends_on:
|
depends_on:
|
||||||
- prep-docs-context
|
- prep-docs-context
|
||||||
@@ -138,7 +138,7 @@ steps:
|
|||||||
dockerfile: api/Dockerfile
|
dockerfile: api/Dockerfile
|
||||||
context: .
|
context: .
|
||||||
tags:
|
tags:
|
||||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
- "${CI_COMMIT_SHA:0:8}"
|
||||||
- latest
|
- latest
|
||||||
|
|
||||||
- name: scan-api
|
- name: scan-api
|
||||||
@@ -159,7 +159,7 @@ steps:
|
|||||||
dockerfile: rustfs/Dockerfile.mc
|
dockerfile: rustfs/Dockerfile.mc
|
||||||
context: rustfs/
|
context: rustfs/
|
||||||
tags:
|
tags:
|
||||||
- "sha-${CI_COMMIT_SHA:0:8}"
|
- "${CI_COMMIT_SHA:0:8}"
|
||||||
- latest
|
- latest
|
||||||
|
|
||||||
# ── Upload scan results ─────────────────────────────────────
|
# ── Upload scan results ─────────────────────────────────────
|
||||||
|
|||||||
@@ -489,11 +489,7 @@ services:
|
|||||||
- loki_data:/loki
|
- loki_data:/loki
|
||||||
command: -config.file=/etc/loki/local-config.yaml
|
command: -config.file=/etc/loki/local-config.yaml
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test:
|
test: ["CMD", "/usr/bin/loki", "--version"]
|
||||||
[
|
|
||||||
"CMD-SHELL",
|
|
||||||
"wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1",
|
|
||||||
]
|
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|||||||
@@ -6,10 +6,8 @@ Usage (in a Woodpecker ``failure`` step)::
|
|||||||
|
|
||||||
Reads CI environment variables to locate the failed pipeline, pulls
|
Reads CI environment variables to locate the failed pipeline, pulls
|
||||||
step logs via the Woodpecker API, parses any Python tracebacks, runs
|
step logs via the Woodpecker API, parses any Python tracebacks, runs
|
||||||
``git blame`` on the offending lines, and files a Gitea issue.
|
``git blame`` on the offending lines, and files a Gitea issue with
|
||||||
|
labels for the pipeline name, step category, and image (if applicable).
|
||||||
For non-Python failures (docker build errors, shell script crashes),
|
|
||||||
it still files an issue with the raw log output.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -20,6 +18,39 @@ import sys
|
|||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ── Label ID mapping ──────────────────────────────────────────────
|
||||||
|
# These IDs correspond to labels created in the Gitea repo.
|
||||||
|
PIPELINE_LABELS: dict[str, int] = {
|
||||||
|
"ci": 14,
|
||||||
|
"deploy": 15,
|
||||||
|
"harden": 16,
|
||||||
|
"infra-ci": 17,
|
||||||
|
"rebuild-all": 18,
|
||||||
|
"release": 19,
|
||||||
|
}
|
||||||
|
|
||||||
|
STEP_LABELS: dict[str, int] = {
|
||||||
|
"lint": 20,
|
||||||
|
"test": 21,
|
||||||
|
"build": 22,
|
||||||
|
"scan": 23,
|
||||||
|
"deploy": 24,
|
||||||
|
"publish": 25,
|
||||||
|
"hadolint": 26,
|
||||||
|
"validate": 27,
|
||||||
|
}
|
||||||
|
|
||||||
|
IMAGE_LABELS: dict[str, int] = {
|
||||||
|
"api": 28,
|
||||||
|
"notebooks": 29,
|
||||||
|
"zotero": 30,
|
||||||
|
"docs": 31,
|
||||||
|
"mc": 32,
|
||||||
|
}
|
||||||
|
|
||||||
|
INFRA_LABEL = 33
|
||||||
|
CI_LABEL = 13 # existing "ci" label
|
||||||
|
|
||||||
|
|
||||||
def _get_env(name: str) -> str:
|
def _get_env(name: str) -> str:
|
||||||
val = os.environ.get(name, "")
|
val = os.environ.get(name, "")
|
||||||
@@ -28,6 +59,49 @@ def _get_env(name: str) -> str:
|
|||||||
return val
|
return val
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_labels(pipeline_name: str, step_name: str) -> list[int]:
|
||||||
|
"""Resolve label IDs from pipeline and step names."""
|
||||||
|
labels: list[int] = [CI_LABEL]
|
||||||
|
|
||||||
|
# Pipeline label
|
||||||
|
for key, lid in PIPELINE_LABELS.items():
|
||||||
|
if key in pipeline_name:
|
||||||
|
labels.append(lid)
|
||||||
|
break
|
||||||
|
|
||||||
|
# Step category label — match the broadest category
|
||||||
|
step_lower = step_name.lower()
|
||||||
|
for key, lid in STEP_LABELS.items():
|
||||||
|
if key in step_lower:
|
||||||
|
labels.append(lid)
|
||||||
|
break
|
||||||
|
|
||||||
|
# Image label — detect which image the step relates to
|
||||||
|
for key, lid in IMAGE_LABELS.items():
|
||||||
|
if key in step_lower:
|
||||||
|
labels.append(lid)
|
||||||
|
break
|
||||||
|
|
||||||
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def _pipeline_name_from_env() -> str:
|
||||||
|
"""Best-effort pipeline name from CI env vars."""
|
||||||
|
event = os.environ.get("CI_PIPELINE_EVENT", "push")
|
||||||
|
branch = os.environ.get("CI_COMMIT_BRANCH", "")
|
||||||
|
|
||||||
|
if event == "tag":
|
||||||
|
return "release"
|
||||||
|
if event in ("manual", "cron"):
|
||||||
|
return "harden"
|
||||||
|
if event == "pull_request":
|
||||||
|
return "ci"
|
||||||
|
# push to main = deploy, push to other = ci
|
||||||
|
if branch == "main":
|
||||||
|
return "deploy"
|
||||||
|
return "ci"
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
@@ -54,6 +128,8 @@ def main() -> int:
|
|||||||
else:
|
else:
|
||||||
owner, repo = "homelab", "stack"
|
owner, repo = "homelab", "stack"
|
||||||
|
|
||||||
|
pipeline_name = _pipeline_name_from_env()
|
||||||
|
|
||||||
# Fetch pipeline steps and find failures
|
# Fetch pipeline steps and find failures
|
||||||
from api.clients.woodpecker import WoodpeckerClient
|
from api.clients.woodpecker import WoodpeckerClient
|
||||||
|
|
||||||
@@ -91,6 +167,8 @@ def main() -> int:
|
|||||||
|
|
||||||
log.info("Processing failed step: %s (id=%d)", step_name, step_id)
|
log.info("Processing failed step: %s (id=%d)", step_name, step_id)
|
||||||
|
|
||||||
|
labels = _resolve_labels(pipeline_name, step_name)
|
||||||
|
|
||||||
# Fetch logs — Woodpecker returns a list of log line dicts
|
# Fetch logs — Woodpecker returns a list of log line dicts
|
||||||
try:
|
try:
|
||||||
log_entries = wp.get_logs(rid, pnum, step_id)
|
log_entries = wp.get_logs(rid, pnum, step_id)
|
||||||
@@ -110,6 +188,8 @@ def main() -> int:
|
|||||||
# Try to parse Python tracebacks
|
# Try to parse Python tracebacks
|
||||||
reports = parse_traceback_text(log_text)
|
reports = parse_traceback_text(log_text)
|
||||||
|
|
||||||
|
sha_short = (commit_sha or "unknown")[:8]
|
||||||
|
|
||||||
if reports:
|
if reports:
|
||||||
# File one issue per traceback found
|
# File one issue per traceback found
|
||||||
for report in reports:
|
for report in reports:
|
||||||
@@ -121,11 +201,13 @@ def main() -> int:
|
|||||||
commit_sha or "unknown",
|
commit_sha or "unknown",
|
||||||
)
|
)
|
||||||
body = (
|
body = (
|
||||||
f"**Pipeline:** #{pnum} step `{step_name}`\n"
|
f"**Pipeline:** `{pipeline_name}` #{pnum} "
|
||||||
|
f"step `{step_name}`\n"
|
||||||
|
f"**Commit:** `{sha_short}`\n"
|
||||||
f"**Status:** `failure`\n\n" + body
|
f"**Status:** `failure`\n\n" + body
|
||||||
)
|
)
|
||||||
title = (
|
title = (
|
||||||
f"ci: {step_name} — {report.exc_type}: "
|
f"{pipeline_name}/{step_name}: {report.exc_type}: "
|
||||||
f"{_truncate(report.exc_value, 50)}"
|
f"{_truncate(report.exc_value, 50)}"
|
||||||
)
|
)
|
||||||
result = gitea.create_issue(
|
result = gitea.create_issue(
|
||||||
@@ -134,16 +216,17 @@ def main() -> int:
|
|||||||
{
|
{
|
||||||
"title": title,
|
"title": title,
|
||||||
"body": body,
|
"body": body,
|
||||||
"labels": [13],
|
"labels": labels,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
log.info("Filed issue #%s: %s", result.get("number"), title)
|
log.info("Filed issue #%s: %s", result.get("number"), title)
|
||||||
else:
|
else:
|
||||||
# Non-Python failure — file with raw log
|
# Non-Python failure — file with raw log
|
||||||
title = f"ci: {step_name} failed (pipeline #{pnum})"
|
title = f"{pipeline_name}/{step_name} failed (#{pnum} @ {sha_short})"
|
||||||
body = (
|
body = (
|
||||||
f"**Pipeline:** #{pnum} step `{step_name}`\n"
|
f"**Pipeline:** `{pipeline_name}` #{pnum} "
|
||||||
f"**Commit:** `{commit_sha}`\n"
|
f"step `{step_name}`\n"
|
||||||
|
f"**Commit:** `{sha_short}`\n"
|
||||||
f"**Status:** `failure`\n\n"
|
f"**Status:** `failure`\n\n"
|
||||||
f"## Step Log\n\n"
|
f"## Step Log\n\n"
|
||||||
f"<details>\n"
|
f"<details>\n"
|
||||||
@@ -157,7 +240,7 @@ def main() -> int:
|
|||||||
{
|
{
|
||||||
"title": title,
|
"title": title,
|
||||||
"body": body,
|
"body": body,
|
||||||
"labels": [13],
|
"labels": labels,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
log.info("Filed issue #%s: %s", result.get("number"), title)
|
log.info("Filed issue #%s: %s", result.get("number"), title)
|
||||||
|
|||||||
Reference in New Issue
Block a user