stack/dev/scripts/nb_issue_filer.py

"""Dedup/auto-close Gitea issue filer for notebook errors.

One open issue per error signature (notebook + ename + normalized message).
Recurrences add a rate-limited comment instead of a new issue; signatures
that stop occurring get auto-closed. This replaces the api.diag.ci
one-issue-per-run pattern that accumulated 23 duplicates for a single
workflow.

Stdlib only — runs on a bare python image (nb-watcher sidecar), in CI, and
on the host.

Usage:
    # file/refresh issues for findings on stdin (JSON list, see FINDING)
    python nb_issue_filer.py report --source nightly-integration < findings.json

    # close open nb issues whose signature is NOT in the active set
    python nb_issue_filer.py sweep --active-sigs sig1,sig2 --source nightly-integration
    # or close signatures unseen since a state file's cutoff
    python nb_issue_filer.py sweep --state data/nb-watcher-state.json --max-age-h 24

FINDING = {"notebook": "x.py", "ename": "exception", "evalue": "...",
           "detail": "traceback/console excerpt"}
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import re
import sys
import time
import urllib.parse
import urllib.request
from pathlib import Path

# Repo root when run in-tree; "/" in shallow deployed contexts (sidecar
# mount, docker cp) where ROOT is only a .env-fallback location anyway.
_parents = Path(__file__).resolve().parents
ROOT = _parents[2] if len(_parents) > 2 else Path("/")

API_BASE = os.environ.get("GITEA_API_BASE", "https://git.fhirworx.io/api/v1")
OWNER_REPO = os.environ.get("GITEA_OWNER_REPO", "homelab/stack")
LABEL_NAME = os.environ.get("NB_ISSUE_LABEL", "notebooks")
MARKER = "nb-sig:"
COOLDOWN_S = int(os.environ.get("NB_ISSUE_COOLDOWN_S", str(6 * 3600)))


# ── pure logic ───────────────────────────────────────────────────


def normalize_evalue(evalue: str) -> str:
    """Collapse volatile details so recurring errors hash identically."""
    s = evalue.strip()
    s = re.sub(r"0x[0-9a-fA-F]+", "0xN", s)  # addresses
    s = re.sub(r"(/[\w.\-]+)+", "/PATH", s)  # absolute paths
    s = re.sub(r"\d+(\.\d+)?", "N", s)  # counts, line numbers, durations
    s = re.sub(r"\s+", " ", s)
    return s.lower()[:300]


def signature(notebook: str, ename: str, evalue: str) -> str:
    key = f"{notebook}|{ename}|{normalize_evalue(evalue)}"
    return hashlib.sha1(key.encode()).hexdigest()[:12]


def decide(
    existing: dict | None, last_comment_age_s: float | None, cooldown_s: int
) -> str:
    """create | comment | skip for one observed signature."""
    if existing is None:
        return "create"
    if last_comment_age_s is None or last_comment_age_s >= cooldown_s:
        return "comment"
    return "skip"


def build_body(
    *, notebook: str, ename: str, evalue: str, source: str, detail: str, sig: str
) -> str:
    detail = detail.strip()[:4000]
    return (
        f"Auto-filed by the notebook error filer (`{MARKER}{sig}` — do not edit "
        f"this marker; dedup and auto-close key on it).\n\n"
        f"- **Notebook:** `{notebook}`\n"
        f"- **Error:** `{ename}`: {evalue[:500]}\n"
        f"- **Source:** {source}\n\n"
        f"```\n{detail or '(no traceback captured)'}\n```\n\n"
        f"This issue is closed automatically when the error stops occurring."
    )


def extract_sig(body: str) -> str | None:
    m = re.search(rf"{MARKER}([0-9a-f]{{12}})", body or "")
    return m.group(1) if m else None


# ── Gitea API (stdlib) ───────────────────────────────────────────


def _token() -> str:
    tok = os.environ.get("GITEA_TOKEN", "")
    if not tok:
        env_file = ROOT / ".env"
        if env_file.exists():
            for line in env_file.read_text().splitlines():
                if line.startswith("GITEA_TOKEN="):
                    tok = line.split("=", 1)[1].strip().strip('"').strip("'")
    if not tok:
        print("ERROR: GITEA_TOKEN not set and not found in .env", file=sys.stderr)
        raise SystemExit(2)
    return tok


def api(method: str, path: str, data: dict | None = None) -> tuple[int, object]:
    url = f"{API_BASE}/{path}"
    body = json.dumps(data).encode() if data is not None else None
    req = urllib.request.Request(
        url,
        data=body,
        method=method,
        headers={
            "Authorization": f"token {_token()}",
            "Content-Type": "application/json",
        },
    )
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            raw = resp.read().decode()
            return resp.status, (json.loads(raw) if raw else None)
    except urllib.error.HTTPError as e:  # noqa: PERF203 — single call site
        return e.code, e.read().decode()[:300]
    except urllib.error.URLError as e:
        return 0, str(e)


def _open_nb_issues() -> list[dict]:
    """All open issues carrying our marker, keyed lookup done by caller."""
    issues: list[dict] = []
    page = 1
    while True:
        status, batch = api(
            "GET",
            f"repos/{OWNER_REPO}/issues?state=open&type=issues"
            f"&q={urllib.parse.quote(MARKER)}&page={page}&limit=50",
        )
        if status != 200 or not isinstance(batch, list) or not batch:
            break
        issues.extend(batch)
        if len(batch) < 50:
            break
        page += 1
    return [i for i in issues if extract_sig(i.get("body", ""))]


def _label_id() -> int | None:
    status, labels = api("GET", f"repos/{OWNER_REPO}/labels?limit=50")
    if status == 200 and isinstance(labels, list):
        for lab in labels:
            if lab.get("name") == LABEL_NAME:
                return lab.get("id")
    return None


def _last_comment_age_s(issue: dict) -> float | None:
    ts = issue.get("updated_at")
    if not ts:
        return None
    try:
        then = time.mktime(time.strptime(ts[:19], "%Y-%m-%dT%H:%M:%S"))
        return max(0.0, time.time() - time.timezone - (then - time.timezone))
    except ValueError:
        return None


# ── commands ─────────────────────────────────────────────────────


def cmd_report(findings: list[dict], source: str) -> int:
    open_by_sig = {extract_sig(i["body"]): i for i in _open_nb_issues()}
    label = _label_id()
    seen: set[str] = set()
    failures = 0

    for f in findings:
        sig = signature(f["notebook"], f["ename"], f.get("evalue", ""))
        if sig in seen:  # same error many times in one run → one action
            continue
        seen.add(sig)
        existing = open_by_sig.get(sig)
        age = _last_comment_age_s(existing) if existing else None
        action = decide(existing, age, COOLDOWN_S)

        if action == "create":
            payload: dict = {
                "title": f"[nb] {f['notebook']}: {f['ename']}: "
                f"{f.get('evalue', '')[:120]}",
                "body": build_body(
                    notebook=f["notebook"],
                    ename=f["ename"],
                    evalue=f.get("evalue", ""),
                    source=source,
                    detail=f.get("detail", ""),
                    sig=sig,
                ),
            }
            if label:
                payload["labels"] = [label]
            status, resp = api("POST", f"repos/{OWNER_REPO}/issues", payload)
            if status == 201 and isinstance(resp, dict):
                print(f"  created #{resp.get('number')} [{sig}] {f['notebook']}")
            else:
                print(f"  CREATE FAILED ({status}) [{sig}]: {resp}", file=sys.stderr)
                failures += 1
        elif action == "comment":
            n = existing["number"]
            status, _ = api(
                "POST",
                f"repos/{OWNER_REPO}/issues/{n}/comments",
                {
                    "body": f"Still occurring via {source} at "
                    f"{time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime())}."
                },
            )
            print(
                f"  commented #{n} [{sig}]"
                if status == 201
                else f"  COMMENT FAILED ({status}) #{n}"
            )
        else:
            print(f"  skip [{sig}] (within cooldown)")

    return 1 if failures else 0


def cmd_sweep(active_sigs: set[str], source: str) -> int:
    closed = 0
    for issue in _open_nb_issues():
        sig = extract_sig(issue["body"])
        if sig in active_sigs:
            continue
        n = issue["number"]
        api(
            "POST",
            f"repos/{OWNER_REPO}/issues/{n}/comments",
            {"body": f"No longer occurring ({source}); auto-closing."},
        )
        status, _ = api("PATCH", f"repos/{OWNER_REPO}/issues/{n}", {"state": "closed"})
        if status in (200, 201):
            print(f"  closed #{n} [{sig}]")
            closed += 1
        else:
            print(f"  CLOSE FAILED ({status}) #{n}", file=sys.stderr)
    print(f"sweep: closed {closed}")
    return 0


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    sub = parser.add_subparsers(dest="cmd", required=True)

    p_report = sub.add_parser("report", help="file/refresh issues for findings")
    p_report.add_argument("--source", required=True)
    p_report.add_argument("--findings", help="path to findings JSON (default: stdin)")

    p_sweep = sub.add_parser("sweep", help="close issues for inactive signatures")
    p_sweep.add_argument("--source", required=True)
    p_sweep.add_argument(
        "--active-sigs", default="", help="comma-separated signatures still failing"
    )
    p_sweep.add_argument("--state", help="watcher state JSON: {sig: last_seen_epoch}")
    p_sweep.add_argument("--max-age-h", type=float, default=24.0)

    args = parser.parse_args()

    if args.cmd == "report":
        raw = Path(args.findings).read_text() if args.findings else sys.stdin.read()
        findings = json.loads(raw) if raw.strip() else []
        if not findings:
            print("no findings — nothing to file")
            return 0
        return cmd_report(findings, args.source)

    active: set[str] = {s for s in args.active_sigs.split(",") if s}
    if args.state and Path(args.state).exists():
        state = json.loads(Path(args.state).read_text())
        cutoff = time.time() - args.max_age_h * 3600
        active |= {sig for sig, ts in state.items() if ts >= cutoff}
    return cmd_sweep(active, args.source)


if __name__ == "__main__":
    raise SystemExit(main())