UPSTREAM: <carry>: provide events, messages, and bodies for probe failures of important pods

Author: deads2k

pkg/kubelet/prober/patch_prober.go

@@ -0,0 +1,49 @@
+package prober
+
+import (
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/probe"
+	httpprobe "k8s.io/kubernetes/pkg/probe/http"
+)
+
+func (pb *prober) maybeProbeForBody(prober httpprobe.Prober, url *url.URL, headers http.Header, timeout time.Duration, pod *v1.Pod, container v1.Container, probeType probeType) (probe.Result, string, error) {
+	if !isInterestingPod(pod) {
+		return prober.Probe(url, headers, timeout)
+	}
+	bodyProber, ok := prober.(httpprobe.DetailedProber)
+	if !ok {
+		return prober.Probe(url, headers, timeout)
+	}
+	result, output, body, probeError := bodyProber.ProbeForBody(url, headers, timeout)
+	switch result {
+	case probe.Success:
+		return result, output, probeError
+	case probe.Warning, probe.Failure, probe.Unknown:
+		// these pods are interesting enough to show the body content
+		klog.Infof("interesting pod/%s container/%s namespace/%s: %s probe status=%v output=%q start-of-body=%s",
+			pod.Name, container.Name, pod.Namespace, probeType, result, output, body)
+
+		// in fact, they are so interesting we'll try to send events for them
+		pb.recordContainerEvent(pod, &container, v1.EventTypeWarning, "ProbeError", "%s probe error: %s\nbody: %s\n", probeType, output, body)
+		return result, output, probeError
+	default:
+		return result, output, probeError
+	}
+}
+
+func isInterestingPod(pod *v1.Pod) bool {
+	if pod == nil {
+		return false
+	}
+	if strings.HasPrefix(pod.Namespace, "openshift-") {
+		return true
+	}
+
+	return false
+}

pkg/kubelet/prober/prober.go

@@ -177,11 +177,11 @@ func (pb *prober) runProbe(probeType probeType, p *v1.Probe, pod *v1.Pod, status
 		klog.V(4).Infof("HTTP-Probe Headers: %v", headers)
 		switch probeType {
 		case liveness:
-			return pb.livenessHTTP.Probe(url, headers, timeout)
+			return pb.maybeProbeForBody(pb.livenessHTTP, url, headers, timeout, pod, container, probeType)
 		case startup:
-			return pb.startupHTTP.Probe(url, headers, timeout)
+			return pb.maybeProbeForBody(pb.startupHTTP, url, headers, timeout, pod, container, probeType)
 		default:
-			return pb.readinessHTTP.Probe(url, headers, timeout)
+			return pb.maybeProbeForBody(pb.readinessHTTP, url, headers, timeout, pod, container, probeType)
 		}
 	}
 	if p.TCPSocket != nil {

pkg/probe/http/http.go

@@ -76,7 +76,8 @@ func (pr httpProber) Probe(url *url.URL, headers http.Header, timeout time.Durat
 		Transport:     pr.transport,
 		CheckRedirect: redirectChecker(pr.followNonLocalRedirects),
 	}
-	return DoHTTPProbe(url, headers, client)
+	result, details, _, err := DoHTTPProbe(url, headers, client)
+	return result, details, err
 }
 
 // GetHTTPInterface is an interface for making HTTP requests, that returns a response and error.
@@ -88,11 +89,11 @@ type GetHTTPInterface interface {
 // If the HTTP response code is successful (i.e. 400 > code >= 200), it returns Success.
 // If the HTTP response code is unsuccessful or HTTP communication fails, it returns Failure.
 // This is exported because some other packages may want to do direct HTTP probes.
-func DoHTTPProbe(url *url.URL, headers http.Header, client GetHTTPInterface) (probe.Result, string, error) {
+func DoHTTPProbe(url *url.URL, headers http.Header, client GetHTTPInterface) (probe.Result, string, string, error) {
 	req, err := http.NewRequest("GET", url.String(), nil)
 	if err != nil {
 		// Convert errors into failures to catch timeouts.
-		return probe.Failure, err.Error(), nil
+		return probe.Failure, err.Error(), "", nil
 	}
 	if _, ok := headers["User-Agent"]; !ok {
 		if headers == nil {
@@ -114,28 +115,28 @@ func DoHTTPProbe(url *url.URL, headers http.Header, client GetHTTPInterface) (pr
 	res, err := client.Do(req)
 	if err != nil {
 		// Convert errors into failures to catch timeouts.
-		return probe.Failure, err.Error(), nil
+		return probe.Failure, err.Error(), "", nil
 	}
 	defer res.Body.Close()
 	b, err := utilio.ReadAtMost(res.Body, maxRespBodyLength)
 	if err != nil {
 		if err == utilio.ErrLimitReached {
 			klog.V(4).Infof("Non fatal body truncation for %s, Response: %v", url.String(), *res)
 		} else {
-			return probe.Failure, "", err
+			return probe.Failure, "", "", err
 		}
 	}
 	body := string(b)
 	if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
 		if res.StatusCode >= http.StatusMultipleChoices { // Redirect
 			klog.V(4).Infof("Probe terminated redirects for %s, Response: %v", url.String(), *res)
-			return probe.Warning, body, nil
+			return probe.Warning, body, body, nil
 		}
 		klog.V(4).Infof("Probe succeeded for %s, Response: %v", url.String(), *res)
-		return probe.Success, body, nil
+		return probe.Success, body, body, nil
 	}
 	klog.V(4).Infof("Probe failed for %s with request headers %v, response body: %v", url.String(), headers, body)
-	return probe.Failure, fmt.Sprintf("HTTP probe failed with statuscode: %d", res.StatusCode), nil
+	return probe.Failure, fmt.Sprintf("HTTP probe failed with statuscode: %d", res.StatusCode), body, nil
 }
 
 func redirectChecker(followNonLocalRedirects bool) func(*http.Request, []*http.Request) error {

pkg/probe/http/patch_http.go

@@ -0,0 +1,26 @@
+package http
+
+import (
+	"net/http"
+	"net/url"
+	"time"
+
+	"k8s.io/kubernetes/pkg/probe"
+)
+
+// Prober is an interface that defines the Probe function for doing HTTP readiness/liveness checks.
+type DetailedProber interface {
+	ProbeForBody(url *url.URL, headers http.Header, timeout time.Duration) (probe.Result, string, string, error)
+}
+
+// ProbeForBody returns a ProbeRunner capable of running an HTTP check.
+// returns result, details, body, error
+func (pr httpProber) ProbeForBody(url *url.URL, headers http.Header, timeout time.Duration) (probe.Result, string, string, error) {
+	pr.transport.DisableCompression = true // removes Accept-Encoding header
+	client := &http.Client{
+		Timeout:       timeout,
+		Transport:     pr.transport,
+		CheckRedirect: redirectChecker(pr.followNonLocalRedirects),
+	}
+	return DoHTTPProbe(url, headers, client)
+}