Merge pull request #18624 from deads2k/server-05-delayhandlercrash

Automatic merge from submit-queue (batch tested with PRs 18624, 18605).

allow a panic to crash the server after a delay

This allows a server to crash after it experiences a panic on a delay.  The delay allows us to break bootstrapping loops if we're panicing, but still making progress.  It also allows us to configure multiple handlers, so we can have this plus sentry by doing `OPENSHIFT_ON_PANIC=["crash-after-delay:10m", "sentry:https://foo"]`



@openshift/sig-master @smarterclayton @eparis @derekwaynecarr as discussed separately

Author: openshift-merge-robot

pkg/cmd/util/serviceability/panic.go

@@ -1,33 +1,70 @@
 package serviceability
 
 import (
+	"encoding/json"
 	"strings"
 	"time"
 
 	"github.com/golang/glog"
+
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 )
 
 // BehaviorOnPanic is a helper for setting the crash mode of OpenShift when a panic is caught.
 // It returns a function that should be the defer handler for the caller.
-func BehaviorOnPanic(mode string) (fn func()) {
-	fn = func() {}
+func BehaviorOnPanic(modeString string) func() {
+	modes := []string{}
+	if err := json.Unmarshal([]byte(modeString), &modes); err != nil {
+		return behaviorOnPanic(modeString)
+	}
+
+	fns := []func(){}
+
+	for _, mode := range modes {
+		fns = append(fns, behaviorOnPanic(mode))
+	}
+
+	return func() {
+		for _, fn := range fns {
+			fn()
+		}
+	}
+}
+
+func behaviorOnPanic(mode string) func() {
+	doNothing := func() {}
+
 	switch {
 	case mode == "crash":
 		glog.Infof("Process will terminate as soon as a panic occurs.")
 		utilruntime.ReallyCrash = true
+		return doNothing
+
+	case strings.HasPrefix(mode, "crash-after-delay:"):
+		delayDurationString := strings.TrimPrefix(mode, "crash-after-delay:")
+		delayDuration, err := time.ParseDuration(delayDurationString)
+		if err != nil {
+			glog.Errorf("Unable to start crash-after-delay.  Crashing immediately instead: %v", err)
+			utilruntime.ReallyCrash = true
+			return doNothing
+		}
+		glog.Infof("Process will terminate %v after a panic occurs.", delayDurationString)
+		utilruntime.ReallyCrash = false
+		utilruntime.PanicHandlers = append(utilruntime.PanicHandlers, crashOnDelay(delayDuration, delayDurationString))
+		return doNothing
+
 	case strings.HasPrefix(mode, "sentry:"):
 		url := strings.TrimPrefix(mode, "sentry:")
 		m, err := NewSentryMonitor(url)
 		if err != nil {
 			glog.Errorf("Unable to start Sentry for panic tracing: %v", err)
-			return
+			return doNothing
 		}
 		glog.Infof("Process will log all panics and errors to Sentry.")
 		utilruntime.ReallyCrash = false
 		utilruntime.PanicHandlers = append(utilruntime.PanicHandlers, m.CapturePanic)
 		utilruntime.ErrorHandlers = append(utilruntime.ErrorHandlers, m.CaptureError)
-		fn = func() {
+		return func() {
 			if r := recover(); r != nil {
 				m.CapturePanicAndWait(r, 2*time.Second)
 				panic(r)
@@ -36,8 +73,20 @@ func BehaviorOnPanic(mode string) (fn func()) {
 	case len(mode) == 0:
 		// default panic behavior
 		utilruntime.ReallyCrash = false
+		return doNothing
+
 	default:
 		glog.Errorf("Unrecognized panic behavior")
+		return doNothing
+	}
+}
+
+func crashOnDelay(delay time.Duration, delayString string) func(interface{}) {
+	return func(in interface{}) {
+		go func() {
+			glog.Errorf("Panic happened.  Process will crash in %v.", delayString)
+			time.Sleep(delay)
+			panic(in)
+		}()
 	}
-	return
 }

pkg/cmd/util/serviceability/panic_test.go

@@ -0,0 +1,23 @@
+package serviceability
+
+import (
+	"testing"
+	"time"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+)
+
+func TestPanicDelayingDeath(t *testing.T) {
+	BehaviorOnPanic(`["crash-after-delay:10s"]`)
+
+	utilruntime.ReallyCrash = false
+	go func() {
+		defer utilruntime.HandleCrash()
+		panic("not dead yet!")
+	}()
+
+	select {
+	case <-time.After(5 * time.Second):
+		t.Log("beat death!")
+	}
+}