Allow to set a livenessProbe for the sidecar (FoundationDB#666)

Author: johscheuer

api/v1beta1/foundationdbcluster_types.go

@@ -1882,6 +1882,10 @@ type FoundationDBStatusBackupTag struct {
 // ContainerOverrides provides options for customizing a container created by
 // the operator.
 type ContainerOverrides struct {
+	// EnableLivenessProbe defines if the sidecar should have a livenessProbe in addition
+	// to the readinessProbe. This setting will be enabled per default in the 1.0.0 release.
+	// This setting will be ignored on the main container.
+	EnableLivenessProbe bool `json:"enableLivenessProbe,omitempty"`
 
 	// EnableTLS controls whether we should be listening on a TLS connection.
 	EnableTLS bool `json:"enableTls,omitempty"`

config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml

@@ -1295,6 +1295,8 @@ spec:
                   type: string
                 mainContainer:
                   properties:
+                    enableLivenessProbe:
+                      type: boolean
                     enableTls:
                       type: boolean
                     env:
@@ -7433,6 +7435,8 @@ spec:
                   type: object
                 sidecarContainer:
                   properties:
+                    enableLivenessProbe:
+                      type: boolean
                     enableTls:
                       type: boolean
                     env:

controllers/pod_models.go

@@ -518,6 +518,21 @@ func configureSidecarContainer(container *corev1.Container, initMode bool, insta
 		}
 
 		sidecarEnv = append(sidecarEnv, corev1.EnvVar{Name: "FDB_INSTANCE_ID", Value: instanceID})
+
+		if !initMode && cluster.Spec.SidecarContainer.EnableLivenessProbe && container.LivenessProbe == nil {
+			// We can't use a HTTP handler here since the server
+			// requires a client certificate
+			container.LivenessProbe = &corev1.Probe{
+				Handler: corev1.Handler{
+					TCPSocket: &corev1.TCPSocketAction{
+						Port: intstr.IntOrString{IntVal: 8080},
+					},
+				},
+				TimeoutSeconds:   1,
+				PeriodSeconds:    30,
+				FailureThreshold: 5,
+			}
+		}
 	}
 
 	if version.PrefersCommandLineArgumentsInSidecar() && initMode {

controllers/pod_models_test.go

@@ -320,6 +320,25 @@ var _ = Describe("pod_models", func() {
 			It("should have no affinity rules", func() {
 				Expect(spec.Affinity).To(BeNil())
 			})
+
+			Context("with the livenessProbe enabled", func() {
+				BeforeEach(func() {
+					cluster.Spec.SidecarContainer.EnableLivenessProbe = true
+					spec, err = GetPodSpec(cluster, fdbtypes.ProcessClassStorage, 1)
+				})
+
+				It("should have a livenessProbe for the sidecar", func() {
+					sidecarContainer := spec.Containers[1]
+					Expect(sidecarContainer.Name).To(Equal("foundationdb-kubernetes-sidecar"))
+					Expect(sidecarContainer.LivenessProbe).NotTo(BeNil())
+				})
+
+				It("should not have a livenessProbe for the init container", func() {
+					sidecarContainer := spec.InitContainers[0]
+					Expect(sidecarContainer.Name).To(Equal("foundationdb-kubernetes-init"))
+					Expect(sidecarContainer.LivenessProbe).To(BeNil())
+				})
+			})
 		})
 
 		Context("with an instance that is crash looping", func() {

docs/cluster_spec.md

@@ -126,6 +126,7 @@ ContainerOverrides provides options for customizing a container created by the o
 
 | Field | Description | Scheme | Required |
 | ----- | ----------- | ------ | -------- |
+| enableLivenessProbe | EnableLivenessProbe defines if the sidecar should have a livenessProbe in addition to the readinessProbe. This setting will be enabled per default in the 1.0.0 release. This setting will be ignored on the main container. | bool | false |
 | enableTls | EnableTLS controls whether we should be listening on a TLS connection. | bool | false |
 | peerVerificationRules | PeerVerificationRules provides the rules for what client certificates the process should accept. | string | false |
 | env | Env provides environment variables.  **Deprecated: Use the PodTemplate field instead.** | [][corev1.EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvar-v1-core) | false |