Make TerminationGracePeriodSeconds configurable

- related to issue #419

Author: ChunyiLyu

api/v1beta1/rabbitmqcluster_types.go

@@ -63,6 +63,12 @@ type RabbitmqClusterSpec struct {
 	// Has no effect if the cluster only consists of one node.
 	// For more information, see https://www.rabbitmq.com/rabbitmq-queues.8.html#rebalance
 	SkipPostDeploySteps bool `json:"skipPostDeploySteps,omitempty"`
+	// TerminationGracePeriodSeconds is the timeout that each rabbitmqcluster pod will have to terminate gracefully
+	// It defaults to 604800 seconds ( a week long) to ensure that the container preStop lifecycle hook can finish running
+	// For more information, see: https://github.com/rabbitmq/cluster-operator/blob/main/docs/design/20200520-graceful-pod-termination.md
+	// +kubebuilder:validation:Minimum:=0
+	// +kubebuilder:default:=604800
+	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
 }
 
 type RabbitmqClusterOverrideSpec struct {

api/v1beta1/rabbitmqcluster_types_test.go

@@ -179,6 +179,7 @@ var _ = Describe("RabbitmqCluster", func() {
 
 			When("CR is fully populated", func() {
 				It("outputs the CR", func() {
+					zero := int64(0)
 					storage := k8sresource.MustParse("987Gi")
 					storageClassName := "some-class"
 					rmqClusterInstance = RabbitmqCluster{
@@ -187,9 +188,10 @@ var _ = Describe("RabbitmqCluster", func() {
 							Namespace: "default",
 						},
 						Spec: RabbitmqClusterSpec{
-							Replicas:         &three,
-							Image:            "rabbitmq-image-from-cr",
-							ImagePullSecrets: []corev1.LocalObjectReference{{Name: "my-super-secret"}},
+							Replicas:                      &three,
+							Image:                         "rabbitmq-image-from-cr",
+							ImagePullSecrets:              []corev1.LocalObjectReference{{Name: "my-super-secret"}},
+							TerminationGracePeriodSeconds: &zero,
 							Service: RabbitmqClusterServiceSpec{
 								Type: "NodePort",
 								Annotations: map[string]string{
@@ -476,8 +478,9 @@ func generateRabbitmqClusterObject(clusterName string) *RabbitmqCluster {
 			Namespace: "default",
 		},
 		Spec: RabbitmqClusterSpec{
-			Replicas: pointer.Int32Ptr(1),
-			Image:    "rabbitmq:3.8.9",
+			Replicas:                      pointer.Int32Ptr(1),
+			Image:                         "rabbitmq:3.8.9",
+			TerminationGracePeriodSeconds: pointer.Int64Ptr(604800),
 			Service: RabbitmqClusterServiceSpec{
 				Type: "ClusterIP",
 			},

api/v1beta1/zz_generated.deepcopy.go

@@ -3723,6 +3723,15 @@ spec:
                   the cluster only consists of one node. For more information, see
                   https://www.rabbitmq.com/rabbitmq-queues.8.html#rebalance
                 type: boolean
+              terminationGracePeriodSeconds:
+                default: 604800
+                description: 'TerminationGracePeriodSeconds is the timeout that each
+                  rabbitmqcluster pod will have to terminate gracefully It defaults
+                  to 604800 seconds ( a week long) to ensure that the container preStop
+                  lifecycle hook can finish running For more information, see: https://github.com/rabbitmq/cluster-operator/blob/main/docs/design/20200520-graceful-pod-termination.md'
+                format: int64
+                minimum: 0
+                type: integer
               tls:
                 properties:
                   caCertName:

config/crd/bases/rabbitmq.com_rabbitmqclusters.yaml

@@ -29,10 +29,9 @@ import (
 )
 
 const (
-	defaultGracePeriodTimeoutSeconds int64  = 60 * 60 * 24 * 7
-	initContainerCPU                 string = "100m"
-	initContainerMemory              string = "500Mi"
-	DeletionMarker                   string = "skipPreStopChecks"
+	initContainerCPU    string = "100m"
+	initContainerMemory string = "500Mi"
+	DeletionMarker      string = "skipPreStopChecks"
 )
 
 type StatefulSetBuilder struct {
@@ -247,8 +246,6 @@ func (builder *StatefulSetBuilder) podTemplateSpec(annotations, labels map[strin
 	rabbitmqGID := int64(999)
 	rabbitmqUID := int64(999)
 
-	terminationGracePeriod := defaultGracePeriodTimeoutSeconds
-
 	volumes := []corev1.Volume{
 		{
 			Name: "server-conf",
@@ -512,7 +509,7 @@ func (builder *StatefulSetBuilder) podTemplateSpec(annotations, labels map[strin
 				RunAsUser:  &rabbitmqUID,
 			},
 			ImagePullSecrets:              builder.Instance.Spec.ImagePullSecrets,
-			TerminationGracePeriodSeconds: &terminationGracePeriod,
+			TerminationGracePeriodSeconds: builder.Instance.Spec.TerminationGracePeriodSeconds,
 			ServiceAccountName:            builder.Instance.ChildResourceName(serviceAccountName),
 			AutomountServiceAccountToken:  &automountServiceAccountToken,
 			Affinity:                      builder.Instance.Spec.Affinity,
@@ -653,7 +650,10 @@ func (builder *StatefulSetBuilder) podTemplateSpec(annotations, labels map[strin
 									fmt.Sprintf("if [ ! -z \"$(cat /etc/pod-info/%s)\" ]; then exit 0; fi;", DeletionMarker) +
 										fmt.Sprintf(" rabbitmq-upgrade await_online_quorum_plus_one -t %d;"+
 											" rabbitmq-upgrade await_online_synchronized_mirror -t %d;"+
-											" rabbitmq-upgrade drain -t %d", defaultGracePeriodTimeoutSeconds, defaultGracePeriodTimeoutSeconds, defaultGracePeriodTimeoutSeconds),
+											" rabbitmq-upgrade drain -t %d",
+											*builder.Instance.Spec.TerminationGracePeriodSeconds,
+											*builder.Instance.Spec.TerminationGracePeriodSeconds,
+											*builder.Instance.Spec.TerminationGracePeriodSeconds),
 								},
 							},
 						},

internal/resource/statefulset.go

@@ -1022,13 +1022,22 @@ var _ = Describe("StatefulSet", func() {
 			}))
 		})
 
-		It("adds the required terminationGracePeriodSeconds", func() {
+		It("sets TerminationGracePeriodSeconds in podTemplate as provided in instance spec", func() {
+			instance.Spec.TerminationGracePeriodSeconds = pointer.Int64Ptr(10)
+			builder = &resource.RabbitmqResourceBuilder{
+				Instance: &instance,
+				Scheme:   scheme,
+			}
+
 			stsBuilder := builder.StatefulSet()
 			Expect(stsBuilder.Update(statefulSet)).To(Succeed())
 
 			gracePeriodSeconds := statefulSet.Spec.Template.Spec.TerminationGracePeriodSeconds
-			expectedGracePeriodSeconds := int64(60 * 60 * 24 * 7)
-			Expect(gracePeriodSeconds).To(Equal(&expectedGracePeriodSeconds))
+			Expect(gracePeriodSeconds).To(Equal(pointer.Int64Ptr(10)))
+
+			// TerminationGracePeriodSeconds is used to set commands timeouts in the preStop hook
+			expectedPreStopCommand := []string{"/bin/bash", "-c", "if [ ! -z \"$(cat /etc/pod-info/skipPreStopChecks)\" ]; then exit 0; fi; rabbitmq-upgrade await_online_quorum_plus_one -t 10; rabbitmq-upgrade await_online_synchronized_mirror -t 10; rabbitmq-upgrade drain -t 10"}
+			Expect(statefulSet.Spec.Template.Spec.Containers[0].Lifecycle.PreStop.Exec.Command).To(Equal(expectedPreStopCommand))
 		})
 
 		It("checks mirror and querum queue status in preStop hook", func() {
@@ -1473,9 +1482,10 @@ func generateRabbitmqCluster() rabbitmqv1beta1.RabbitmqCluster {
 			Namespace: "foo-namespace",
 		},
 		Spec: rabbitmqv1beta1.RabbitmqClusterSpec{
-			Replicas:         &one,
-			Image:            "rabbitmq-image-from-cr",
-			ImagePullSecrets: []corev1.LocalObjectReference{{Name: "my-super-secret"}},
+			Replicas:                      &one,
+			Image:                         "rabbitmq-image-from-cr",
+			ImagePullSecrets:              []corev1.LocalObjectReference{{Name: "my-super-secret"}},
+			TerminationGracePeriodSeconds: pointer.Int64Ptr(604800),
 			Service: rabbitmqv1beta1.RabbitmqClusterServiceSpec{
 				Type:        "this-is-a-service",
 				Annotations: map[string]string{},