example,pkg/test/e2eutil,test/e2e: update go e2e test to support and check for status conditions

Author: joelanford

example/memcached-operator/memcached_controller.go.tmpl

@@ -1,3 +1,17 @@
+// Copyright 2018 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package memcached
 
 import (
@@ -6,6 +20,7 @@ import (
 
 	cachev1alpha1 "github.com/example-inc/memcached-operator/pkg/apis/cache/v1alpha1"
 
+	"github.com/operator-framework/operator-sdk/pkg/status"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -108,31 +123,48 @@ func (r *ReconcileMemcached) Reconcile(request reconcile.Request) (reconcile.Res
 
 	// Check if the deployment already exists, if not create a new one
 	found := &appsv1.Deployment{}
-	err = r.client.Get(context.TODO(), types.NamespacedName{Name: memcached.Name, Namespace: memcached.Namespace}, found)
-	if err != nil && errors.IsNotFound(err) {
+	getErr := r.client.Get(context.TODO(), types.NamespacedName{Name: memcached.Name, Namespace: memcached.Namespace}, found)
+	if getErr != nil && errors.IsNotFound(getErr) {
 		// Define a new deployment
 		dep := r.deploymentForMemcached(memcached)
 		reqLogger.Info("Creating a new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
-		err = r.client.Create(context.TODO(), dep)
-		if err != nil {
-			reqLogger.Error(err, "Failed to create new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
-			return reconcile.Result{}, err
+		if createErr := r.client.Create(context.TODO(), dep); createErr != nil {
+			reqLogger.Error(createErr, "Failed to create new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
+
+			// Update status not ready
+			if setStatusErr := r.setStatusNotReady(memcached, createErr); setStatusErr != nil {
+				return reconcile.Result{}, setStatusErr
+			}
+			return reconcile.Result{}, createErr
+		}
+		// Update status not ready
+		if setStatusErr := r.setStatusNotReady(memcached, getErr); setStatusErr != nil {
+			return reconcile.Result{}, setStatusErr
 		}
 		// Deployment created successfully - return and requeue
 		return reconcile.Result{Requeue: true}, nil
-	} else if err != nil {
-		reqLogger.Error(err, "Failed to get Deployment")
+	} else if getErr != nil {
+		reqLogger.Error(getErr, "Failed to get Deployment")
+
+		// Update status not ready
+		if setStatusErr := r.setStatusNotReady(memcached, getErr); setStatusErr != nil {
+			return reconcile.Result{}, setStatusErr
+		}
 		return reconcile.Result{}, err
 	}
 
 	// Ensure the deployment size is the same as the spec
 	size := memcached.Spec.Size
 	if *found.Spec.Replicas != size {
 		found.Spec.Replicas = &size
-		err = r.client.Update(context.TODO(), found)
-		if err != nil {
-			reqLogger.Error(err, "Failed to update Deployment", "Deployment.Namespace", found.Namespace, "Deployment.Name", found.Name)
-			return reconcile.Result{}, err
+		if updateErr := r.client.Update(context.TODO(), found); updateErr != nil {
+			reqLogger.Error(updateErr, "Failed to update Deployment", "Deployment.Namespace", found.Namespace, "Deployment.Name", found.Name)
+
+			// Update status not ready
+			if setStatusErr := r.setStatusNotReady(memcached, updateErr); setStatusErr != nil {
+				return reconcile.Result{}, setStatusErr
+			}
+			return reconcile.Result{}, updateErr
 		}
 		// Spec updated - return and requeue
 		return reconcile.Result{Requeue: true}, nil
@@ -153,13 +185,18 @@ func (r *ReconcileMemcached) Reconcile(request reconcile.Request) (reconcile.Res
 	// Update status.Nodes if needed
 	if !reflect.DeepEqual(podNames, memcached.Status.Nodes) {
 		memcached.Status.Nodes = podNames
-		err := r.client.Status().Update(context.TODO(), memcached)
+		err := r.updateStatus(memcached)
 		if err != nil {
 			reqLogger.Error(err, "Failed to update Memcached status")
 			return reconcile.Result{}, err
 		}
 	}
 
+	// Set the Memcached status Ready condition to True, if necessary
+	if setStatusErr := r.setStatusReady(memcached); setStatusErr != nil {
+		return reconcile.Result{}, setStatusErr
+	}
+
 	return reconcile.Result{}, nil
 }
 
@@ -219,3 +256,44 @@ func getPodNames(pods []corev1.Pod) []string {
 	}
 	return podNames
 }
+
+func (r *ReconcileMemcached) updateStatus(m *cachev1alpha1.Memcached) error {
+	if m.Status.Nodes == nil {
+		m.Status.Nodes = []string{}
+	}
+	if m.Status.Conditions == nil {
+		m.Status.Conditions = status.Conditions{}
+	}
+	return r.client.Status().Update(context.TODO(), m)
+}
+
+func (r *ReconcileMemcached) setStatusReady(m *cachev1alpha1.Memcached) error {
+	if !m.Status.Conditions.IsTrue(status.ConditionType("Ready")) {
+		if setErr := m.Status.Conditions.SetCondition(&status.BasicCondition{
+			Type:   status.ConditionType("Ready"),
+			Status: corev1.ConditionTrue,
+		}); setErr != nil {
+			return setErr
+		}
+		if updateErr := r.updateStatus(m); updateErr != nil {
+			return updateErr
+		}
+	}
+	return nil
+}
+
+func (r *ReconcileMemcached) setStatusNotReady(m *cachev1alpha1.Memcached, origErr error) error {
+	if !m.Status.Conditions.IsFalse(status.ConditionType("Ready")) {
+		if setErr := m.Status.Conditions.SetCondition(&status.BasicCondition{
+			Type:    status.ConditionType("Ready"),
+			Status:  corev1.ConditionFalse,
+			Message: origErr.Error(),
+		}); setErr != nil {
+			return setErr
+		}
+		if updateErr := r.updateStatus(m); updateErr != nil {
+			return updateErr
+		}
+	}
+	return nil
+}

pkg/test/e2eutil/wait_util.go

@@ -16,11 +16,14 @@ package e2eutil
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
 	"time"
 
+	"github.com/operator-framework/operator-sdk/pkg/status"
 	"github.com/operator-framework/operator-sdk/pkg/test"
 
+	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -71,6 +74,8 @@ func waitForDeployment(t *testing.T, kubeclient kubernetes.Interface, namespace,
 	return nil
 }
 
+// WaitForDeletion checks to see if a given object is deleted, trying every retryInterval until the timeout has elapsed.
+// If the object has not been deleted before the timeout, the function returns an error.
 func WaitForDeletion(t *testing.T, dynclient client.Client, obj runtime.Object, retryInterval, timeout time.Duration) error {
 	key, err := client.ObjectKeyFromObject(obj)
 	if err != nil {
@@ -97,3 +102,58 @@ func WaitForDeletion(t *testing.T, dynclient client.Client, obj runtime.Object,
 	t.Logf("%s %s was deleted\n", kind, key)
 	return nil
 }
+
+// WaitForCondition checks to see if a given object has a given condition type/status pair, trying every retryInterval until
+// the timeout has elapsed. If the condition is not found before the timeout, the function returns an error.
+func WaitForCondition(t *testing.T, dynclient client.Client, obj runtime.Object, cType status.ConditionType, cStatus v1.ConditionStatus, retryInterval, timeout time.Duration) error {
+	key, err := client.ObjectKeyFromObject(obj)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	err = wait.Poll(retryInterval, timeout, func() (done bool, err error) {
+		err = dynclient.Get(ctx, key, obj)
+		if err != nil {
+			return false, err
+		}
+
+		data, err := json.Marshal(obj)
+		if err != nil {
+			return false, err
+		}
+
+		var cObj struct {
+			Status struct {
+				Conditions status.Conditions `json:"conditions"`
+			} `json:"status"`
+		}
+		err = json.Unmarshal(data, &cObj)
+		if err != nil {
+			return false, err
+		}
+
+		var c status.BasicCondition
+		err = cObj.Status.Conditions.GetCondition(cType, &c)
+		if err != nil {
+			if err != status.ErrConditionNotFound {
+				return false, err
+			}
+			t.Logf("waiting for status %s %s, condition not found", cType, cStatus)
+			return false, nil
+		}
+
+		if cStatus != c.GetStatus() {
+			t.Logf("waiting for status %s %s, got %s", cType, cStatus, c.GetStatus())
+			return false, nil
+		}
+
+		return true, nil
+	})
+	if err != nil {
+		return err
+	}
+	t.Logf("Found condition %s %s\n", cType, cStatus)
+	return nil
+}

test/e2e/memcached_test.go

@@ -484,7 +484,10 @@ func memcachedScaleTest(t *testing.T, f *framework.Framework, ctx *framework.Tes
 	}
 
 	// wait for example-memcached to reach 4 replicas
-	return e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-memcached", 4, retryInterval, timeout)
+	if err := e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-memcached", 4, retryInterval, timeout); err != nil {
+		return err
+	}
+	return e2eutil.WaitForCondition(t, f.Client.Client, &obj, "Ready", v1.ConditionTrue, retryInterval, timeout)
 }
 
 func MemcachedLocal(t *testing.T) {