Make autoscaler e2e test more stable

Author: fseldow

Diff for: tests/e2e/autoscaling/autoscaler.go

@@ -17,7 +17,6 @@ limitations under the License.
 package autoscaling
 
 import (
-	"fmt"
 	"strconv"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -80,6 +79,7 @@ var _ = Describe("Cluster size autoscaler [Serial][Slow]", func() {
 		emptyQuantity.Sub(runningQuantity)
 
 		podCount = int(emptyQuantity.MilliValue()/podSize) + 1
+		utils.Logf("%vm space are already in use", runningQuantity.Value())
 		utils.Logf("will create %v pod, each %vm size", podCount, podSize)
 	})
 
@@ -107,24 +107,6 @@ var _ = Describe("Cluster size autoscaler [Serial][Slow]", func() {
 		podCount = 0
 	})
 
-	It("should scale up if created pods exceed the node capacity [Feature:Autoscaling]", func() {
-		utils.Logf("creating pods")
-		for i := 0; i < podCount; i++ {
-			pod := createScalerPodManifest(fmt.Sprintf("%s-pod-%v", basename, i))
-			_, err := cs.CoreV1().Pods(ns.Name).Create(pod)
-			Expect(err).NotTo(HaveOccurred())
-		}
-		defer func() {
-			utils.Logf("deleting pods")
-			err := utils.DeletePodsInNamespace(cs, ns.Name)
-			Expect(err).NotTo(HaveOccurred())
-		}()
-		By("scale up")
-		targetNodeCount := initNodeCount + 1
-		err := utils.WaitAutoScaleNodes(cs, targetNodeCount)
-		Expect(err).NotTo(HaveOccurred())
-	})
-
 	It("should scale up or down if deployment replicas leave nodes busy or idle [Feature:Autoscaling]", func() {
 		utils.Logf("Create deployment")
 		replicas := int32(podCount)
@@ -135,6 +117,7 @@ var _ = Describe("Cluster size autoscaler [Serial][Slow]", func() {
 		By("Scale up")
 		targetNodeCount := initNodeCount + 1
 		err = utils.WaitAutoScaleNodes(cs, targetNodeCount)
+		utils.LogPodStatus(cs, ns.Name)
 		Expect(err).NotTo(HaveOccurred())
 
 		By("Scale down")
@@ -145,6 +128,7 @@ var _ = Describe("Cluster size autoscaler [Serial][Slow]", func() {
 		Expect(err).NotTo(HaveOccurred())
 		targetNodeCount = initNodeCount
 		err = utils.WaitAutoScaleNodes(cs, targetNodeCount)
+		utils.LogPodStatus(cs, ns.Name)
 		Expect(err).NotTo(HaveOccurred())
 	})
 })

Diff for: tests/e2e/utils/node_utils.go

@@ -54,10 +54,7 @@ func getNodeList(cs clientset.Interface) (*v1.NodeList, error) {
 	if wait.PollImmediate(poll, singleCallTimeout, func() (bool, error) {
 		nodes, err = cs.CoreV1().Nodes().List(metav1.ListOptions{})
 		if err != nil {
-			if IsRetryableAPIError(err) {
-				return false, nil
-			}
-			return false, err
+			return false, nil
 		}
 		return true, nil
 	}) != nil {
@@ -132,8 +129,8 @@ func WaitAutoScaleNodes(cs clientset.Interface, targetNodeCount int) error {
 	Logf(fmt.Sprintf("waiting for auto-scaling the node... Target node count: %v", targetNodeCount))
 	var nodes []v1.Node
 	var err error
-	poll := 20 * time.Second
-	autoScaleTimeOut := 30 * time.Minute
+	poll := 60 * time.Second
+	autoScaleTimeOut := 50 * time.Minute
 	if wait.PollImmediate(poll, autoScaleTimeOut, func() (bool, error) {
 		nodes, err = GetAgentNodes(cs)
 		if err != nil {

Diff for: tests/e2e/utils/pod_utils.go

@@ -47,6 +47,22 @@ func getPodList(cs clientset.Interface, ns string) (*v1.PodList, error) {
 	return pods, nil
 }
 
+// LogPodStatus logs the rate of pending
+func LogPodStatus(cs clientset.Interface, ns string) error {
+	pods, err := getPodList(cs, ns)
+	if err != nil {
+		return err
+	}
+	pendingPodCount := 0
+	for _, p := range pods.Items {
+		if p.Status.Phase == v1.PodPending {
+			pendingPodCount++
+		}
+	}
+	Logf("%d of %d pods in namespace %s are pending", pendingPodCount, len(pods.Items), ns)
+	return nil
+}
+
 // DeletePodsInNamespace deletes all pods in the namespace
 func DeletePodsInNamespace(cs clientset.Interface, ns string) error {
 	Logf("Deleting all pods in namespace %s", ns)