apps: deployment config stuck in the new state should respect timeoutSecods

Author: mfojtik

pkg/apps/apis/apps/types.go

@@ -113,6 +113,7 @@ const (
 	DeploymentCancelledNewerDeploymentExists  = "newer deployment was found running"
 	DeploymentFailedUnrelatedDeploymentExists = "unrelated pod with the same name as this deployment is already running"
 	DeploymentFailedDeployerPodNoLongerExists = "deployer pod no longer exists"
+	DeploymentFailedUnableToCreateDeployerPod = "unable to create deployer pod"
 )
 
 // DeploymentStatus describes the possible states a deployment can be in.

pkg/apps/controller/deployer/deployer_controller.go

@@ -127,6 +127,21 @@ func (c *DeploymentController) handle(deployment *v1.ReplicationController, will
 			}
 			break
 		}
+		// In case the deployment is stuck in "new" state because we fail to create
+		// deployer pod (quota, etc..) we should respect the timeoutSeconds in the
+		// config strategy and transition the rollout to failed instead of waiting for
+		// the deployment pod forever.
+		config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec)
+		if err != nil {
+			return err
+		}
+		if deployutil.RolloutExceededTimeoutSeconds(config, deployment) {
+			nextStatus = deployapi.DeploymentStatusFailed
+			updatedAnnotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnableToCreateDeployerPod
+			c.emitDeploymentEvent(deployment, v1.EventTypeWarning, "RolloutTimeout", fmt.Sprintf("Rollout for %q failed to create deployer pod (timeoutSeconds: %ds)", deployutil.LabelForDeploymentV1(deployment), deployutil.GetTimeoutSecondsForStrategy(config)))
+			glog.V(4).Infof("Failing deployment %s/%s as we timeout out while waiting for the deployer pod to be created", deployment.Namespace, deployment.Name)
+			break
+		}
 
 		switch {
 		case kerrors.IsNotFound(deployerErr):

pkg/apps/util/util.go

@@ -652,6 +652,12 @@ func IsTerminatedDeployment(deployment runtime.Object) bool {
 	return IsCompleteDeployment(deployment) || IsFailedDeployment(deployment)
 }
 
+// IsNewDeployment returns true if the passed deployment is in new state.
+func IsNewDeployment(deployment runtime.Object) bool {
+	current := DeploymentStatusFor(deployment)
+	return current == deployapi.DeploymentStatusNew
+}
+
 // IsCompleteDeployment returns true if the passed deployment is in state complete.
 func IsCompleteDeployment(deployment runtime.Object) bool {
 	current := DeploymentStatusFor(deployment)
@@ -782,6 +788,42 @@ func DeploymentsForCleanup(configuration *deployapi.DeploymentConfig, deployment
 	return relevantDeployments
 }
 
+// GetTimeoutSecondsForStrategy returns the timeout in seconds defined in the
+// deployment config strategy.
+func GetTimeoutSecondsForStrategy(config *deployapi.DeploymentConfig) int64 {
+	var timeoutSeconds int64
+	switch config.Spec.Strategy.Type {
+	case deployapi.DeploymentStrategyTypeRolling:
+		timeoutSeconds = deployapi.DefaultRollingTimeoutSeconds
+		if t := config.Spec.Strategy.RollingParams.TimeoutSeconds; t != nil {
+			timeoutSeconds = *t
+		}
+	case deployapi.DeploymentStrategyTypeRecreate:
+		timeoutSeconds = deployapi.DefaultRecreateTimeoutSeconds
+		if t := config.Spec.Strategy.RecreateParams.TimeoutSeconds; t != nil {
+			timeoutSeconds = *t
+		}
+	case deployapi.DeploymentStrategyTypeCustom:
+		timeoutSeconds = deployapi.DefaultRecreateTimeoutSeconds
+	}
+	return timeoutSeconds
+}
+
+// RolloutExceededTimeoutSeconds returns true if the current deployment exceeded
+// the timeoutSeconds defined for its strategy.
+// Note that this is different than activeDeadlineSeconds which is the timeout
+// set for the deployer pod. In some cases, the deployer pod cannot be created
+// (like quota, etc...). In that case deployer controller use this function to
+// measure if the created deployment (RC) exceeded the timeout.
+func RolloutExceededTimeoutSeconds(config *deployapi.DeploymentConfig, latestRC *v1.ReplicationController) bool {
+	timeoutSeconds := GetTimeoutSecondsForStrategy(config)
+	// If user set the timeoutSeconds to 0, we assume there should be no timeout.
+	if timeoutSeconds == 0 {
+		return false
+	}
+	return int64(time.Since(latestRC.CreationTimestamp.Time).Seconds()) > timeoutSeconds
+}
+
 // WaitForRunningDeployerPod waits a given period of time until the deployer pod
 // for given replication controller is not running.
 func WaitForRunningDeployerPod(podClient kcoreclient.PodsGetter, rc *api.ReplicationController, timeout time.Duration) error {

pkg/apps/util/util_test.go

@@ -586,3 +586,111 @@ func TestRemoveCondition(t *testing.T) {
 		}
 	}
 }
+
+func TestRolloutExceededTimeoutSeconds(t *testing.T) {
+	now := time.Now()
+	tests := []struct {
+		name                   string
+		config                 func(int64) *deployapi.DeploymentConfig
+		timeoutSeconds         int64
+		deploymentCreationTime time.Time
+		expectTimeout          bool
+	}{
+		// Recreate strategy with deployment running for 20s (exceeding 10s timeout)
+		{
+			name: "recreate timeout",
+			config: func(timeoutSeconds int64) *deployapi.DeploymentConfig {
+				config := deploytest.OkDeploymentConfig(1)
+				config.Spec.Strategy.RecreateParams.TimeoutSeconds = &timeoutSeconds
+				return config
+			},
+			timeoutSeconds:         int64(10),
+			deploymentCreationTime: now.Add(-20 * time.Second),
+			expectTimeout:          true,
+		},
+		// Recreate strategy with no timeout
+		{
+			name: "recreate no timeout",
+			config: func(timeoutSeconds int64) *deployapi.DeploymentConfig {
+				config := deploytest.OkDeploymentConfig(1)
+				config.Spec.Strategy.RecreateParams.TimeoutSeconds = &timeoutSeconds
+				return config
+			},
+			timeoutSeconds:         int64(0),
+			deploymentCreationTime: now.Add(-700 * time.Second),
+			expectTimeout:          false,
+		},
+
+		// Rolling strategy with deployment running for 20s (exceeding 10s timeout)
+		{
+			name: "rolling timeout",
+			config: func(timeoutSeconds int64) *deployapi.DeploymentConfig {
+				config := deploytest.OkDeploymentConfig(1)
+				config.Spec.Strategy = deploytest.OkRollingStrategy()
+				config.Spec.Strategy.RollingParams.TimeoutSeconds = &timeoutSeconds
+				return config
+			},
+			timeoutSeconds:         int64(10),
+			deploymentCreationTime: now.Add(-20 * time.Second),
+			expectTimeout:          true,
+		},
+		// Rolling strategy with deployment with no timeout specified.
+		{
+			name: "rolling using default timeout",
+			config: func(timeoutSeconds int64) *deployapi.DeploymentConfig {
+				config := deploytest.OkDeploymentConfig(1)
+				config.Spec.Strategy = deploytest.OkRollingStrategy()
+				config.Spec.Strategy.RollingParams.TimeoutSeconds = nil
+				return config
+			},
+			deploymentCreationTime: now.Add(-20 * time.Second),
+			expectTimeout:          false,
+		},
+		// Recreate strategy with deployment with no timeout specified.
+		{
+			name: "recreate using default timeout",
+			config: func(timeoutSeconds int64) *deployapi.DeploymentConfig {
+				config := deploytest.OkDeploymentConfig(1)
+				config.Spec.Strategy.RecreateParams.TimeoutSeconds = nil
+				return config
+			},
+			deploymentCreationTime: now.Add(-20 * time.Second),
+			expectTimeout:          false,
+		},
+		// Custom strategy with deployment with no timeout specified.
+		{
+			name: "custom using default timeout",
+			config: func(timeoutSeconds int64) *deployapi.DeploymentConfig {
+				config := deploytest.OkDeploymentConfig(1)
+				config.Spec.Strategy = deploytest.OkCustomStrategy()
+				return config
+			},
+			deploymentCreationTime: now.Add(-20 * time.Second),
+			expectTimeout:          false,
+		},
+		// Custom strategy use default timeout exceeding it.
+		{
+			name: "custom using default timeout timing out",
+			config: func(timeoutSeconds int64) *deployapi.DeploymentConfig {
+				config := deploytest.OkDeploymentConfig(1)
+				config.Spec.Strategy = deploytest.OkCustomStrategy()
+				return config
+			},
+			deploymentCreationTime: now.Add(-700 * time.Second),
+			expectTimeout:          true,
+		},
+	}
+
+	for _, tc := range tests {
+		config := tc.config(tc.timeoutSeconds)
+		deployment, err := MakeDeploymentV1(config, kapi.Codecs.LegacyCodec(deployv1.SchemeGroupVersion))
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		deployment.ObjectMeta.CreationTimestamp = metav1.Time{Time: tc.deploymentCreationTime}
+		gotTimeout := RolloutExceededTimeoutSeconds(config, deployment)
+		if tc.expectTimeout && !gotTimeout {
+			t.Errorf("[%s]: expected timeout, but got no timeout", tc.name)
+		}
+	}
+}