WIP: Implement pausing for machine controller

Author: theobarberbany

internal/controllers/machine/machine_controller.go

@@ -183,12 +183,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
 			m.Spec.ClusterName, m.Name, m.Namespace)
 	}
 
-	// Return early if the object or Cluster is paused.
-	if annotations.IsPaused(cluster, m) {
-		log.Info("Reconciliation is paused for this object")
-		return ctrl.Result{}, nil
-	}
-
 	// Initialize the patch helper
 	patchHelper, err := patch.NewHelper(m, r.Client)
 	if err != nil {
@@ -209,6 +203,16 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Re
 		}
 	}()
 
+	// Return early and set the paused condition to True if the object or Cluster
+	// is paused.
+	if annotations.IsPaused(cluster, m) {
+		log.Info("Reconciliation is paused for this object")
+		conditions.MarkTrue(m, clusterv1.PausedCondition)
+		return ctrl.Result{}, nil
+	}
+
+	conditions.MarkFalse(m, clusterv1.PausedCondition, clusterv1.ResourceNotPausedReason, clusterv1.ConditionSeverityInfo, "Resource is operating as expected")
+
 	// Reconcile labels.
 	if m.Labels == nil {
 		m.Labels = make(map[string]string)
@@ -275,6 +279,7 @@ func patchMachine(ctx context.Context, patchHelper *patch.Helper, machine *clust
 			clusterv1.ReadyCondition,
 			clusterv1.BootstrapReadyCondition,
 			clusterv1.InfrastructureReadyCondition,
+			clusterv1.PausedCondition,
 			clusterv1.DrainingSucceededCondition,
 			clusterv1.MachineHealthCheckSucceededCondition,
 			clusterv1.MachineOwnerRemediatedCondition,

internal/controllers/machine/machine_controller_test.go

@@ -1050,6 +1050,19 @@ func TestMachineConditions(t *testing.T) {
 				conditions.TrueCondition(clusterv1.ReadyCondition),
 			},
 		},
+		{
+			name:           "paused initialises false",
+			infraReady:     true,
+			bootstrapReady: true,
+			beforeFunc: func(_, _ *unstructured.Unstructured, m *clusterv1.Machine) {
+				// since these conditions are set by an external controller
+				conditions.MarkTrue(m, clusterv1.MachineHealthCheckSucceededCondition)
+				conditions.MarkTrue(m, clusterv1.MachineOwnerRemediatedCondition)
+			},
+			conditionsToAssert: []*clusterv1.Condition{
+				conditions.FalseCondition(clusterv1.PausedCondition, "ResourceNotPaused", clusterv1.ConditionSeverityInfo, "Resource is operating as expected"),
+			},
+		},
 		{
 			name:           "infra condition consumes reason from the infra config",
 			infraReady:     false,
@@ -2423,6 +2436,163 @@ func TestNodeDeletion(t *testing.T) {
 	}
 }
 
+func TestPauseConditionReconcile(t *testing.T) {
+	g := NewWithT(t)
+
+	ns, err := env.CreateNamespace(ctx, "test-paused-condition-reconcile")
+	g.Expect(err).ToNot(HaveOccurred())
+
+	infraMachine := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       "GenericInfrastructureMachine",
+			"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
+			"metadata": map[string]interface{}{
+				"name":      "infra-config1",
+				"namespace": ns.Name,
+			},
+			"spec": map[string]interface{}{
+				"providerID": "test://id-1",
+			},
+		},
+	}
+
+	defaultBootstrap := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       "GenericBootstrapConfig",
+			"apiVersion": "bootstrap.cluster.x-k8s.io/v1beta1",
+			"metadata": map[string]interface{}{
+				"name":      "bootstrap-config-pausereconcile",
+				"namespace": ns.Name,
+			},
+			"spec":   map[string]interface{}{},
+			"status": map[string]interface{}{},
+		},
+	}
+
+	testCluster := &clusterv1.Cluster{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pause-condition-reconcile-",
+			Namespace:    ns.Name,
+		},
+	}
+
+	g.Expect(env.Create(ctx, testCluster)).To(Succeed())
+	g.Expect(env.Create(ctx, infraMachine)).To(Succeed())
+	g.Expect(env.Create(ctx, defaultBootstrap)).To(Succeed())
+
+	defer func(do ...client.Object) {
+		g.Expect(env.Cleanup(ctx, do...)).To(Succeed())
+	}(ns, testCluster, defaultBootstrap, machine)
+
+	machine := &clusterv1.Machine{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "machine-created-",
+			Namespace:    ns.Name,
+			Finalizers:   []string{clusterv1.MachineFinalizer},
+		},
+		Spec: clusterv1.MachineSpec{
+			ClusterName: testCluster.Name,
+			InfrastructureRef: corev1.ObjectReference{
+				APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1",
+				Kind:       "GenericInfrastructureMachine",
+				Name:       "infra-config1",
+			},
+			Bootstrap: clusterv1.Bootstrap{
+				ConfigRef: &corev1.ObjectReference{
+					APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1",
+					Kind:       "GenericBootstrapConfig",
+					Name:       "bootstrap-config-machinereconcile",
+				},
+			},
+		},
+		Status: clusterv1.MachineStatus{
+			NodeRef: &corev1.ObjectReference{
+				Name: "test",
+			},
+		},
+	}
+	g.Expect(env.Create(ctx, machine)).To(Succeed())
+
+	key := client.ObjectKey{Name: machine.Name, Namespace: machine.Namespace}
+
+	// Wait for reconciliation to happen when infra and bootstrap objects are not ready.
+	g.Eventually(func() bool {
+		if err := env.Get(ctx, key, machine); err != nil {
+			return false
+		}
+		return len(machine.Finalizers) > 0
+	}, timeout).Should(BeTrue())
+
+	// Set bootstrap ready.
+	bootstrapPatch := client.MergeFrom(defaultBootstrap.DeepCopy())
+	g.Expect(unstructured.SetNestedField(defaultBootstrap.Object, true, "status", "ready")).ToNot(HaveOccurred())
+	g.Expect(env.Status().Patch(ctx, defaultBootstrap, bootstrapPatch)).To(Succeed())
+
+	// Set infrastructure ready.
+	infraMachinePatch := client.MergeFrom(infraMachine.DeepCopy())
+	g.Expect(unstructured.SetNestedField(infraMachine.Object, true, "status", "ready")).To(Succeed())
+	g.Expect(env.Status().Patch(ctx, infraMachine, infraMachinePatch)).To(Succeed())
+
+	// Wait for Machine Ready Condition to become True.
+	g.Eventually(func() bool {
+		if err := env.Get(ctx, key, machine); err != nil {
+			return false
+		}
+		if !conditions.Has(machine, clusterv1.InfrastructureReadyCondition) {
+			return false
+		}
+		readyCondition := conditions.Get(machine, clusterv1.ReadyCondition)
+		return readyCondition.Status == corev1.ConditionTrue
+	}, timeout).Should(BeTrue())
+
+	// The cluster starts unpaused, so the machine machine has a Paused: false
+	// condition
+	g.Eventually(func() bool {
+		if err := env.Get(ctx, key, machine); err != nil {
+			return false
+		}
+		if !conditions.Has(machine, clusterv1.PausedCondition) {
+			return false
+		}
+		pausedCondition := conditions.Get(machine, clusterv1.PausedCondition)
+		return pausedCondition.Status == corev1.ConditionFalse
+	}, timeout).Should(BeTrue())
+
+	// Case: We pause the machine indirectly by pausing the cluster, eventually
+	// the machine has a Paused: true condition
+
+	testCluster.Spec.Paused = true
+	g.Expect(env.Update(ctx, testCluster)).To(Succeed())
+
+	// Eventually, machine has condition paused: true
+	g.Eventually(func() bool {
+		if err := env.Get(ctx, key, machine); err != nil {
+			return false
+		}
+		if !conditions.Has(machine, clusterv1.PausedCondition) {
+			return false
+		}
+		pausedCondition := conditions.Get(machine, clusterv1.PausedCondition)
+		return pausedCondition.Status == corev1.ConditionTrue
+	}, timeout).Should(BeTrue())
+
+	// Case: Then we unpause, eventually machine has paused:false
+	testCluster.Spec.Paused = false
+	g.Expect(env.Update(ctx, testCluster)).To(Succeed())
+
+	// Eventually, machine has condition paused: true
+	g.Eventually(func() bool {
+		if err := env.Get(ctx, key, machine); err != nil {
+			return false
+		}
+		if !conditions.Has(machine, clusterv1.PausedCondition) {
+			return false
+		}
+		pausedCondition := conditions.Get(machine, clusterv1.PausedCondition)
+		return pausedCondition.Status == corev1.ConditionFalse
+	}, timeout).Should(BeTrue())
+}
+
 // adds a condition list to an external object.
 func addConditionsToExternal(u *unstructured.Unstructured, newConditions clusterv1.Conditions) {
 	existingConditions := clusterv1.Conditions{}