add more lofic for rollout wait

Author: RishabhSaini

test/extended/machine_config/ocl.go

@@ -13,8 +13,9 @@ import (
 	e2e "k8s.io/kubernetes/test/e2e/framework"
 
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
-	//mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
+	mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
 	commonstate "github.com/openshift/machine-config-operator/pkg/controller/common/state"
+	ctrlnode "github.com/openshift/machine-config-operator/pkg/controller/node"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -105,7 +106,10 @@ func AllNodePoolOptInTest(oc *exutil.CLI, moscFixture string, mcpFixture string)
 
 	mcp, err := getMCPFromFixture(mcpFixture)
 	o.Expect(err).NotTo(o.HaveOccurred())
-
+	
+	mosc, err := getMOSCFromFixture(moscFixture)
+	o.Expect(err).NotTo(o.HaveOccurred())
+	
 	waitTime := time.Minute * 20
 	ctx, cancel := context.WithTimeout(context.Background(), waitTime)
 	defer cancel()
@@ -114,7 +118,9 @@ func AllNodePoolOptInTest(oc *exutil.CLI, moscFixture string, mcpFixture string)
 	err = waitForBuild(ctx, machineConfigClient, mcp)
 	o.Expect(err).NotTo(o.HaveOccurred(), "Waiting for MOSB to be created and builder pod to Succeed")
 
-	// Wait for the build Image to be applied to all nodes
+	// Wait for the build Image to be applied to mcp successfully
+	err = waitForRollout(ctx, machineConfigClient, oc, mcp, mosc)
+	o.Expect(err).NotTo(o.HaveOccurred(), "Waiting for MCP to have desired config as the new built image")
 }
 
 func getMCPFromFixture(path string) (*mcfgv1.MachineConfigPool, error) {
@@ -132,6 +138,84 @@ func getMCPFromFixture(path string) (*mcfgv1.MachineConfigPool, error) {
 	return mcp, err
 }
 
+func getMOSCFromFixture(path string) (*mcfgv1alpha1.MachineOSConfig, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	mosc := new(mcfgv1alpha1.MachineOSConfig)
+	err = yaml.Unmarshal(data, mosc)
+	if err != nil {
+		return nil, err
+	}
+
+	return mosc, err
+}
+
+func getNodesForPool(ctx context.Context, oc *exutil.CLI, pool *mcfgv1.MachineConfigPool) (*corev1.NodeList, error) {
+	selector, err := metav1.LabelSelectorAsSelector(pool.Spec.NodeSelector)
+	if err != nil {
+		return nil, fmt.Errorf("invalid label selector: %w", err)
+	}
+	nodes, err := oc.KubeClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String(),})
+	if err != nil {
+		return nil, fmt.Errorf("couldnt get nodes for mcp: %w", err)
+	}
+	return nodes, nil
+}
+
+func waitForRollout(ctx context.Context, clientset *mcClient.Clientset, oc *exutil.CLI, mcpG *mcfgv1.MachineConfigPool, moscG *mcfgv1alpha1.MachineOSConfig) error {
+	return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (done bool, err error) {
+		
+		//Get mosb
+		var mosb mcfgv1alpha1.MachineOSBuild 
+		var mosbFound bool
+		mosbList, err := clientset.MachineconfigurationV1alpha1().MachineOSBuilds().List(ctx, metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, m := range mosbList.Items {
+			if m.Spec.DesiredConfig.Name == mcpG.Spec.Configuration.Name {
+				mosb = m
+				mosbFound = true
+				break
+			}
+		}
+		if !mosbFound {
+			return false, fmt.Errorf("could not find mosb")
+		}
+		
+		// Get mosc
+		mosc, err := clientset.MachineconfigurationV1alpha1().MachineOSConfigs().Get(ctx, moscG.ObjectMeta.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, fmt.Errorf("couldnt get mosc: %w", err)
+		}
+
+		// Get Pool
+		pool, err := clientset.MachineconfigurationV1().MachineConfigPools().Get(ctx, mcpG.ObjectMeta.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, fmt.Errorf("couldnt get pool: %w", err)
+		}
+		
+		// Get nodes
+		nodes, err := getNodesForPool(ctx, oc, pool)
+		
+		// Ensure rollout
+		doneNodes := 0
+		for _, node := range nodes.Items {
+			lns := commonstate.NewLayeredNodeState(&node)
+			if mosb.Spec.DesiredConfig.Name == pool.Spec.Configuration.Name && ctrlnode.IsNodeDoneAt(&node, pool, true) && lns.IsCurrentImageEqualToBuild(mosc){
+				doneNodes += 1
+			}	
+		}
+		if doneNodes == len(nodes.Items){
+			return true, nil
+		}
+		return false, nil
+	})
+}
+
 func waitForBuild(ctx context.Context, clientset *mcClient.Clientset, mcp *mcfgv1.MachineConfigPool) error {
 	return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (done bool, err error) {
 
@@ -164,6 +248,7 @@ func waitForBuild(ctx context.Context, clientset *mcClient.Clientset, mcp *mcfgv
 				if state.IsBuildFailure() {
 					return false, fmt.Errorf("build %s failed after %s", mosb.Name, time.Since(start))
 				}
+				break
 			}
 		}
 		return false, nil