update error catching

Author: isabella-janssen

test/extended/machine_config/machine_config_node.go

@@ -70,7 +70,7 @@ var _ = g.Describe("[sig-mco][OCPFeatureGate:MachineConfigNodes]", func() {
 func ValidateMCNProperties(oc *exutil.CLI, fixture string) {
 	// Create client set for test
 	clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
-	o.Expect(clientErr).NotTo(o.HaveOccurred())
+	o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")
 
 	// Grab a random node from each default pool
 	workerNode := GetRandomNode(oc, worker)
@@ -92,13 +92,13 @@ func ValidateMCNProperties(oc *exutil.CLI, fixture string) {
 	defer func() {
 		// Get starting state of default worker MCP
 		workerMcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), worker, metav1.GetOptions{})
-		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(err).NotTo(o.HaveOccurred(), "Could not get worker MCP.")
 		workerMcpReadyMachines := workerMcp.Status.ReadyMachineCount
 
 		// Unlabel node
 		framework.Logf("Removing label node-role.kubernetes.io/%v from node %v", custom, workerNode.Name)
 		unlabelErr := oc.Run("label").Args(fmt.Sprintf("node/%s", workerNode.Name), fmt.Sprintf("node-role.kubernetes.io/%s-", custom)).Execute()
-		o.Expect(unlabelErr).NotTo(o.HaveOccurred())
+		o.Expect(unlabelErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Could not remove label 'node-role.kubernetes.io/%s' from node '%v'.", custom, workerNode.Name))
 
 		// Wait for infra pool to report no nodes & for worker MCP to be ready
 		framework.Logf("Waiting for %v MCP to be updated with %v ready machines.", custom, 0)
@@ -109,22 +109,22 @@ func ValidateMCNProperties(oc *exutil.CLI, fixture string) {
 		// Delete custom MCP
 		framework.Logf("Deleting MCP %v", custom)
 		deleteMCPErr := oc.Run("delete").Args("mcp", custom).Execute()
-		o.Expect(deleteMCPErr).NotTo(o.HaveOccurred())
+		o.Expect(deleteMCPErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Error deleting MCP '%v': %v", custom, deleteMCPErr))
 	}()
 
 	// Apply the fixture to create a custom MCP called "infra" & label the worker node accordingly
 	mcpErr := oc.Run("apply").Args("-f", fixture).Execute()
-	o.Expect(mcpErr).NotTo(o.HaveOccurred())
+	o.Expect(mcpErr).NotTo(o.HaveOccurred(), "Could not create custom MCP.")
 	labelErr := oc.Run("label").Args(fmt.Sprintf("node/%s", workerNode.Name), fmt.Sprintf("node-role.kubernetes.io/%s=", custom)).Execute()
-	o.Expect(labelErr).NotTo(o.HaveOccurred())
+	o.Expect(labelErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Could not add label 'node-role.kubernetes.io/%s' to node '%v'.", custom, workerNode.Name))
 
 	// Wait for the custom pool to be updated with the node ready
 	framework.Logf("Waiting for '%v' MCP to be updated with %v ready machines.", custom, 1)
 	WaitForMCPToBeReady(oc, clientSet, custom, 1)
 
-	// Get node desired and current config versions
+	// Get node in custom pool
 	customNodes, customNodeErr := GetNodesByRole(oc, custom)
-	o.Expect(customNodeErr).NotTo(o.HaveOccurred())
+	o.Expect(customNodeErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Could not get node in MCP '%v'.", custom))
 	customNode := customNodes[0]
 
 	// Validate MCN for node in custom pool
@@ -139,7 +139,7 @@ func ValidateMCNProperties(oc *exutil.CLI, fixture string) {
 func ValidateMCNPropertiesSNO(oc *exutil.CLI, fixture string) {
 	// Create client set for test
 	clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
-	o.Expect(clientErr).NotTo(o.HaveOccurred())
+	o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")
 
 	// Grab the cluster's node
 	node := GetRandomNode(oc, master)
@@ -155,21 +155,21 @@ func ValidateMCNPropertiesSNO(oc *exutil.CLI, fixture string) {
 func ValidateMCNConditionTransitions(oc *exutil.CLI, fixture string) {
 	// Create client set for test
 	clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
-	o.Expect(clientErr).NotTo(o.HaveOccurred())
+	o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")
 
 	// Delete MC on failure or test completion
 	defer func() {
 		deleteMCErr := oc.Run("delete").Args("machineconfig", "90-master-testfile").Execute()
-		o.Expect(deleteMCErr).NotTo(o.HaveOccurred())
+		o.Expect(deleteMCErr).NotTo(o.HaveOccurred(), "Could not delete MachineConfig '90-master-testfile'.")
 	}()
 
 	// Apply MC targeting master pool
 	mcErr := oc.Run("apply").Args("-f", fixture).Execute()
-	o.Expect(mcErr).NotTo(o.HaveOccurred())
+	o.Expect(mcErr).NotTo(o.HaveOccurred(), "Could not apply MachineConfig.")
 
 	// Get an updating master node
 	updatingNodes := GetCordonedNodes(oc, master)
-	o.Expect(len(updatingNodes) > 0, "No ready nodes found for MCP '%v'.", master)
+	o.Expect(len(updatingNodes) > 0, fmt.Sprintf("No ready nodes found for MCP '%v'.", master))
 	masterNode := updatingNodes[0]
 
 	// Validate transition through conditions for MCN
@@ -178,37 +178,37 @@ func ValidateMCNConditionTransitions(oc *exutil.CLI, fixture string) {
 	// a warning will be logged instead of erroring out the test.
 	framework.Logf("Waiting for Updated=False")
 	err := WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdated, metav1.ConditionFalse, 1*time.Minute, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect Updated=False.")
 	framework.Logf("Waiting for UpdatePrepared=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdatePrepared, metav1.ConditionTrue, 1*time.Minute, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect UpdatePrepared=True.")
 	framework.Logf("Waiting for UpdateExecuted=Unknown")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateExecuted, metav1.ConditionUnknown, 30*time.Second, 1*time.Second)
 	if err != nil {
 		framework.Logf("Warning, could not detect UpdateExecuted=Unknown.")
 	}
 	framework.Logf("Waiting for Cordoned=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateCordoned, metav1.ConditionTrue, 30*time.Second, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect Cordoned=True.")
 	framework.Logf("Waiting for Drained=Unknown")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateDrained, metav1.ConditionUnknown, 15*time.Second, 1*time.Second)
 	if err != nil {
 		framework.Logf("Warning, could not detect Drained=Unknown.")
 	}
 	framework.Logf("Waiting for Drained=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateDrained, metav1.ConditionTrue, 4*time.Minute, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect Drained=True.")
 	framework.Logf("Waiting for AppliedFilesAndOS=Unknown")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateFilesAndOS, metav1.ConditionUnknown, 30*time.Second, 1*time.Second)
 	if err != nil {
 		framework.Logf("Warning, could not detect AppliedFilesAndOS=Unknown.")
 	}
 	framework.Logf("Waiting for AppliedFilesAndOS=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateFilesAndOS, metav1.ConditionTrue, 3*time.Minute, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect AppliedFilesAndOS=True.")
 	framework.Logf("Waiting for UpdateExecuted=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateExecuted, metav1.ConditionTrue, 20*time.Second, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect UpdateExecuted=True.")
 	framework.Logf("Waiting for UpdatePostActionComplete=Unknown")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdatePostActionComplete, metav1.ConditionUnknown, 30*time.Second, 1*time.Second)
 	if err != nil {
@@ -221,30 +221,30 @@ func ValidateMCNConditionTransitions(oc *exutil.CLI, fixture string) {
 	}
 	framework.Logf("Waiting for RebootedNode=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateRebooted, metav1.ConditionTrue, 5*time.Minute, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect RebootedNode=True.")
 	framework.Logf("Waiting for Resumed=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeResumed, metav1.ConditionTrue, 15*time.Second, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect Resumed=True.")
 	framework.Logf("Waiting for UpdateComplete=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateComplete, metav1.ConditionTrue, 10*time.Second, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect UpdateComplete=True.")
 	framework.Logf("Waiting for Uncordoned=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdateUncordoned, metav1.ConditionTrue, 10*time.Second, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect Uncordoned=True.")
 	framework.Logf("Waiting for Updated=True")
 	err = WaitForMCNConditionStatus(clientSet, masterNode.Name, mcfgv1alpha1.MachineConfigNodeUpdated, metav1.ConditionTrue, 1*time.Minute, 1*time.Second)
-	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error, could not detect Updated=True.")
 
 	// When an update is complete, all conditions other than `Updated` must be false
 	framework.Logf("Checking all conditions other than 'Updated' are False.")
-	o.Expect(ConfirmUpdatedMCNStatus(clientSet, masterNode.Name)).Should(o.BeTrue())
+	o.Expect(ConfirmUpdatedMCNStatus(clientSet, masterNode.Name)).Should(o.BeTrue(), "Error, all conditions must be 'False' when Updated=True.")
 }
 
 // `ValidateMCNConditionOnNodeDegrade` checks that Conditions properly update on a node failure (MCP degrade)
 func ValidateMCNConditionOnNodeDegrade(oc *exutil.CLI, fixture string, isSno bool) {
 	// Create client set for test
 	clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
-	o.Expect(clientErr).NotTo(o.HaveOccurred())
+	o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")
 
 	// In SNO, master pool will degrade
 	poolName := worker
@@ -258,33 +258,34 @@ func ValidateMCNConditionOnNodeDegrade(oc *exutil.CLI, fixture string, isSno boo
 	defer func() {
 		// Delete the applied MC
 		deleteMCErr := oc.Run("delete").Args("machineconfig", mcName).Execute()
-		o.Expect(deleteMCErr).NotTo(o.HaveOccurred())
+		o.Expect(deleteMCErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Could not delete MachineConfig '%v'.", mcName))
 
 		// Recover the degraded MCP
 		recoverErr := RecoverFromDegraded(oc, poolName)
-		o.Expect(recoverErr).NotTo(o.HaveOccurred())
+		o.Expect(recoverErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Could not recover MCP '%v' from degraded state.", poolName))
 	}()
 
 	// Apply invalid MC
 	mcErr := oc.Run("apply").Args("-f", fixture).Execute()
-	o.Expect(mcErr).NotTo(o.HaveOccurred())
+	o.Expect(mcErr).NotTo(o.HaveOccurred(), "Could not apply MachineConfig.")
 
 	// Wait for MCP to be in a degraded state with one degraded machine
-	o.Expect(WaitForMCPConditionStatus(oc, poolName, "Degraded", corev1.ConditionTrue, 8*time.Minute, 3*time.Second)).NotTo(o.HaveOccurred(), fmt.Sprintf("Error waiting for '%v' MCP to be in a degraded state.", poolName))
+	degradedErr := WaitForMCPConditionStatus(oc, poolName, "Degraded", corev1.ConditionTrue, 8*time.Minute, 3*time.Second)
+	o.Expect(degradedErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Error waiting for '%v' MCP to be in a degraded state.", poolName))
 	mcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), poolName, metav1.GetOptions{})
-	o.Expect(err).NotTo(o.HaveOccurred(), "Error getting '%v' MCP.", poolName)
+	o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Error getting '%v' MCP.", poolName))
 	o.Expect(mcp.Status.DegradedMachineCount).To(o.BeNumerically("==", 1), fmt.Sprintf("Degraded machine count is not 1. It is %v.", mcp.Status.DegradedMachineCount))
 
 	// Get degraded node
 	degradedNode, degradedNodeErr := GetDegradedNode(oc, poolName)
-	o.Expect(degradedNodeErr).NotTo(o.HaveOccurred())
+	o.Expect(degradedNodeErr).NotTo(o.HaveOccurred(), "Could not get degraded node.")
 
 	// Validate MCN of degraded node
 	degradedNodeMCN, degradedErr := clientSet.MachineconfigurationV1alpha1().MachineConfigNodes().Get(context.TODO(), degradedNode.Name, metav1.GetOptions{})
-	o.Expect(degradedErr).NotTo(o.HaveOccurred())
+	o.Expect(degradedErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Error getting MCN of degraded node '%v'.", degradedNode.Name))
 	framework.Logf("Validating that `AppliedFilesAndOS` and `UpdateExecuted` conditions in '%v' MCN have a status of 'Unknown'.", degradedNodeMCN.Name)
-	o.Expect(CheckMCNConditionStatus(degradedNodeMCN, mcfgv1alpha1.MachineConfigNodeUpdateFilesAndOS, metav1.ConditionUnknown))
-	o.Expect(CheckMCNConditionStatus(degradedNodeMCN, mcfgv1alpha1.MachineConfigNodeUpdateExecuted, metav1.ConditionUnknown))
+	o.Expect(CheckMCNConditionStatus(degradedNodeMCN, mcfgv1alpha1.MachineConfigNodeUpdateFilesAndOS, metav1.ConditionUnknown)).Should(o.BeTrue(), "Condition 'AppliedFilesAndOS' does not have the expected status of 'Unknown'.")
+	o.Expect(CheckMCNConditionStatus(degradedNodeMCN, mcfgv1alpha1.MachineConfigNodeUpdateExecuted, metav1.ConditionUnknown)).Should(o.BeTrue(), "Condition 'UpdateExecuted' does not have the expected status of 'Unknown'.")
 }
 
 // `ValidateMCNProperties` checks that MCNs with correct properties are created on node creation
@@ -296,15 +297,15 @@ func ValidateMCNOnNodeCreationAndDeletion(oc *exutil.CLI) {
 
 	// Create machine client for test
 	machineClient, machineErr := machineclient.NewForConfig(oc.KubeFramework().ClientConfig())
-	o.Expect(machineErr).NotTo(o.HaveOccurred())
+	o.Expect(machineErr).NotTo(o.HaveOccurred(), "Error creating machine client for test.")
 
 	// Create client set for test
 	clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
-	o.Expect(clientErr).NotTo(o.HaveOccurred())
+	o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")
 
 	// Skip test if worker nodes cannot be scaled
 	canBeScaled, canScaleErr := WorkersCanBeScaled(oc, machineClient)
-	o.Expect(canScaleErr).NotTo(o.HaveOccurred())
+	o.Expect(canScaleErr).NotTo(o.HaveOccurred(), "Error occured when determining whether worker nodes can be scaled.")
 	if !canBeScaled {
 		g.Skip("Worker nodes cannot be scaled using MachineSets. This test cannot be executed if workers cannot be scaled via MachineSets.")
 	}
@@ -325,7 +326,7 @@ func ValidateMCNOnNodeCreationAndDeletion(oc *exutil.CLI) {
 	// original value, when needed (in the case where the replica value patch was successful).
 	defer func() {
 		cleanupErr := ScaleMachineSetDown(oc, machineSet, originalReplica, cleanupCompleted)
-		o.Expect(cleanupErr).NotTo(o.HaveOccurred())
+		o.Expect(cleanupErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Error cleaning up cluster by scaling down MachineSet '%v'.", machineSet.Name))
 		cleanupCompleted = true
 	}()
 
@@ -339,7 +340,7 @@ func ValidateMCNOnNodeCreationAndDeletion(oc *exutil.CLI) {
 	// original value and ensuring that the newly provisioned Machine is deleted.
 	defer func() {
 		cleanupErr := CleanupProvisionedMachine(oc, machineClient, machineSet.Name, originalReplica, newMachineName, cleanupCompleted)
-		o.Expect(cleanupErr).NotTo(o.HaveOccurred())
+		o.Expect(cleanupErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Error removing provisioned Machine '%v' by scaling down MachineSet '%v'.", newMachineName, machineSet.Name))
 		cleanupCompleted = true
 	}()
 
@@ -362,13 +363,13 @@ func ValidateMCNOnNodeCreationAndDeletion(oc *exutil.CLI) {
 	// original value and ensuring that the newly created Node is deleted.
 	defer func() {
 		cleanupErr := CleanupCreatedNode(oc, newMachineName, originalReplica, newNode.Name, cleanupCompleted)
-		o.Expect(cleanupErr).NotTo(o.HaveOccurred())
+		o.Expect(cleanupErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Error removing created Node '%v' by scaling down MachineSet '%v'.", newNode.Name, machineSet.Name))
 		cleanupCompleted = true
 	}()
 
 	// Validate new MCN
 	validMCNErr := WaitForValidMCNProperties(clientSet, newNode)
-	o.Expect(validMCNErr).NotTo(o.HaveOccurred())
+	o.Expect(validMCNErr).NotTo(o.HaveOccurred(), fmt.Sprintf("MCN for node '%v' has invalid properties.", newNode))
 
 	// Scale down the MachineSet to delete the created node
 	framework.Logf("Scaling down MachineSet to delete node.")