MCO-1523: PinnedImageSet v1alpha1 Testing

test/extended/machine_config/helpers.go

@@ -21,6 +21,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	"k8s.io/utils/ptr"
@@ -317,3 +318,16 @@ func ApplyBootImageFixture(oc *exutil.CLI, fixture string) {
 	WaitForMachineConfigurationStatusUpdate(oc)
 
 }
+
+// Get nodes from a Pool
+func getNodesForPool(ctx context.Context, oc *exutil.CLI, kubeClient *kubernetes.Clientset, pool *mcfgv1.MachineConfigPool) (*corev1.NodeList, error) {
+	selector, err := metav1.LabelSelectorAsSelector(pool.Spec.NodeSelector)
+	if err != nil {
+		return nil, fmt.Errorf("invalid label selector: %w", err)
+	}
+	nodes, err := kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
+	if err != nil {
+		return nil, fmt.Errorf("couldnt get nodes for mcp: %w", err)
+	}
+	return nodes, nil
+}

test/extended/machine_config/pinnedimages.go

@@ -0,0 +1,254 @@
+package machine_config
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"time"
+
+	g "github.com/onsi/ginkgo/v2"
+	o "github.com/onsi/gomega"
+
+	mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
+	mcClient "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
+	exutil "github.com/openshift/origin/test/extended/util"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"sigs.k8s.io/yaml"
+)
+
+// This test is [Serial] because it modifies the state of the images present on Node in each test.
+var _ = g.Describe("[sig-mco][OCPFeatureGate:PinnedImages][OCPFeatureGate:MachineConfigNodes][Serial]", func() {
+	defer g.GinkgoRecover()
+	var (
+		MCOPinnedImageBaseDir       = exutil.FixturePath("testdata", "machine_config", "pinnedimage")
+		MCOMachineConfigPoolBaseDir = exutil.FixturePath("testdata", "machine_config", "machineconfigpool")
+		MCOKubeletConfigBaseDir     = exutil.FixturePath("testdata", "machine_config", "kubeletconfig")
+
+		pinnedImageSetFixture            = filepath.Join(MCOPinnedImageBaseDir, "pis.yaml")
+		customMCPFixture                 = filepath.Join(MCOMachineConfigPoolBaseDir, "customMCP.yaml")
+		customMCPpinnedImageSetFixture   = filepath.Join(MCOPinnedImageBaseDir, "customMCPpis.yaml")
+		customGCMCPpinnedImageSetFixture = filepath.Join(MCOPinnedImageBaseDir, "customGCMCPpis.yaml")
+		customGcKCFixture                = filepath.Join(MCOKubeletConfigBaseDir, "gcKC.yaml")
+		invalidPinnedImageSetFixture     = filepath.Join(MCOPinnedImageBaseDir, "invalidPis.yaml")
+
+		oc = exutil.NewCLIWithoutNamespace("machine-config")
+	)
+	// Ensure each test pins a separate image, since we are not deleting them after each
+
+	g.It("All Nodes in a custom Pool should have the PinnedImages even after Garbage Collection [apigroup:machineconfiguration.openshift.io]", func() {
+		kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
+		o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
+
+		// Create custom MCP
+		err = oc.Run("apply").Args("-f", customMCPFixture).Execute()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		defer deleteMCP(oc, "custom")
+
+		// Add node to pool
+		nodes, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{"node-role.kubernetes.io/worker": ""}).String()})
+		o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
+		o.Expect(nodes.Items).NotTo(o.BeEmpty())
+		err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom=").Execute()
+		o.Expect(err).NotTo(o.HaveOccurred(), "Label node")
+		defer unlabelNode(oc, nodes.Items[0].Name)
+
+		// Apply KC to Pool
+		err = oc.Run("apply").Args("-f", customGcKCFixture).Execute()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		defer deleteKC(oc, "custom-gc-config")
+
+		SimplePISTest(oc, kubeClient, customGCMCPpinnedImageSetFixture, true)
+	})
+
+	g.It("All Nodes in a Custom Pool should have the PinnedImages in PIS [apigroup:machineconfiguration.openshift.io]", func() {
+		kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
+		o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
+
+		// Create custom MCP
+		err = oc.Run("apply").Args("-f", customMCPFixture).Execute()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		defer deleteMCP(oc, "custom")
+
+		// Add node to pool
+		nodes, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{"node-role.kubernetes.io/worker": ""}).String()})
+		o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
+		o.Expect(nodes.Items).NotTo(o.BeEmpty())
+		err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom=").Execute()
+		o.Expect(err).NotTo(o.HaveOccurred(), "Label node")
+		defer unlabelNode(oc, nodes.Items[0].Name)
+
+		SimplePISTest(oc, kubeClient, customMCPpinnedImageSetFixture, true)
+	})
+
+	g.It("All Nodes in a standard Pool should have the PinnedImages PIS [apigroup:machineconfiguration.openshift.io]", func() {
+		kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
+		o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
+
+		SimplePISTest(oc, kubeClient, pinnedImageSetFixture, true)
+	})
+
+	g.It("Invalid PIS leads to degraded MCN in a standard Pool [apigroup:machineconfiguration.openshift.io]", func() {
+		kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
+		o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
+
+		SimplePISTest(oc, kubeClient, invalidPinnedImageSetFixture, false)
+	})
+
+	g.It("Invalid PIS leads to degraded MCN in a custom Pool [apigroup:machineconfiguration.openshift.io]", func() {
+		kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
+		o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
+
+		// Create custom MCP
+		err = oc.Run("apply").Args("-f", customMCPFixture).Execute()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		defer deleteMCP(oc, "custom")
+
+		// Add node to pool
+		nodes, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{"node-role.kubernetes.io/worker": ""}).String()})
+		o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
+		o.Expect(nodes.Items).NotTo(o.BeEmpty())
+		err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom=").Execute()
+		o.Expect(err).NotTo(o.HaveOccurred(), "Label node")
+		defer unlabelNode(oc, nodes.Items[0].Name)
+
+		SimplePISTest(oc, kubeClient, invalidPinnedImageSetFixture, false)
+	})
+
+})
+
+func SimplePISTest(oc *exutil.CLI, kubeClient *kubernetes.Clientset, fixture string, success bool) {
+	clientSet, err := mcClient.NewForConfig(oc.KubeFramework().ClientConfig())
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	pis, err := getPISFromFixture(fixture)
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	err = oc.Run("apply").Args("-f", fixture).Execute()
+	o.Expect(err).NotTo(o.HaveOccurred(), "Applied PIS")
+	defer deletePIS(oc, pis.Name)
+
+	waitTime := time.Minute * 20
+	ctx, cancel := context.WithTimeout(context.Background(), waitTime)
+	defer cancel()
+
+	err = waitForPISStatusX(ctx, oc, kubeClient, clientSet, pis.Name, success)
+	o.Expect(err).NotTo(o.HaveOccurred(), "Checking status of PIS")
+}
+
+func detectXCondition(oc *exutil.CLI, node corev1.Node, mcn *mcfgv1alpha1.MachineConfigNode, appliedPIS *mcfgv1alpha1.PinnedImageSet, detectingSuccess bool) (bool, bool, error) {
+	if detectingSuccess {
+		for _, cond := range mcn.Status.Conditions {
+			if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsDegraded && cond.Status == "True" {
+				return false, true, fmt.Errorf("PIS degraded for MCN %s with reason: %s and message: %s", mcn.Name, cond.Reason, cond.Message)
+			}
+
+			if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsProgressing && cond.Status == "True" {
+				return false, false, nil
+			}
+		}
+		for _, img := range appliedPIS.Spec.PinnedImages {
+			crictlStatus, err := exutil.DebugNodeRetryWithOptionsAndChroot(oc, node.Name, "openshift-machine-config-operator", "crictl", "inspecti", img.Name)
+			if err != nil {
+				return false, false, fmt.Errorf("failed to execute `crictl inspecti %s` on node %s: %w", img.Name, node.Name, err)
+			}
+			if !strings.Contains(crictlStatus, "imageSpec") {
+				return false, false, fmt.Errorf("Image %s not present on node %s: %w", img.Name, node.Name, err)
+			}
+		}
+		return true, false, nil
+	} else {
+		for _, cond := range mcn.Status.Conditions {
+			if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsDegraded && cond.Status == "True" {
+				continue
+			}
+			if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsProgressing && cond.Status == "True" {
+				return false, false, nil
+			}
+		}
+		return true, false, nil
+	}
+}
+
+func waitForPISStatusX(ctx context.Context, oc *exutil.CLI, kubeClient *kubernetes.Clientset, clientSet *mcClient.Clientset, pisName string, success bool) error {
+	return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (done bool, err error) {
+		// Wait for PIS object to get created
+		appliedPIS, err := clientSet.MachineconfigurationV1alpha1().PinnedImageSets().Get(context.TODO(), pisName, metav1.GetOptions{})
+		if err != nil {
+			return false, fmt.Errorf("PIS Object not created yet: %w", err)
+		}
+
+		pool, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(ctx, appliedPIS.Labels["machineconfiguration.openshift.io/role"], metav1.GetOptions{})
+		if err != nil {
+			return false, fmt.Errorf("failed to get MCP mentioned in PIS: %w", err)
+		}
+
+		nodes, err := getNodesForPool(ctx, oc, kubeClient, pool)
+		if err != nil {
+			return false, fmt.Errorf("failed to get Nodes from MCP %q  mentioned in PIS: %w", pool.Name, err)
+		}
+
+		doneNodes := 0
+		for _, node := range nodes.Items {
+			mcn, err := clientSet.MachineconfigurationV1alpha1().MachineConfigNodes().Get(ctx, node.Name, metav1.GetOptions{})
+			if err != nil {
+				return false, fmt.Errorf("failed to get mcn: %w", err)
+			}
+			toContinue, isFatal, err := detectXCondition(oc, node, mcn, appliedPIS, success)
+			if !toContinue {
+				if isFatal {
+					return true, err
+				} else {
+					if err != nil {
+						framework.Logf("Retrying PIS Status with non-fatal error: %v", err)
+					}
+					return false, nil
+				}
+			}
+			doneNodes += 1
+		}
+		if doneNodes == len(nodes.Items) {
+			return true, nil
+		}
+
+		return false, nil
+	})
+}
+
+func unlabelNode(oc *exutil.CLI, name string) error {
+	return oc.AsAdmin().Run("label").Args("node", name, "node-role.kubernetes.io/custom-").Execute()
+}
+
+func deleteKC(oc *exutil.CLI, name string) error {
+	return oc.Run("delete").Args("kubeletconfig", name).Execute()
+}
+
+func deleteMCP(oc *exutil.CLI, name string) error {
+	return oc.Run("delete").Args("mcp", name).Execute()
+}
+
+func deletePIS(oc *exutil.CLI, name string) error {
+	return oc.Run("delete").Args("pinnedimageset", name).Execute()
+}
+
+func getPISFromFixture(path string) (*mcfgv1alpha1.PinnedImageSet, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	ob := new(mcfgv1alpha1.PinnedImageSet)
+	err = yaml.Unmarshal(data, ob)
+	if err != nil {
+		return nil, err
+	}
+
+	return ob, nil
+}