Skip to content

Commit 8f3fb6a

Browse files
RishabhSainiRishabhSaini
committed
pis testing
1 parent 44873fa commit 8f3fb6a

File tree

11 files changed

+536
-0
lines changed

11 files changed

+536
-0
lines changed

‎test/extended/machine_config/helpers.go

+14
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
corev1 "k8s.io/api/core/v1"
2222
apierrors "k8s.io/apimachinery/pkg/api/errors"
2323
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/client-go/kubernetes"
2425
"k8s.io/kubernetes/test/e2e/framework"
2526
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
2627
"k8s.io/utils/ptr"
@@ -317,3 +318,16 @@ func ApplyBootImageFixture(oc *exutil.CLI, fixture string) {
317318
WaitForMachineConfigurationStatusUpdate(oc)
318319

319320
}
321+
322+
// Get nodes from a Pool
323+
func getNodesForPool(ctx context.Context, oc *exutil.CLI, kubeClient *kubernetes.Clientset, pool *mcfgv1.MachineConfigPool) (*corev1.NodeList, error) {
324+
selector, err := metav1.LabelSelectorAsSelector(pool.Spec.NodeSelector)
325+
if err != nil {
326+
return nil, fmt.Errorf("invalid label selector: %w", err)
327+
}
328+
nodes, err := kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
329+
if err != nil {
330+
return nil, fmt.Errorf("couldnt get nodes for mcp: %w", err)
331+
}
332+
return nodes, nil
333+
}

‎test/extended/machine_config/pinnedimages.go

+242
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
package machine_config
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io/ioutil"
7+
"path/filepath"
8+
"strings"
9+
"time"
10+
11+
g "github.com/onsi/ginkgo/v2"
12+
o "github.com/onsi/gomega"
13+
14+
mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
15+
mcClient "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
16+
exutil "github.com/openshift/origin/test/extended/util"
17+
corev1 "k8s.io/api/core/v1"
18+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
19+
20+
"k8s.io/apimachinery/pkg/labels"
21+
"k8s.io/apimachinery/pkg/util/wait"
22+
"k8s.io/client-go/kubernetes"
23+
"sigs.k8s.io/yaml"
24+
)
25+
26+
// This test is [Serial] because it modifies the state of the images present on Node in each test.
27+
var _ = g.Describe("[sig-mco][OCPFeatureGate:PinnedImages][OCPFeatureGate:MachineConfigNodes][Serial]", func() {
28+
defer g.GinkgoRecover()
29+
var (
30+
MCOPinnedImageBaseDir = exutil.FixturePath("testdata", "machine_config", "pinnedimage")
31+
pinnedImageSetFixture = filepath.Join(MCOPinnedImageBaseDir, "pis.yaml")
32+
customMCP = filepath.Join(MCOPinnedImageBaseDir, "customMCP.yaml")
33+
customMCPpinnedImageSetFixture = filepath.Join(MCOPinnedImageBaseDir, "customMCPpis.yaml")
34+
customGCMCPpinnedImageSetFixture = filepath.Join(MCOPinnedImageBaseDir, "customGCMCPpis.yaml")
35+
customGcKC = filepath.Join(MCOPinnedImageBaseDir, "gcKC.yaml")
36+
invalidPinnedImageSetFixture = filepath.Join(MCOPinnedImageBaseDir, "invalidPis.yaml")
37+
oc = exutil.NewCLIWithoutNamespace("machine-config")
38+
)
39+
// Ensure each test pins a separate image, since we are not deleting them after each
40+
41+
g.It("All Nodes in a custom Pool should have the PinnedImages even after Garbage Collection [apigroup:machineconfiguration.openshift.io]", func() {
42+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
43+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
44+
45+
// Create custom MCP
46+
err = oc.Run("apply").Args("-f", customMCP).Execute()
47+
o.Expect(err).NotTo(o.HaveOccurred())
48+
defer deleteMCP(oc, "custom")
49+
50+
// Add node to pool
51+
nodes, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{"node-role.kubernetes.io/worker": ""}).String()})
52+
o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
53+
o.Expect(nodes.Items).NotTo(o.BeEmpty())
54+
err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom=").Execute()
55+
o.Expect(err).NotTo(o.HaveOccurred(), "Label node")
56+
defer unlabelNode(oc, nodes.Items[0].Name)
57+
58+
// Apply KC to Pool
59+
err = oc.Run("apply").Args("-f", customGcKC).Execute()
60+
o.Expect(err).NotTo(o.HaveOccurred())
61+
defer deleteKC(oc, "custom-gc-config")
62+
63+
SimplePISTest(oc, kubeClient, customGCMCPpinnedImageSetFixture, true)
64+
})
65+
66+
g.It("All Nodes in a Custom Pool should have the PinnedImages in PIS [apigroup:machineconfiguration.openshift.io]", func() {
67+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
68+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
69+
70+
// Create custom MCP
71+
err = oc.Run("apply").Args("-f", customMCP).Execute()
72+
o.Expect(err).NotTo(o.HaveOccurred())
73+
defer deleteMCP(oc, "custom")
74+
75+
// Add node to pool
76+
nodes, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{"node-role.kubernetes.io/worker": ""}).String()})
77+
o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
78+
o.Expect(nodes.Items).NotTo(o.BeEmpty())
79+
err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom=").Execute()
80+
o.Expect(err).NotTo(o.HaveOccurred(), "Label node")
81+
defer unlabelNode(oc, nodes.Items[0].Name)
82+
83+
SimplePISTest(oc, kubeClient, customMCPpinnedImageSetFixture, true)
84+
})
85+
86+
g.It("All Nodes in a standard Pool should have the PinnedImages PIS [apigroup:machineconfiguration.openshift.io]", func() {
87+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
88+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
89+
90+
SimplePISTest(oc, kubeClient, pinnedImageSetFixture, true)
91+
})
92+
93+
g.It("Invalid PIS leads to degraded MCN in a standard Pool [apigroup:machineconfiguration.openshift.io]", func() {
94+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
95+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
96+
97+
SimplePISTest(oc, kubeClient, invalidPinnedImageSetFixture, false)
98+
})
99+
100+
g.It("Invalid PIS leads to degraded MCN in a custom Pool [apigroup:machineconfiguration.openshift.io]", func() {
101+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
102+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
103+
104+
// Create custom MCP
105+
err = oc.Run("apply").Args("-f", customMCP).Execute()
106+
o.Expect(err).NotTo(o.HaveOccurred())
107+
defer deleteMCP(oc, "custom")
108+
109+
// Add node to pool
110+
nodes, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{"node-role.kubernetes.io/worker": ""}).String()})
111+
o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
112+
o.Expect(nodes.Items).NotTo(o.BeEmpty())
113+
err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom=").Execute()
114+
o.Expect(err).NotTo(o.HaveOccurred(), "Label node")
115+
defer unlabelNode(oc, nodes.Items[0].Name)
116+
117+
SimplePISTest(oc, kubeClient, invalidPinnedImageSetFixture, false)
118+
})
119+
120+
})
121+
122+
func SimplePISTest(oc *exutil.CLI, kubeClient *kubernetes.Clientset, fixture string, success bool) {
123+
clientSet, err := mcClient.NewForConfig(oc.KubeFramework().ClientConfig())
124+
o.Expect(err).NotTo(o.HaveOccurred())
125+
126+
pis, err := getPISFromFixture(fixture)
127+
o.Expect(err).NotTo(o.HaveOccurred())
128+
129+
err = oc.Run("apply").Args("-f", fixture).Execute()
130+
o.Expect(err).NotTo(o.HaveOccurred(), "Applied PIS")
131+
defer deletePIS(oc, pis.Name)
132+
133+
waitTime := time.Minute * 20
134+
ctx, cancel := context.WithTimeout(context.Background(), waitTime)
135+
defer cancel()
136+
137+
err = waitForPISStatusX(ctx, oc, kubeClient, clientSet, pis.Name, success)
138+
o.Expect(err).NotTo(o.HaveOccurred(), "Checking status of PIS")
139+
}
140+
141+
func detectXCondition(oc *exutil.CLI, node corev1.Node, mcn *mcfgv1alpha1.MachineConfigNode, appliedPIS *mcfgv1alpha1.PinnedImageSet, detectingSuccess bool) (bool, bool, error) {
142+
if detectingSuccess {
143+
for _, cond := range mcn.Status.Conditions {
144+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsDegraded && cond.Status == "True" {
145+
return false, true, fmt.Errorf("PIS degraded for MCN %s with reason: %s and message: %s", mcn.Name, cond.Reason, cond.Message)
146+
}
147+
148+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsProgressing && cond.Status == "True" {
149+
return false, false, nil
150+
}
151+
}
152+
for _, img := range appliedPIS.Spec.PinnedImages {
153+
crictlStatus, err := exutil.DebugNodeRetryWithOptionsAndChroot(oc, node.Name, "openshift-machine-config-operator", "crictl", "inspecti", img.Name)
154+
if err != nil {
155+
return false, false, fmt.Errorf("failed to execute `crictl inspecti %s` on node %s: %w", img.Name, node.Name, err)
156+
}
157+
if !strings.Contains(crictlStatus, "imageSpec") {
158+
return false, true, fmt.Errorf("Image %s not present on node %s: %w", img.Name, node.Name, err)
159+
}
160+
}
161+
return true, false, nil
162+
} else {
163+
for _, cond := range mcn.Status.Conditions {
164+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsDegraded && cond.Status == "True" {
165+
continue
166+
}
167+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsProgressing && cond.Status == "True" {
168+
return false, false, nil
169+
}
170+
}
171+
return true, false, nil
172+
}
173+
}
174+
175+
func waitForPISStatusX(ctx context.Context, oc *exutil.CLI, kubeClient *kubernetes.Clientset, clientSet *mcClient.Clientset, pisName string, success bool) error {
176+
return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (done bool, err error) {
177+
// Wait for PIS object to get created
178+
appliedPIS, err := clientSet.MachineconfigurationV1alpha1().PinnedImageSets().Get(context.TODO(), pisName, metav1.GetOptions{})
179+
if err != nil {
180+
return false, fmt.Errorf("PIS Object not created yet: %w", err)
181+
}
182+
183+
pool, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(ctx, appliedPIS.Labels["machineconfiguration.openshift.io/role"], metav1.GetOptions{})
184+
if err != nil {
185+
return true, fmt.Errorf("failed to get MCP mentioned in PIS: %w", err)
186+
}
187+
188+
nodes, err := getNodesForPool(ctx, oc, kubeClient, pool)
189+
if err != nil {
190+
return true, fmt.Errorf("failed to get Nodes from MCP %q mentioned in PIS: %w", pool.Name, err)
191+
}
192+
193+
doneNodes := 0
194+
for _, node := range nodes.Items {
195+
mcn, err := clientSet.MachineconfigurationV1alpha1().MachineConfigNodes().Get(ctx, node.Name, metav1.GetOptions{})
196+
if err != nil {
197+
return true, fmt.Errorf("failed to get mcn: %w", err)
198+
}
199+
toContinue, isDone, err := detectXCondition(oc, node, mcn, appliedPIS, success)
200+
if !toContinue {
201+
return isDone, err
202+
}
203+
doneNodes += 1
204+
}
205+
if doneNodes == len(nodes.Items) {
206+
return true, nil
207+
}
208+
209+
return false, nil
210+
})
211+
}
212+
213+
func unlabelNode(oc *exutil.CLI, name string) error {
214+
return oc.AsAdmin().Run("label").Args("node", name, "node-role.kubernetes.io/custom-").Execute()
215+
}
216+
217+
func deleteKC(oc *exutil.CLI, name string) error {
218+
return oc.Run("delete").Args("kubeletconfig", name).Execute()
219+
}
220+
221+
func deleteMCP(oc *exutil.CLI, name string) error {
222+
return oc.Run("delete").Args("mcp", name).Execute()
223+
}
224+
225+
func deletePIS(oc *exutil.CLI, name string) error {
226+
return oc.Run("delete").Args("pinnedimageset", name).Execute()
227+
}
228+
229+
func getPISFromFixture(path string) (*mcfgv1alpha1.PinnedImageSet, error) {
230+
data, err := ioutil.ReadFile(path)
231+
if err != nil {
232+
return nil, err
233+
}
234+
235+
ob := new(mcfgv1alpha1.PinnedImageSet)
236+
err = yaml.Unmarshal(data, ob)
237+
if err != nil {
238+
return nil, err
239+
}
240+
241+
return ob, nil
242+
}

0 commit comments

Comments
 (0)