Skip to content

Commit 57e0966

Browse files
RishabhSainiRishabhSaini
committed
pis testing
1 parent 44873fa commit 57e0966

File tree

11 files changed

+552
-0
lines changed

11 files changed

+552
-0
lines changed

‎test/extended/machine_config/helpers.go

+14
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
corev1 "k8s.io/api/core/v1"
2222
apierrors "k8s.io/apimachinery/pkg/api/errors"
2323
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/client-go/kubernetes"
2425
"k8s.io/kubernetes/test/e2e/framework"
2526
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
2627
"k8s.io/utils/ptr"
@@ -317,3 +318,16 @@ func ApplyBootImageFixture(oc *exutil.CLI, fixture string) {
317318
WaitForMachineConfigurationStatusUpdate(oc)
318319

319320
}
321+
322+
// Get nodes from a Pool
323+
func getNodesForPool(ctx context.Context, oc *exutil.CLI, kubeClient *kubernetes.Clientset, pool *mcfgv1.MachineConfigPool) (*corev1.NodeList, error) {
324+
selector, err := metav1.LabelSelectorAsSelector(pool.Spec.NodeSelector)
325+
if err != nil {
326+
return nil, fmt.Errorf("invalid label selector: %w", err)
327+
}
328+
nodes, err := kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
329+
if err != nil {
330+
return nil, fmt.Errorf("couldnt get nodes for mcp: %w", err)
331+
}
332+
return nodes, nil
333+
}

‎test/extended/machine_config/pinnedimages.go

+254
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,254 @@
1+
package machine_config
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io/ioutil"
7+
"path/filepath"
8+
"strings"
9+
"time"
10+
11+
g "github.com/onsi/ginkgo/v2"
12+
o "github.com/onsi/gomega"
13+
14+
mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
15+
mcClient "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
16+
exutil "github.com/openshift/origin/test/extended/util"
17+
corev1 "k8s.io/api/core/v1"
18+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
19+
20+
"k8s.io/apimachinery/pkg/labels"
21+
"k8s.io/apimachinery/pkg/util/wait"
22+
"k8s.io/client-go/kubernetes"
23+
"sigs.k8s.io/yaml"
24+
"k8s.io/kubernetes/test/e2e/framework"
25+
)
26+
27+
// This test is [Serial] because it modifies the state of the images present on Node in each test.
28+
var _ = g.Describe("[sig-mco][OCPFeatureGate:PinnedImages][OCPFeatureGate:MachineConfigNodes][Serial]", func() {
29+
defer g.GinkgoRecover()
30+
var (
31+
MCOPinnedImageBaseDir = exutil.FixturePath("testdata", "machine_config", "pinnedimage")
32+
MCOMachineConfigPoolBaseDir = exutil.FixturePath("testdata", "machine_config", "machineconfigpool")
33+
MCOKubeletConfigBaseDir = exutil.FixturePath("testdata", "machine_config", "kubeletconfig")
34+
35+
pinnedImageSetFixture = filepath.Join(MCOPinnedImageBaseDir, "pis.yaml")
36+
customMCPFixture = filepath.Join(MCOMachineConfigPoolBaseDir, "customMCP.yaml")
37+
customMCPpinnedImageSetFixture = filepath.Join(MCOPinnedImageBaseDir, "customMCPpis.yaml")
38+
customGCMCPpinnedImageSetFixture = filepath.Join(MCOPinnedImageBaseDir, "customGCMCPpis.yaml")
39+
customGcKCFixture = filepath.Join(MCOKubeletConfigBaseDir, "gcKC.yaml")
40+
invalidPinnedImageSetFixture = filepath.Join(MCOPinnedImageBaseDir, "invalidPis.yaml")
41+
42+
oc = exutil.NewCLIWithoutNamespace("machine-config")
43+
)
44+
// Ensure each test pins a separate image, since we are not deleting them after each
45+
46+
g.It("All Nodes in a custom Pool should have the PinnedImages even after Garbage Collection [apigroup:machineconfiguration.openshift.io]", func() {
47+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
48+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
49+
50+
// Create custom MCP
51+
err = oc.Run("apply").Args("-f", customMCPFixture).Execute()
52+
o.Expect(err).NotTo(o.HaveOccurred())
53+
defer deleteMCP(oc, "custom")
54+
55+
// Add node to pool
56+
nodes, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{"node-role.kubernetes.io/worker": ""}).String()})
57+
o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
58+
o.Expect(nodes.Items).NotTo(o.BeEmpty())
59+
err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom=").Execute()
60+
o.Expect(err).NotTo(o.HaveOccurred(), "Label node")
61+
defer unlabelNode(oc, nodes.Items[0].Name)
62+
63+
// Apply KC to Pool
64+
err = oc.Run("apply").Args("-f", customGcKCFixture).Execute()
65+
o.Expect(err).NotTo(o.HaveOccurred())
66+
defer deleteKC(oc, "custom-gc-config")
67+
68+
SimplePISTest(oc, kubeClient, customGCMCPpinnedImageSetFixture, true)
69+
})
70+
71+
g.It("All Nodes in a Custom Pool should have the PinnedImages in PIS [apigroup:machineconfiguration.openshift.io]", func() {
72+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
73+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
74+
75+
// Create custom MCP
76+
err = oc.Run("apply").Args("-f", customMCPFixture).Execute()
77+
o.Expect(err).NotTo(o.HaveOccurred())
78+
defer deleteMCP(oc, "custom")
79+
80+
// Add node to pool
81+
nodes, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{"node-role.kubernetes.io/worker": ""}).String()})
82+
o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
83+
o.Expect(nodes.Items).NotTo(o.BeEmpty())
84+
err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom=").Execute()
85+
o.Expect(err).NotTo(o.HaveOccurred(), "Label node")
86+
defer unlabelNode(oc, nodes.Items[0].Name)
87+
88+
SimplePISTest(oc, kubeClient, customMCPpinnedImageSetFixture, true)
89+
})
90+
91+
g.It("All Nodes in a standard Pool should have the PinnedImages PIS [apigroup:machineconfiguration.openshift.io]", func() {
92+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
93+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
94+
95+
SimplePISTest(oc, kubeClient, pinnedImageSetFixture, true)
96+
})
97+
98+
g.It("Invalid PIS leads to degraded MCN in a standard Pool [apigroup:machineconfiguration.openshift.io]", func() {
99+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
100+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
101+
102+
SimplePISTest(oc, kubeClient, invalidPinnedImageSetFixture, false)
103+
})
104+
105+
g.It("Invalid PIS leads to degraded MCN in a custom Pool [apigroup:machineconfiguration.openshift.io]", func() {
106+
kubeClient, err := kubernetes.NewForConfig(oc.KubeFramework().ClientConfig())
107+
o.Expect(err).NotTo(o.HaveOccurred(), "Get KubeClient")
108+
109+
// Create custom MCP
110+
err = oc.Run("apply").Args("-f", customMCPFixture).Execute()
111+
o.Expect(err).NotTo(o.HaveOccurred())
112+
defer deleteMCP(oc, "custom")
113+
114+
// Add node to pool
115+
nodes, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{"node-role.kubernetes.io/worker": ""}).String()})
116+
o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
117+
o.Expect(nodes.Items).NotTo(o.BeEmpty())
118+
err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom=").Execute()
119+
o.Expect(err).NotTo(o.HaveOccurred(), "Label node")
120+
defer unlabelNode(oc, nodes.Items[0].Name)
121+
122+
SimplePISTest(oc, kubeClient, invalidPinnedImageSetFixture, false)
123+
})
124+
125+
})
126+
127+
func SimplePISTest(oc *exutil.CLI, kubeClient *kubernetes.Clientset, fixture string, success bool) {
128+
clientSet, err := mcClient.NewForConfig(oc.KubeFramework().ClientConfig())
129+
o.Expect(err).NotTo(o.HaveOccurred())
130+
131+
pis, err := getPISFromFixture(fixture)
132+
o.Expect(err).NotTo(o.HaveOccurred())
133+
134+
err = oc.Run("apply").Args("-f", fixture).Execute()
135+
o.Expect(err).NotTo(o.HaveOccurred(), "Applied PIS")
136+
defer deletePIS(oc, pis.Name)
137+
138+
waitTime := time.Minute * 5
139+
ctx, cancel := context.WithTimeout(context.Background(), waitTime)
140+
defer cancel()
141+
142+
err = waitForPISStatusX(ctx, oc, kubeClient, clientSet, pis.Name, success)
143+
o.Expect(err).NotTo(o.HaveOccurred(), "Checking status of PIS")
144+
}
145+
146+
func detectXCondition(oc *exutil.CLI, node corev1.Node, mcn *mcfgv1alpha1.MachineConfigNode, appliedPIS *mcfgv1alpha1.PinnedImageSet, detectingSuccess bool) (bool, bool, error) {
147+
if detectingSuccess {
148+
for _, cond := range mcn.Status.Conditions {
149+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsDegraded && cond.Status == "True" {
150+
return false, true, fmt.Errorf("PIS degraded for MCN %s with reason: %s and message: %s", mcn.Name, cond.Reason, cond.Message)
151+
}
152+
153+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsProgressing && cond.Status == "True" {
154+
return false, false, nil
155+
}
156+
}
157+
for _, img := range appliedPIS.Spec.PinnedImages {
158+
crictlStatus, err := exutil.DebugNodeRetryWithOptionsAndChroot(oc, node.Name, "openshift-machine-config-operator", "crictl", "inspecti", img.Name)
159+
if err != nil {
160+
return false, false, fmt.Errorf("failed to execute `crictl inspecti %s` on node %s: %w", img.Name, node.Name, err)
161+
}
162+
if !strings.Contains(crictlStatus, "imageSpec") {
163+
return false, false, fmt.Errorf("Image %s not present on node %s: %w", img.Name, node.Name, err)
164+
}
165+
}
166+
return true, false, nil
167+
} else {
168+
for _, cond := range mcn.Status.Conditions {
169+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsDegraded && cond.Status == "True" {
170+
continue
171+
}
172+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsProgressing && cond.Status == "True" {
173+
return false, false, nil
174+
}
175+
}
176+
return true, false, nil
177+
}
178+
}
179+
180+
func waitForPISStatusX(ctx context.Context, oc *exutil.CLI, kubeClient *kubernetes.Clientset, clientSet *mcClient.Clientset, pisName string, success bool) error {
181+
return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (done bool, err error) {
182+
// Wait for PIS object to get created
183+
appliedPIS, err := clientSet.MachineconfigurationV1alpha1().PinnedImageSets().Get(context.TODO(), pisName, metav1.GetOptions{})
184+
if err != nil {
185+
return false, fmt.Errorf("PIS Object not created yet: %w", err)
186+
}
187+
188+
pool, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(ctx, appliedPIS.Labels["machineconfiguration.openshift.io/role"], metav1.GetOptions{})
189+
if err != nil {
190+
return false, fmt.Errorf("failed to get MCP mentioned in PIS: %w", err)
191+
}
192+
193+
nodes, err := getNodesForPool(ctx, oc, kubeClient, pool)
194+
if err != nil {
195+
return false, fmt.Errorf("failed to get Nodes from MCP %q mentioned in PIS: %w", pool.Name, err)
196+
}
197+
198+
doneNodes := 0
199+
for _, node := range nodes.Items {
200+
mcn, err := clientSet.MachineconfigurationV1alpha1().MachineConfigNodes().Get(ctx, node.Name, metav1.GetOptions{})
201+
if err != nil {
202+
return false, fmt.Errorf("failed to get mcn: %w", err)
203+
}
204+
toContinue, isFatal, err := detectXCondition(oc, node, mcn, appliedPIS, success)
205+
if !toContinue {
206+
if isFatal {
207+
return true, err
208+
} else {
209+
if err != nil {
210+
framework.Logf("Retrying PIS Status with non-fatal error: %w", err)
211+
}
212+
return false, nil
213+
}
214+
}
215+
doneNodes += 1
216+
}
217+
if doneNodes == len(nodes.Items) {
218+
return true, nil
219+
}
220+
221+
return false, nil
222+
})
223+
}
224+
225+
func unlabelNode(oc *exutil.CLI, name string) error {
226+
return oc.AsAdmin().Run("label").Args("node", name, "node-role.kubernetes.io/custom-").Execute()
227+
}
228+
229+
func deleteKC(oc *exutil.CLI, name string) error {
230+
return oc.Run("delete").Args("kubeletconfig", name).Execute()
231+
}
232+
233+
func deleteMCP(oc *exutil.CLI, name string) error {
234+
return oc.Run("delete").Args("mcp", name).Execute()
235+
}
236+
237+
func deletePIS(oc *exutil.CLI, name string) error {
238+
return oc.Run("delete").Args("pinnedimageset", name).Execute()
239+
}
240+
241+
func getPISFromFixture(path string) (*mcfgv1alpha1.PinnedImageSet, error) {
242+
data, err := ioutil.ReadFile(path)
243+
if err != nil {
244+
return nil, err
245+
}
246+
247+
ob := new(mcfgv1alpha1.PinnedImageSet)
248+
err = yaml.Unmarshal(data, ob)
249+
if err != nil {
250+
return nil, err
251+
}
252+
253+
return ob, nil
254+
}

0 commit comments

Comments
 (0)