Skip to content

Commit 275d2b6

Browse files
RishabhSainiRishabhSaini
committed
pis testing
1 parent 59d86be commit 275d2b6

File tree

10 files changed

+399
-0
lines changed

10 files changed

+399
-0
lines changed

‎test/extended/machine_config/helpers.go

+12
Original file line numberDiff line numberDiff line change
@@ -292,3 +292,15 @@ func WaitForOneMasterNodeToBeReady(oc *exutil.CLI) error {
292292
}, 5*time.Minute, 10*time.Second).Should(o.BeTrue())
293293
return nil
294294
}
295+
296+
func getNodesForPool(ctx context.Context, oc *exutil.CLI, pool *mcfgv1.MachineConfigPool) (*corev1.NodeList, error) {
297+
selector, err := metav1.LabelSelectorAsSelector(pool.Spec.NodeSelector)
298+
if err != nil {
299+
return nil, fmt.Errorf("invalid label selector: %w", err)
300+
}
301+
nodes, err := oc.KubeClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
302+
if err != nil {
303+
return nil, fmt.Errorf("couldnt get nodes for mcp: %w", err)
304+
}
305+
return nodes, nil
306+
}

‎test/extended/machine_config/pinnedimages.go

+229
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
package machine_config
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io/ioutil"
7+
"path/filepath"
8+
"strings"
9+
"time"
10+
11+
g "github.com/onsi/ginkgo/v2"
12+
o "github.com/onsi/gomega"
13+
mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1"
14+
mcClient "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
15+
exutil "github.com/openshift/origin/test/extended/util"
16+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17+
18+
"k8s.io/apimachinery/pkg/util/wait"
19+
"sigs.k8s.io/yaml"
20+
)
21+
22+
// This test is [Serial] because it modifies the cluster/machineconfigurations.operator.openshift.io object in each test.
23+
var _ = g.Describe("[sig-mco][OCPFeatureGate:PinnedImages][OCPFeatureGate:MachineConfigNode][Serial]", func() {
24+
defer g.GinkgoRecover()
25+
var (
26+
MCOMachineConfigurationBaseDir = exutil.FixturePath("testdata", "machine_config", "machineconfigurations")
27+
pinnedImageSetFixture = filepath.Join(MCOMachineConfigurationBaseDir, "pis.yaml")
28+
customMCP = filepath.Join(MCOMachineConfigurationBaseDir, "customMCP.yaml")
29+
customMCPpinnedImageSetFixture = filepath.Join(MCOMachineConfigurationBaseDir, "customMCPpis.yaml")
30+
customGcKC = filepath.Join(MCOMachineConfigurationBaseDir, "gcKC.yaml")
31+
invalidPinnedImageSetFixture = filepath.Join(MCOMachineConfigurationBaseDir, "invalidPis.yaml")
32+
oc = exutil.NewCLIWithoutNamespace("machine-config")
33+
)
34+
// Ensure each test pins a separate image, since we are not deleting them after each
35+
36+
37+
38+
g.It("All Nodes in the Pool should have the PinnedImages even after Garbage Collection [apigroup:machineconfiguration.openshift.io]", func() {
39+
// Create custom MCP
40+
err := oc.Run("apply").Args("-f", customMCP).Execute()
41+
o.Expect(err).NotTo(o.HaveOccurred())
42+
defer deleteMCP(oc, "custom")
43+
44+
// Add node to pool
45+
nodes, err := oc.KubeClient().CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
46+
o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
47+
err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom="+"").Execute()
48+
49+
// Apply KC to Pool
50+
err := oc.Run("apply").Args("-f", gcKC).Execute()
51+
o.Expect(err).NotTo(o.HaveOccurred())
52+
defer deleteKC(oc, "custom-gc-config")
53+
54+
SimplePISTest(oc, customGCMCPpinnedImageSetFixture, true)
55+
})
56+
57+
58+
g.It("All Nodes in the Custom Pool should have the PinnedImages in PIS [apigroup:machineconfiguration.openshift.io]", func() {
59+
// Create custom MCP
60+
err := oc.Run("apply").Args("-f", customMCP).Execute()
61+
o.Expect(err).NotTo(o.HaveOccurred())
62+
defer deleteMCP(oc, "custom")
63+
64+
// Add node to pool
65+
nodes, err := oc.KubeClient().CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
66+
o.Expect(err).NotTo(o.HaveOccurred(), "Get all nodes")
67+
err = oc.AsAdmin().Run("label").Args("node", nodes.Items[0].Name, "node-role.kubernetes.io/custom="+"").Execute()
68+
69+
SimplePISTest(oc, customMCPpinnedImageSetFixture, true)
70+
})
71+
72+
g.It("All Nodes in the Pool should have the PinnedImages PIS [apigroup:machineconfiguration.openshift.io]", func() {
73+
SimplePISTest(oc, pinnedImageSetFixture, true)
74+
})
75+
76+
g.It("Invalid PIS leads to degraded MCN [apigroup:machineconfiguration.openshift.io]", func() {
77+
SimplePISTest(oc, invalidPinnedImageSetFixture, false)
78+
})
79+
80+
})
81+
82+
func SimplePISTest(oc *exutil.CLI, fixture string, success bool) {
83+
clientSet, err := mcClient.NewForConfig(oc.KubeFramework().ClientConfig())
84+
o.Expect(err).NotTo(o.HaveOccurred())
85+
86+
pis, err := getPISFromFixture(fixture)
87+
o.Expect(err).NotTo(o.HaveOccurred())
88+
89+
err = oc.Run("apply").Args("-f", fixture).Execute()
90+
o.Expect(err).NotTo(o.HaveOccurred(), "Applied PIS")
91+
defer deletePIS(oc, pis.Name)
92+
93+
waitTime := time.Minute * 20
94+
ctx, cancel := context.WithTimeout(context.Background(), waitTime)
95+
defer cancel()
96+
97+
if success {
98+
waitForPISStatusSuccess(ctx, oc, clientSet, pis.Name)
99+
} else {
100+
waitForPISStatusDegraded(ctx, oc, clientSet, pis.Name)
101+
}
102+
103+
}
104+
105+
func waitForPISStatusDegraded(ctx context.Context, oc *exutil.CLI, clientSet *mcClient.Clientset, pisName string) error {
106+
return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (done bool, err error) {
107+
// Wait for PIS object to get created
108+
appliedPIS, err := clientSet.MachineconfigurationV1alpha1().PinnedImageSets().Get(context.TODO(), pisName, metav1.GetOptions{})
109+
if err != nil {
110+
return false, fmt.Errorf("PIS Object not created yet: %w", err)
111+
}
112+
113+
pool, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(ctx, appliedPIS.Labels["machineconfiguration.openshift.io/role"], metav1.GetOptions{})
114+
if err != nil {
115+
return true, fmt.Errorf("failed to get MCP mentioned in PIS: %w", err)
116+
}
117+
118+
nodes, err := getNodesForPool(ctx, oc, pool)
119+
doneNodes := 0
120+
for _, node := range nodes.Items {
121+
mcn, err := clientSet.MachineconfigurationV1alpha1().MachineConfigNodes().Get(ctx, node.Name, metav1.GetOptions{})
122+
if err != nil {
123+
return true, fmt.Errorf("failed to get mcn: %w", err)
124+
}
125+
for _, cond := range mcn.Status.Conditions {
126+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsDegraded && cond.Status == "True" {
127+
continue
128+
}
129+
else {
130+
return false
131+
}
132+
}
133+
for _, img := range appliedPIS.Spec.PinnedImages {
134+
crictlStatus, err := exutil.DebugNodeRetryWithOptionsAndChroot(oc, node.Name, "openshift-machine-config-operator", "crictl", "inspecti", img.Name)
135+
if err != nil {
136+
return false, fmt.Errorf("failed to execute `crictl inspecti %s` on node %s: %w", img.Name, node.Name, err)
137+
}
138+
if !strings.Contains(crictlStatus, "imageSpec") {
139+
return false, fmt.Errorf("Image %s not present on node %s: %w", img.Name, node.Name, err)
140+
}
141+
}
142+
doneNodes += 1
143+
}
144+
if doneNodes == len(nodes.Items) {
145+
return true, nil
146+
}
147+
148+
return false, nil
149+
})
150+
}
151+
152+
153+
func waitForPISStatusSuccess(ctx context.Context, oc *exutil.CLI, clientSet *mcClient.Clientset, pisName string) error {
154+
return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (done bool, err error) {
155+
// Wait for PIS object to get created
156+
appliedPIS, err := clientSet.MachineconfigurationV1alpha1().PinnedImageSets().Get(context.TODO(), pisName, metav1.GetOptions{})
157+
if err != nil {
158+
return false, fmt.Errorf("PIS Object not created yet: %w", err)
159+
}
160+
161+
pool, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(ctx, appliedPIS.Labels["machineconfiguration.openshift.io/role"], metav1.GetOptions{})
162+
if err != nil {
163+
return true, fmt.Errorf("failed to get MCP mentioned in PIS: %w", err)
164+
}
165+
166+
nodes, err := getNodesForPool(ctx, oc, pool)
167+
doneNodes := 0
168+
for _, node := range nodes.Items {
169+
mcn, err := clientSet.MachineconfigurationV1alpha1().MachineConfigNodes().Get(ctx, node.Name, metav1.GetOptions{})
170+
if err != nil {
171+
return true, fmt.Errorf("failed to get mcn: %w", err)
172+
}
173+
for _, cond := range mcn.Status.Conditions {
174+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsDegraded && cond.Status == "True" {
175+
return true, fmt.Errorf("PIS degraded for MCN %s with reason: %s and message: %s", mcn.Name, cond.Reason, cond.Message)
176+
}
177+
178+
if mcfgv1alpha1.StateProgress(cond.Type) == mcfgv1alpha1.MachineConfigNodePinnedImageSetsProgressing && cond.Status == "True" {
179+
return false, nil
180+
}
181+
}
182+
for _, img := range appliedPIS.Spec.PinnedImages {
183+
crictlStatus, err := exutil.DebugNodeRetryWithOptionsAndChroot(oc, node.Name, "openshift-machine-config-operator", "crictl", "inspecti", img.Name)
184+
if err != nil {
185+
return false, fmt.Errorf("failed to execute `crictl inspecti %s` on node %s: %w", img.Name, node.Name, err)
186+
}
187+
if !strings.Contains(crictlStatus, "imageSpec") {
188+
return false, fmt.Errorf("Image %s not present on node %s: %w", img.Name, node.Name, err)
189+
}
190+
}
191+
doneNodes += 1
192+
}
193+
if doneNodes == len(nodes.Items) {
194+
return true, nil
195+
}
196+
197+
return false, nil
198+
})
199+
}
200+
201+
func deleteKC(oc *exutil.CLI, name string) error {
202+
err = oc.Run("delete").Args("kubeletconfig", name).Execute()
203+
return err
204+
}
205+
206+
func deleteMCP(oc *exutil.CLI, name string) error {
207+
err = oc.Run("delete").Args("mcp", name).Execute()
208+
return err
209+
}
210+
211+
func deletePIS(oc *exutil.CLI, name string) error {
212+
err = oc.Run("delete").Args("pinnedimageset", name).Execute()
213+
return err
214+
}
215+
216+
func getPISFromFixture(path string) (*mcfgv1alpha1.PinnedImageSet, error) {
217+
data, err := ioutil.ReadFile(path)
218+
if err != nil {
219+
return nil, err
220+
}
221+
222+
ob := new(mcfgv1alpha1.PinnedImageSet)
223+
err = yaml.Unmarshal(data, ob)
224+
if err != nil {
225+
return nil, err
226+
}
227+
228+
return ob, err
229+
}

‎test/extended/testdata/bindata.go

+90
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/extended/testdata/machine_config/machineconfigurations/customGCMCPpis.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: machineconfiguration.openshift.io/v1alpha1
2+
kind: PinnedImageSet
3+
metadata:
4+
name: test-pinned
5+
labels:
6+
machineconfiguration.openshift.io/role: "custom"
7+
spec:
8+
pinnedImages:
9+
- name: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e43b2ef4fbc42dbcbea5d67f57f3feed38f6b45fb712c99acb06490103e277a9

0 commit comments

Comments
 (0)