Skip to content

Commit be0c556

Browse files
perdasilvaperdasilva
authoredDec 14, 2021
feat: catalog source pod scheduling config overrides (#2512)
Signed-off-by: Per G. da Silva <[email protected]>
·
v0.31.0v0.20.0
1 parent e3dfbc5 commit be0c556

File tree

4 files changed

+401
-0
lines changed

4 files changed

+401
-0
lines changed
 

‎pkg/controller/registry/reconciler/reconciler.go

+26
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,32 @@ func Pod(source *v1alpha1.CatalogSource, name string, image string, saName strin
169169
},
170170
}
171171

172+
// Override scheduling options if specified
173+
if source.Spec.GrpcPodConfig != nil {
174+
grpcPodConfig := source.Spec.GrpcPodConfig
175+
176+
// Override node selector
177+
if grpcPodConfig.NodeSelector != nil {
178+
pod.Spec.NodeSelector = make(map[string]string, len(grpcPodConfig.NodeSelector))
179+
for key, value := range grpcPodConfig.NodeSelector {
180+
pod.Spec.NodeSelector[key] = value
181+
}
182+
}
183+
184+
// Override priority class name
185+
if grpcPodConfig.PriorityClassName != nil {
186+
pod.Spec.PriorityClassName = *grpcPodConfig.PriorityClassName
187+
}
188+
189+
// Override tolerations
190+
if grpcPodConfig.Tolerations != nil {
191+
pod.Spec.Tolerations = make([]v1.Toleration, len(grpcPodConfig.Tolerations))
192+
for index, toleration := range grpcPodConfig.Tolerations {
193+
pod.Spec.Tolerations[index] = *toleration.DeepCopy()
194+
}
195+
}
196+
}
197+
172198
// Set priorityclass if its annotation exists
173199
if prio, ok := annotations[CatalogPriorityClassKey]; ok && prio != "" {
174200
pod.Spec.PriorityClassName = prio

‎pkg/controller/registry/reconciler/reconciler_test.go

+176
Original file line numberDiff line numberDiff line change
@@ -94,3 +94,179 @@ func TestPodContainerSecurityContext(t *testing.T) {
9494
gotContainerSecCtx := gotPod.Spec.Containers[0].SecurityContext
9595
require.Equal(t, expectedContainerSecCtx, gotContainerSecCtx)
9696
}
97+
98+
func TestPodSchedulingOverrides(t *testing.T) {
99+
// This test ensures that any overriding pod scheduling configuration elements
100+
// defined in spec.grpcPodConfig are applied to the catalog source pod created
101+
// when spec.sourceType = 'grpc' and spec.image is set.
102+
var tolerationSeconds int64 = 120
103+
var overriddenPriorityClassName = "some-prio-class"
104+
var overriddenNodeSelectors = map[string]string{
105+
"label": "value",
106+
"label2": "value2",
107+
}
108+
var defaultNodeSelectors = map[string]string{
109+
"kubernetes.io/os": "linux",
110+
}
111+
var defaultPriorityClassName = ""
112+
113+
var overriddenTolerations = []corev1.Toleration{
114+
{
115+
Key: "some/key",
116+
Operator: corev1.TolerationOpExists,
117+
Effect: corev1.TaintEffectNoExecute,
118+
TolerationSeconds: &tolerationSeconds,
119+
},
120+
{
121+
Key: "someother/key",
122+
Operator: corev1.TolerationOpEqual,
123+
Effect: corev1.TaintEffectNoSchedule,
124+
},
125+
}
126+
127+
testCases := []struct {
128+
title string
129+
catalogSource *v1alpha1.CatalogSource
130+
expectedNodeSelectors map[string]string
131+
expectedTolerations []corev1.Toleration
132+
expectedPriorityClassName string
133+
annotations map[string]string
134+
}{
135+
{
136+
title: "no overrides",
137+
catalogSource: &v1alpha1.CatalogSource{
138+
ObjectMeta: metav1.ObjectMeta{
139+
Name: "test",
140+
Namespace: "testns",
141+
},
142+
Spec: v1alpha1.CatalogSourceSpec{
143+
SourceType: v1alpha1.SourceTypeGrpc,
144+
Image: "repo/image:tag",
145+
},
146+
},
147+
expectedTolerations: nil,
148+
expectedPriorityClassName: defaultPriorityClassName,
149+
expectedNodeSelectors: defaultNodeSelectors,
150+
}, {
151+
title: "override node selectors",
152+
catalogSource: &v1alpha1.CatalogSource{
153+
ObjectMeta: metav1.ObjectMeta{
154+
Name: "test",
155+
Namespace: "testns",
156+
},
157+
Spec: v1alpha1.CatalogSourceSpec{
158+
SourceType: v1alpha1.SourceTypeGrpc,
159+
Image: "repo/image:tag",
160+
GrpcPodConfig: &v1alpha1.GrpcPodConfig{
161+
NodeSelector: overriddenNodeSelectors,
162+
},
163+
},
164+
},
165+
expectedTolerations: nil,
166+
expectedPriorityClassName: defaultPriorityClassName,
167+
expectedNodeSelectors: overriddenNodeSelectors,
168+
}, {
169+
title: "override priority class name",
170+
catalogSource: &v1alpha1.CatalogSource{
171+
ObjectMeta: metav1.ObjectMeta{
172+
Name: "test",
173+
Namespace: "testns",
174+
},
175+
Spec: v1alpha1.CatalogSourceSpec{
176+
SourceType: v1alpha1.SourceTypeGrpc,
177+
Image: "repo/image:tag",
178+
GrpcPodConfig: &v1alpha1.GrpcPodConfig{
179+
PriorityClassName: &overriddenPriorityClassName,
180+
},
181+
},
182+
},
183+
expectedTolerations: nil,
184+
expectedPriorityClassName: overriddenPriorityClassName,
185+
expectedNodeSelectors: defaultNodeSelectors,
186+
}, {
187+
title: "doesn't override priority class name when its nil",
188+
catalogSource: &v1alpha1.CatalogSource{
189+
ObjectMeta: metav1.ObjectMeta{
190+
Name: "test",
191+
Namespace: "testns",
192+
},
193+
Spec: v1alpha1.CatalogSourceSpec{
194+
SourceType: v1alpha1.SourceTypeGrpc,
195+
Image: "repo/image:tag",
196+
GrpcPodConfig: &v1alpha1.GrpcPodConfig{
197+
PriorityClassName: nil,
198+
},
199+
},
200+
},
201+
expectedTolerations: nil,
202+
expectedPriorityClassName: defaultPriorityClassName,
203+
expectedNodeSelectors: defaultNodeSelectors,
204+
}, {
205+
title: "Override node tolerations",
206+
catalogSource: &v1alpha1.CatalogSource{
207+
ObjectMeta: metav1.ObjectMeta{
208+
Name: "test",
209+
Namespace: "testns",
210+
},
211+
Spec: v1alpha1.CatalogSourceSpec{
212+
SourceType: v1alpha1.SourceTypeGrpc,
213+
Image: "repo/image:tag",
214+
GrpcPodConfig: &v1alpha1.GrpcPodConfig{
215+
Tolerations: overriddenTolerations,
216+
},
217+
},
218+
},
219+
expectedTolerations: overriddenTolerations,
220+
expectedPriorityClassName: defaultPriorityClassName,
221+
expectedNodeSelectors: defaultNodeSelectors,
222+
}, {
223+
title: "Override all the things",
224+
catalogSource: &v1alpha1.CatalogSource{
225+
ObjectMeta: metav1.ObjectMeta{
226+
Name: "test",
227+
Namespace: "testns",
228+
},
229+
Spec: v1alpha1.CatalogSourceSpec{
230+
SourceType: v1alpha1.SourceTypeGrpc,
231+
Image: "repo/image:tag",
232+
GrpcPodConfig: &v1alpha1.GrpcPodConfig{
233+
NodeSelector: overriddenNodeSelectors,
234+
PriorityClassName: &overriddenPriorityClassName,
235+
Tolerations: overriddenTolerations,
236+
},
237+
},
238+
},
239+
expectedTolerations: overriddenTolerations,
240+
expectedPriorityClassName: overriddenPriorityClassName,
241+
expectedNodeSelectors: overriddenNodeSelectors,
242+
}, {
243+
title: "priorityClassName annotation takes precedence",
244+
catalogSource: &v1alpha1.CatalogSource{
245+
ObjectMeta: metav1.ObjectMeta{
246+
Name: "test",
247+
Namespace: "testns",
248+
},
249+
Spec: v1alpha1.CatalogSourceSpec{
250+
SourceType: v1alpha1.SourceTypeGrpc,
251+
Image: "repo/image:tag",
252+
GrpcPodConfig: &v1alpha1.GrpcPodConfig{
253+
PriorityClassName: &overriddenPriorityClassName,
254+
},
255+
},
256+
},
257+
expectedTolerations: nil,
258+
annotations: map[string]string{
259+
CatalogPriorityClassKey: "some-OTHER-prio-class",
260+
},
261+
expectedPriorityClassName: "some-OTHER-prio-class",
262+
expectedNodeSelectors: defaultNodeSelectors,
263+
},
264+
}
265+
266+
for _, testCase := range testCases {
267+
pod := Pod(testCase.catalogSource, "hello", "busybox", "", map[string]string{}, testCase.annotations, int32(0), int32(0))
268+
require.Equal(t, testCase.expectedNodeSelectors, pod.Spec.NodeSelector)
269+
require.Equal(t, testCase.expectedPriorityClassName, pod.Spec.PriorityClassName)
270+
require.Equal(t, testCase.expectedTolerations, pod.Spec.Tolerations)
271+
}
272+
}

‎test/e2e/catsrc_pod_config_e2e_test.go

+198
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
8+
. "github.com/onsi/ginkgo"
9+
. "github.com/onsi/gomega"
10+
"github.com/operator-framework/api/pkg/operators/v1alpha1"
11+
"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
k8scontrollerclient "sigs.k8s.io/controller-runtime/pkg/client"
14+
15+
corev1 "k8s.io/api/core/v1"
16+
)
17+
18+
const catalogSourceLabel = "olm.catalogSource"
19+
20+
var _ = By
21+
22+
var _ = Describe("CatalogSource Grpc Pod Config", func() {
23+
var (
24+
generatedNamespace corev1.Namespace
25+
)
26+
27+
BeforeEach(func() {
28+
generatedNamespace = SetupGeneratedTestNamespace(genName("catsrc-grpc-pod-config-e2e-"))
29+
})
30+
31+
AfterEach(func() {
32+
TeardownNamespace(generatedNamespace.GetName())
33+
})
34+
35+
When("the user wants more control over where the grpc catalog source pod gets scheduled", func() {
36+
var (
37+
client k8scontrollerclient.Client
38+
catalogSource *v1alpha1.CatalogSource
39+
defaultNodeSelector = map[string]string{
40+
"kubernetes.io/os": "linux",
41+
}
42+
defaultTolerations []corev1.Toleration = nil
43+
catalogSourceName = "test-catsrc"
44+
defaultPriorityClassName = ""
45+
)
46+
47+
BeforeEach(func() {
48+
client = ctx.Ctx().Client()
49+
50+
// must be a grpc source type with spec.image defined
51+
catalogSource = &v1alpha1.CatalogSource{
52+
ObjectMeta: metav1.ObjectMeta{
53+
Name: catalogSourceName,
54+
Namespace: generatedNamespace.GetName(),
55+
},
56+
Spec: v1alpha1.CatalogSourceSpec{
57+
SourceType: v1alpha1.SourceTypeGrpc,
58+
Image: "repo/image:tag",
59+
},
60+
}
61+
})
62+
63+
AfterEach(func() {
64+
// assume the catalog source was created and just delete it
65+
_ = client.Delete(context.TODO(), catalogSource)
66+
67+
// wait for it to go away
68+
Expect(waitForDelete(func() error {
69+
return client.Get(context.TODO(), k8scontrollerclient.ObjectKey{
70+
Name: catalogSource.GetName(),
71+
Namespace: catalogSource.GetNamespace(),
72+
}, &v1alpha1.CatalogSource{})
73+
})).To(BeNil())
74+
})
75+
76+
It("should override the pod's spec.priorityClassName", func() {
77+
var overridenPriorityClassName = "system-node-critical"
78+
79+
// create catalog source
80+
catalogSource.Spec.GrpcPodConfig = &v1alpha1.GrpcPodConfig{
81+
PriorityClassName: &overridenPriorityClassName,
82+
}
83+
mustCreateCatalogSource(client, catalogSource)
84+
85+
// Check overrides are present in the spec
86+
catalogSourcePod := mustGetCatalogSourcePod(client, catalogSource)
87+
Expect(catalogSourcePod).ToNot(BeNil())
88+
Expect(catalogSourcePod.Spec.NodeSelector).To(BeEquivalentTo(defaultNodeSelector))
89+
Expect(catalogSourcePod.Spec.Tolerations).To(ContainElements(defaultTolerations))
90+
Expect(catalogSourcePod.Spec.PriorityClassName).To(Equal(overridenPriorityClassName))
91+
})
92+
93+
It("should override the pod's spec.priorityClassName when it is empty", func() {
94+
var overridenPriorityClassName = ""
95+
96+
// create catalog source
97+
catalogSource.Spec.GrpcPodConfig = &v1alpha1.GrpcPodConfig{
98+
PriorityClassName: &overridenPriorityClassName,
99+
}
100+
mustCreateCatalogSource(client, catalogSource)
101+
102+
// Check overrides are present in the spec
103+
catalogSourcePod := mustGetCatalogSourcePod(client, catalogSource)
104+
Expect(catalogSourcePod).ToNot(BeNil())
105+
Expect(catalogSourcePod.Spec.NodeSelector).To(BeEquivalentTo(defaultNodeSelector))
106+
Expect(catalogSourcePod.Spec.Tolerations).To(ContainElements(defaultTolerations))
107+
Expect(catalogSourcePod.Spec.PriorityClassName).To(Equal(overridenPriorityClassName))
108+
})
109+
110+
It("should override the pod's spec.nodeSelector", func() {
111+
var overridenNodeSelector = map[string]string{
112+
"kubernetes.io/os": "linux",
113+
"some": "tag",
114+
}
115+
116+
// create catalog source
117+
catalogSource.Spec.GrpcPodConfig = &v1alpha1.GrpcPodConfig{
118+
NodeSelector: overridenNodeSelector,
119+
}
120+
mustCreateCatalogSource(client, catalogSource)
121+
122+
// Check overrides are present in the spec
123+
catalogSourcePod := mustGetCatalogSourcePod(client, catalogSource)
124+
Expect(catalogSourcePod).ToNot(BeNil())
125+
Expect(catalogSourcePod.Spec.NodeSelector).To(BeEquivalentTo(overridenNodeSelector))
126+
Expect(catalogSourcePod.Spec.Tolerations).To(ContainElements(defaultTolerations))
127+
Expect(catalogSourcePod.Spec.PriorityClassName).To(Equal(defaultPriorityClassName))
128+
})
129+
130+
It("should override the pod's spec.tolerations", func() {
131+
var tolerationSeconds int64 = 120
132+
var overriddenTolerations = []corev1.Toleration{
133+
{
134+
Key: "some/key",
135+
Operator: corev1.TolerationOpExists,
136+
Effect: corev1.TaintEffectNoExecute,
137+
TolerationSeconds: &tolerationSeconds,
138+
},
139+
{
140+
Key: "someother/key",
141+
Operator: corev1.TolerationOpEqual,
142+
Effect: corev1.TaintEffectNoSchedule,
143+
},
144+
}
145+
146+
// create catalog source
147+
catalogSource.Spec.GrpcPodConfig = &v1alpha1.GrpcPodConfig{
148+
Tolerations: overriddenTolerations,
149+
}
150+
mustCreateCatalogSource(client, catalogSource)
151+
152+
// Check overrides are present in the spec
153+
catalogSourcePod := mustGetCatalogSourcePod(client, catalogSource)
154+
Expect(catalogSourcePod).ToNot(BeNil())
155+
Expect(catalogSourcePod.Spec.NodeSelector).To(BeEquivalentTo(defaultNodeSelector))
156+
Expect(catalogSourcePod.Spec.Tolerations).To(ContainElements(overriddenTolerations))
157+
Expect(catalogSourcePod.Spec.PriorityClassName).To(Equal(defaultPriorityClassName))
158+
})
159+
})
160+
})
161+
162+
func mustGetCatalogSourcePod(client k8scontrollerclient.Client, catalogSource *v1alpha1.CatalogSource) *corev1.Pod {
163+
var podList = corev1.PodList{}
164+
165+
var opts = []k8scontrollerclient.ListOption{
166+
k8scontrollerclient.InNamespace(catalogSource.GetNamespace()),
167+
// NOTE: this will fail if we stop setting the label on the catalog source pod
168+
k8scontrollerclient.MatchingLabels{
169+
catalogSourceLabel: catalogSource.GetName(),
170+
},
171+
}
172+
173+
// Try to get a pod until its found and there's only one of them
174+
Eventually(func() error {
175+
if err := client.List(context.TODO(), &podList, opts...); err != nil {
176+
return err
177+
}
178+
if len(podList.Items) != 1 {
179+
return errors.New(fmt.Sprintf("expecting one catalog source pod but found %d", len(podList.Items)))
180+
}
181+
return nil
182+
}).Should(BeNil())
183+
184+
return &podList.Items[0]
185+
}
186+
187+
func mustCreateCatalogSource(client k8scontrollerclient.Client, catalogSource *v1alpha1.CatalogSource) {
188+
// create the object
189+
Expect(client.Create(context.TODO(), catalogSource)).To(BeNil())
190+
191+
// wait for object to be appear
192+
Eventually(func() error {
193+
return client.Get(context.TODO(), k8scontrollerclient.ObjectKey{
194+
Name: catalogSource.Name,
195+
Namespace: catalogSource.GetNamespace(),
196+
}, &v1alpha1.CatalogSource{})
197+
}).Should(BeNil())
198+
}

‎test/e2e/ctx/provisioner_kind.go

+1
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ func Provision(ctx *TestContext) (func(), error) {
8383
}
8484
kubeconfigPath := filepath.Join(dir, "kubeconfig")
8585

86+
ctx.Logf("e2e cluster kubeconfig: %s\n", kubeconfigPath)
8687
provider := cluster.NewProvider(
8788
cluster.ProviderWithLogger(kindLogAdapter{ctx}),
8889
)

0 commit comments

Comments
 (0)
Please sign in to comment.