diff --git a/pkg/controller/registry/reconciler/reconciler.go b/pkg/controller/registry/reconciler/reconciler.go index 88315b5688..31e42eba04 100644 --- a/pkg/controller/registry/reconciler/reconciler.go +++ b/pkg/controller/registry/reconciler/reconciler.go @@ -169,6 +169,32 @@ func Pod(source *v1alpha1.CatalogSource, name string, image string, saName strin }, } + // Override scheduling options if specified + if source.Spec.GrpcPodConfig != nil { + grpcPodConfig := source.Spec.GrpcPodConfig + + // Override node selector + if grpcPodConfig.NodeSelector != nil { + pod.Spec.NodeSelector = make(map[string]string, len(grpcPodConfig.NodeSelector)) + for key, value := range grpcPodConfig.NodeSelector { + pod.Spec.NodeSelector[key] = value + } + } + + // Override priority class name + if grpcPodConfig.PriorityClassName != nil { + pod.Spec.PriorityClassName = *grpcPodConfig.PriorityClassName + } + + // Override tolerations + if grpcPodConfig.Tolerations != nil { + pod.Spec.Tolerations = make([]v1.Toleration, len(grpcPodConfig.Tolerations)) + for index, toleration := range grpcPodConfig.Tolerations { + pod.Spec.Tolerations[index] = *toleration.DeepCopy() + } + } + } + // Set priorityclass if its annotation exists if prio, ok := annotations[CatalogPriorityClassKey]; ok && prio != "" { pod.Spec.PriorityClassName = prio diff --git a/pkg/controller/registry/reconciler/reconciler_test.go b/pkg/controller/registry/reconciler/reconciler_test.go index 87cc7b555e..f73b6f3762 100644 --- a/pkg/controller/registry/reconciler/reconciler_test.go +++ b/pkg/controller/registry/reconciler/reconciler_test.go @@ -94,3 +94,179 @@ func TestPodContainerSecurityContext(t *testing.T) { gotContainerSecCtx := gotPod.Spec.Containers[0].SecurityContext require.Equal(t, expectedContainerSecCtx, gotContainerSecCtx) } + +func TestPodSchedulingOverrides(t *testing.T) { + // This test ensures that any overriding pod scheduling configuration elements + // defined in spec.grpcPodConfig are applied to the catalog source pod created + // when spec.sourceType = 'grpc' and spec.image is set. + var tolerationSeconds int64 = 120 + var overriddenPriorityClassName = "some-prio-class" + var overriddenNodeSelectors = map[string]string{ + "label": "value", + "label2": "value2", + } + var defaultNodeSelectors = map[string]string{ + "kubernetes.io/os": "linux", + } + var defaultPriorityClassName = "" + + var overriddenTolerations = []corev1.Toleration{ + { + Key: "some/key", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoExecute, + TolerationSeconds: &tolerationSeconds, + }, + { + Key: "someother/key", + Operator: corev1.TolerationOpEqual, + Effect: corev1.TaintEffectNoSchedule, + }, + } + + testCases := []struct { + title string + catalogSource *v1alpha1.CatalogSource + expectedNodeSelectors map[string]string + expectedTolerations []corev1.Toleration + expectedPriorityClassName string + annotations map[string]string + }{ + { + title: "no overrides", + catalogSource: &v1alpha1.CatalogSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "testns", + }, + Spec: v1alpha1.CatalogSourceSpec{ + SourceType: v1alpha1.SourceTypeGrpc, + Image: "repo/image:tag", + }, + }, + expectedTolerations: nil, + expectedPriorityClassName: defaultPriorityClassName, + expectedNodeSelectors: defaultNodeSelectors, + }, { + title: "override node selectors", + catalogSource: &v1alpha1.CatalogSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "testns", + }, + Spec: v1alpha1.CatalogSourceSpec{ + SourceType: v1alpha1.SourceTypeGrpc, + Image: "repo/image:tag", + GrpcPodConfig: &v1alpha1.GrpcPodConfig{ + NodeSelector: overriddenNodeSelectors, + }, + }, + }, + expectedTolerations: nil, + expectedPriorityClassName: defaultPriorityClassName, + expectedNodeSelectors: overriddenNodeSelectors, + }, { + title: "override priority class name", + catalogSource: &v1alpha1.CatalogSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "testns", + }, + Spec: v1alpha1.CatalogSourceSpec{ + SourceType: v1alpha1.SourceTypeGrpc, + Image: "repo/image:tag", + GrpcPodConfig: &v1alpha1.GrpcPodConfig{ + PriorityClassName: &overriddenPriorityClassName, + }, + }, + }, + expectedTolerations: nil, + expectedPriorityClassName: overriddenPriorityClassName, + expectedNodeSelectors: defaultNodeSelectors, + }, { + title: "doesn't override priority class name when its nil", + catalogSource: &v1alpha1.CatalogSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "testns", + }, + Spec: v1alpha1.CatalogSourceSpec{ + SourceType: v1alpha1.SourceTypeGrpc, + Image: "repo/image:tag", + GrpcPodConfig: &v1alpha1.GrpcPodConfig{ + PriorityClassName: nil, + }, + }, + }, + expectedTolerations: nil, + expectedPriorityClassName: defaultPriorityClassName, + expectedNodeSelectors: defaultNodeSelectors, + }, { + title: "Override node tolerations", + catalogSource: &v1alpha1.CatalogSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "testns", + }, + Spec: v1alpha1.CatalogSourceSpec{ + SourceType: v1alpha1.SourceTypeGrpc, + Image: "repo/image:tag", + GrpcPodConfig: &v1alpha1.GrpcPodConfig{ + Tolerations: overriddenTolerations, + }, + }, + }, + expectedTolerations: overriddenTolerations, + expectedPriorityClassName: defaultPriorityClassName, + expectedNodeSelectors: defaultNodeSelectors, + }, { + title: "Override all the things", + catalogSource: &v1alpha1.CatalogSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "testns", + }, + Spec: v1alpha1.CatalogSourceSpec{ + SourceType: v1alpha1.SourceTypeGrpc, + Image: "repo/image:tag", + GrpcPodConfig: &v1alpha1.GrpcPodConfig{ + NodeSelector: overriddenNodeSelectors, + PriorityClassName: &overriddenPriorityClassName, + Tolerations: overriddenTolerations, + }, + }, + }, + expectedTolerations: overriddenTolerations, + expectedPriorityClassName: overriddenPriorityClassName, + expectedNodeSelectors: overriddenNodeSelectors, + }, { + title: "priorityClassName annotation takes precedence", + catalogSource: &v1alpha1.CatalogSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "testns", + }, + Spec: v1alpha1.CatalogSourceSpec{ + SourceType: v1alpha1.SourceTypeGrpc, + Image: "repo/image:tag", + GrpcPodConfig: &v1alpha1.GrpcPodConfig{ + PriorityClassName: &overriddenPriorityClassName, + }, + }, + }, + expectedTolerations: nil, + annotations: map[string]string{ + CatalogPriorityClassKey: "some-OTHER-prio-class", + }, + expectedPriorityClassName: "some-OTHER-prio-class", + expectedNodeSelectors: defaultNodeSelectors, + }, + } + + for _, testCase := range testCases { + pod := Pod(testCase.catalogSource, "hello", "busybox", "", map[string]string{}, testCase.annotations, int32(0), int32(0)) + require.Equal(t, testCase.expectedNodeSelectors, pod.Spec.NodeSelector) + require.Equal(t, testCase.expectedPriorityClassName, pod.Spec.PriorityClassName) + require.Equal(t, testCase.expectedTolerations, pod.Spec.Tolerations) + } +} diff --git a/test/e2e/catsrc_pod_config_e2e_test.go b/test/e2e/catsrc_pod_config_e2e_test.go new file mode 100644 index 0000000000..8d07b4700f --- /dev/null +++ b/test/e2e/catsrc_pod_config_e2e_test.go @@ -0,0 +1,198 @@ +package e2e + +import ( + "context" + "errors" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/operator-framework/api/pkg/operators/v1alpha1" + "github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8scontrollerclient "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" +) + +const catalogSourceLabel = "olm.catalogSource" + +var _ = By + +var _ = Describe("CatalogSource Grpc Pod Config", func() { + var ( + generatedNamespace corev1.Namespace + ) + + BeforeEach(func() { + generatedNamespace = SetupGeneratedTestNamespace(genName("catsrc-grpc-pod-config-e2e-")) + }) + + AfterEach(func() { + TeardownNamespace(generatedNamespace.GetName()) + }) + + When("the user wants more control over where the grpc catalog source pod gets scheduled", func() { + var ( + client k8scontrollerclient.Client + catalogSource *v1alpha1.CatalogSource + defaultNodeSelector = map[string]string{ + "kubernetes.io/os": "linux", + } + defaultTolerations []corev1.Toleration = nil + catalogSourceName = "test-catsrc" + defaultPriorityClassName = "" + ) + + BeforeEach(func() { + client = ctx.Ctx().Client() + + // must be a grpc source type with spec.image defined + catalogSource = &v1alpha1.CatalogSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: catalogSourceName, + Namespace: generatedNamespace.GetName(), + }, + Spec: v1alpha1.CatalogSourceSpec{ + SourceType: v1alpha1.SourceTypeGrpc, + Image: "repo/image:tag", + }, + } + }) + + AfterEach(func() { + // assume the catalog source was created and just delete it + _ = client.Delete(context.TODO(), catalogSource) + + // wait for it to go away + Expect(waitForDelete(func() error { + return client.Get(context.TODO(), k8scontrollerclient.ObjectKey{ + Name: catalogSource.GetName(), + Namespace: catalogSource.GetNamespace(), + }, &v1alpha1.CatalogSource{}) + })).To(BeNil()) + }) + + It("should override the pod's spec.priorityClassName", func() { + var overridenPriorityClassName = "system-node-critical" + + // create catalog source + catalogSource.Spec.GrpcPodConfig = &v1alpha1.GrpcPodConfig{ + PriorityClassName: &overridenPriorityClassName, + } + mustCreateCatalogSource(client, catalogSource) + + // Check overrides are present in the spec + catalogSourcePod := mustGetCatalogSourcePod(client, catalogSource) + Expect(catalogSourcePod).ToNot(BeNil()) + Expect(catalogSourcePod.Spec.NodeSelector).To(BeEquivalentTo(defaultNodeSelector)) + Expect(catalogSourcePod.Spec.Tolerations).To(ContainElements(defaultTolerations)) + Expect(catalogSourcePod.Spec.PriorityClassName).To(Equal(overridenPriorityClassName)) + }) + + It("should override the pod's spec.priorityClassName when it is empty", func() { + var overridenPriorityClassName = "" + + // create catalog source + catalogSource.Spec.GrpcPodConfig = &v1alpha1.GrpcPodConfig{ + PriorityClassName: &overridenPriorityClassName, + } + mustCreateCatalogSource(client, catalogSource) + + // Check overrides are present in the spec + catalogSourcePod := mustGetCatalogSourcePod(client, catalogSource) + Expect(catalogSourcePod).ToNot(BeNil()) + Expect(catalogSourcePod.Spec.NodeSelector).To(BeEquivalentTo(defaultNodeSelector)) + Expect(catalogSourcePod.Spec.Tolerations).To(ContainElements(defaultTolerations)) + Expect(catalogSourcePod.Spec.PriorityClassName).To(Equal(overridenPriorityClassName)) + }) + + It("should override the pod's spec.nodeSelector", func() { + var overridenNodeSelector = map[string]string{ + "kubernetes.io/os": "linux", + "some": "tag", + } + + // create catalog source + catalogSource.Spec.GrpcPodConfig = &v1alpha1.GrpcPodConfig{ + NodeSelector: overridenNodeSelector, + } + mustCreateCatalogSource(client, catalogSource) + + // Check overrides are present in the spec + catalogSourcePod := mustGetCatalogSourcePod(client, catalogSource) + Expect(catalogSourcePod).ToNot(BeNil()) + Expect(catalogSourcePod.Spec.NodeSelector).To(BeEquivalentTo(overridenNodeSelector)) + Expect(catalogSourcePod.Spec.Tolerations).To(ContainElements(defaultTolerations)) + Expect(catalogSourcePod.Spec.PriorityClassName).To(Equal(defaultPriorityClassName)) + }) + + It("should override the pod's spec.tolerations", func() { + var tolerationSeconds int64 = 120 + var overriddenTolerations = []corev1.Toleration{ + { + Key: "some/key", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoExecute, + TolerationSeconds: &tolerationSeconds, + }, + { + Key: "someother/key", + Operator: corev1.TolerationOpEqual, + Effect: corev1.TaintEffectNoSchedule, + }, + } + + // create catalog source + catalogSource.Spec.GrpcPodConfig = &v1alpha1.GrpcPodConfig{ + Tolerations: overriddenTolerations, + } + mustCreateCatalogSource(client, catalogSource) + + // Check overrides are present in the spec + catalogSourcePod := mustGetCatalogSourcePod(client, catalogSource) + Expect(catalogSourcePod).ToNot(BeNil()) + Expect(catalogSourcePod.Spec.NodeSelector).To(BeEquivalentTo(defaultNodeSelector)) + Expect(catalogSourcePod.Spec.Tolerations).To(ContainElements(overriddenTolerations)) + Expect(catalogSourcePod.Spec.PriorityClassName).To(Equal(defaultPriorityClassName)) + }) + }) +}) + +func mustGetCatalogSourcePod(client k8scontrollerclient.Client, catalogSource *v1alpha1.CatalogSource) *corev1.Pod { + var podList = corev1.PodList{} + + var opts = []k8scontrollerclient.ListOption{ + k8scontrollerclient.InNamespace(catalogSource.GetNamespace()), + // NOTE: this will fail if we stop setting the label on the catalog source pod + k8scontrollerclient.MatchingLabels{ + catalogSourceLabel: catalogSource.GetName(), + }, + } + + // Try to get a pod until its found and there's only one of them + Eventually(func() error { + if err := client.List(context.TODO(), &podList, opts...); err != nil { + return err + } + if len(podList.Items) != 1 { + return errors.New(fmt.Sprintf("expecting one catalog source pod but found %d", len(podList.Items))) + } + return nil + }).Should(BeNil()) + + return &podList.Items[0] +} + +func mustCreateCatalogSource(client k8scontrollerclient.Client, catalogSource *v1alpha1.CatalogSource) { + // create the object + Expect(client.Create(context.TODO(), catalogSource)).To(BeNil()) + + // wait for object to be appear + Eventually(func() error { + return client.Get(context.TODO(), k8scontrollerclient.ObjectKey{ + Name: catalogSource.Name, + Namespace: catalogSource.GetNamespace(), + }, &v1alpha1.CatalogSource{}) + }).Should(BeNil()) +} diff --git a/test/e2e/ctx/provisioner_kind.go b/test/e2e/ctx/provisioner_kind.go index 9f305fa6a8..f9f7971ffc 100644 --- a/test/e2e/ctx/provisioner_kind.go +++ b/test/e2e/ctx/provisioner_kind.go @@ -83,6 +83,7 @@ func Provision(ctx *TestContext) (func(), error) { } kubeconfigPath := filepath.Join(dir, "kubeconfig") + ctx.Logf("e2e cluster kubeconfig: %s\n", kubeconfigPath) provider := cluster.NewProvider( cluster.ProviderWithLogger(kindLogAdapter{ctx}), )