Skip to content

Commit c0c61fe

Browse files
stevekuznetsovstevekuznetsov
authored
OCPBUGS-17157: *: filter informers when preconditions are met (#3021)
* *: filter informers when preconditions are met When we can detect at startup time that all of the objects we're about to look at have the labels we're expecting, we can filter our informer factories upfront. Signed-off-by: Steve Kuznetsov <[email protected]> * test/e2e: improvements Signed-off-by: Steve Kuznetsov <[email protected]> --------- Signed-off-by: Steve Kuznetsov <[email protected]>
1 parent c55c24d commit c0c61fe

File tree

6 files changed

+201
-51
lines changed

6 files changed

+201
-51
lines changed

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ require (
3737
github.com/spf13/pflag v1.0.5
3838
github.com/stretchr/testify v1.8.3
3939
golang.org/x/net v0.10.0
40+
golang.org/x/sync v0.2.0
4041
golang.org/x/time v0.3.0
4142
google.golang.org/grpc v1.54.0
4243
gopkg.in/yaml.v2 v2.4.0
@@ -208,7 +209,6 @@ require (
208209
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
209210
golang.org/x/mod v0.10.0 // indirect
210211
golang.org/x/oauth2 v0.5.0 // indirect
211-
golang.org/x/sync v0.2.0 // indirect
212212
golang.org/x/sys v0.8.0 // indirect
213213
golang.org/x/term v0.8.0 // indirect
214214
golang.org/x/text v0.9.0 // indirect

‎pkg/controller/operators/catalog/operator.go

+39-39
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
"sync"
1212
"time"
1313

14-
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/internal/alongside"
1514
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/labeller"
1615
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/validatingroundtripper"
1716
errorwrap "github.com/pkg/errors"
@@ -187,6 +186,11 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
187186
return nil, err
188187
}
189188

189+
canFilter, err := labeller.Validate(ctx, logger, metadataClient)
190+
if err != nil {
191+
return nil, err
192+
}
193+
190194
// Allocate the new instance of an Operator.
191195
op := &Operator{
192196
Operator: queueOperator,
@@ -363,7 +367,14 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
363367
}
364368

365369
// Wire k8s sharedIndexInformers
366-
k8sInformerFactory := informers.NewSharedInformerFactoryWithOptions(op.opClient.KubernetesInterface(), resyncPeriod())
370+
k8sInformerFactory := informers.NewSharedInformerFactoryWithOptions(op.opClient.KubernetesInterface(), resyncPeriod(), func() []informers.SharedInformerOption {
371+
if !canFilter {
372+
return nil
373+
}
374+
return []informers.SharedInformerOption{informers.WithTweakListOptions(func(options *metav1.ListOptions) {
375+
options.LabelSelector = labels.SelectorFromSet(labels.Set{install.OLMManagedLabelKey: install.OLMManagedLabelValue}).String()
376+
})}
377+
}()...)
367378
sharedIndexInformers := []cache.SharedIndexInformer{}
368379

369380
// Wire Roles
@@ -372,6 +383,9 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
372383
sharedIndexInformers = append(sharedIndexInformers, roleInformer.Informer())
373384

374385
labelObjects := func(gvr schema.GroupVersionResource, informer cache.SharedIndexInformer, sync queueinformer.LegacySyncHandler) error {
386+
if canFilter {
387+
return nil
388+
}
375389
op.k8sLabelQueueSets[gvr] = workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{
376390
Name: gvr.String(),
377391
})
@@ -392,8 +406,9 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
392406
return nil
393407
}
394408

395-
if err := labelObjects(rbacv1.SchemeGroupVersion.WithResource("roles"), roleInformer.Informer(), labeller.ObjectLabeler[*rbacv1.Role, *rbacv1applyconfigurations.RoleApplyConfiguration](
396-
ctx, op.logger, labeller.HasOLMOwnerRef,
409+
rolesgvk := rbacv1.SchemeGroupVersion.WithResource("roles")
410+
if err := labelObjects(rolesgvk, roleInformer.Informer(), labeller.ObjectLabeler[*rbacv1.Role, *rbacv1applyconfigurations.RoleApplyConfiguration](
411+
ctx, op.logger, labeller.Filter(rolesgvk),
397412
rbacv1applyconfigurations.Role,
398413
func(namespace string, ctx context.Context, cfg *rbacv1applyconfigurations.RoleApplyConfiguration, opts metav1.ApplyOptions) (*rbacv1.Role, error) {
399414
return op.opClient.KubernetesInterface().RbacV1().Roles(namespace).Apply(ctx, cfg, opts)
@@ -407,8 +422,9 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
407422
op.lister.RbacV1().RegisterRoleBindingLister(metav1.NamespaceAll, roleBindingInformer.Lister())
408423
sharedIndexInformers = append(sharedIndexInformers, roleBindingInformer.Informer())
409424

410-
if err := labelObjects(rbacv1.SchemeGroupVersion.WithResource("rolebindings"), roleBindingInformer.Informer(), labeller.ObjectLabeler[*rbacv1.RoleBinding, *rbacv1applyconfigurations.RoleBindingApplyConfiguration](
411-
ctx, op.logger, labeller.HasOLMOwnerRef,
425+
rolebindingsgvk := rbacv1.SchemeGroupVersion.WithResource("rolebindings")
426+
if err := labelObjects(rolebindingsgvk, roleBindingInformer.Informer(), labeller.ObjectLabeler[*rbacv1.RoleBinding, *rbacv1applyconfigurations.RoleBindingApplyConfiguration](
427+
ctx, op.logger, labeller.Filter(rolebindingsgvk),
412428
rbacv1applyconfigurations.RoleBinding,
413429
func(namespace string, ctx context.Context, cfg *rbacv1applyconfigurations.RoleBindingApplyConfiguration, opts metav1.ApplyOptions) (*rbacv1.RoleBinding, error) {
414430
return op.opClient.KubernetesInterface().RbacV1().RoleBindings(namespace).Apply(ctx, cfg, opts)
@@ -422,10 +438,9 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
422438
op.lister.CoreV1().RegisterServiceAccountLister(metav1.NamespaceAll, serviceAccountInformer.Lister())
423439
sharedIndexInformers = append(sharedIndexInformers, serviceAccountInformer.Informer())
424440

425-
if err := labelObjects(corev1.SchemeGroupVersion.WithResource("serviceaccounts"), serviceAccountInformer.Informer(), labeller.ObjectLabeler[*corev1.ServiceAccount, *corev1applyconfigurations.ServiceAccountApplyConfiguration](
426-
ctx, op.logger, func(object metav1.Object) bool {
427-
return labeller.HasOLMOwnerRef(object) || labeller.HasOLMLabel(object)
428-
},
441+
serviceaccountsgvk := corev1.SchemeGroupVersion.WithResource("serviceaccounts")
442+
if err := labelObjects(serviceaccountsgvk, serviceAccountInformer.Informer(), labeller.ObjectLabeler[*corev1.ServiceAccount, *corev1applyconfigurations.ServiceAccountApplyConfiguration](
443+
ctx, op.logger, labeller.Filter(serviceaccountsgvk),
429444
corev1applyconfigurations.ServiceAccount,
430445
func(namespace string, ctx context.Context, cfg *corev1applyconfigurations.ServiceAccountApplyConfiguration, opts metav1.ApplyOptions) (*corev1.ServiceAccount, error) {
431446
return op.opClient.KubernetesInterface().CoreV1().ServiceAccounts(namespace).Apply(ctx, cfg, opts)
@@ -439,8 +454,9 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
439454
op.lister.CoreV1().RegisterServiceLister(metav1.NamespaceAll, serviceInformer.Lister())
440455
sharedIndexInformers = append(sharedIndexInformers, serviceInformer.Informer())
441456

442-
if err := labelObjects(corev1.SchemeGroupVersion.WithResource("services"), serviceInformer.Informer(), labeller.ObjectLabeler[*corev1.Service, *corev1applyconfigurations.ServiceApplyConfiguration](
443-
ctx, op.logger, labeller.HasOLMOwnerRef,
457+
servicesgvk := corev1.SchemeGroupVersion.WithResource("services")
458+
if err := labelObjects(servicesgvk, serviceInformer.Informer(), labeller.ObjectLabeler[*corev1.Service, *corev1applyconfigurations.ServiceApplyConfiguration](
459+
ctx, op.logger, labeller.Filter(servicesgvk),
444460
corev1applyconfigurations.Service,
445461
func(namespace string, ctx context.Context, cfg *corev1applyconfigurations.ServiceApplyConfiguration, opts metav1.ApplyOptions) (*corev1.Service, error) {
446462
return op.opClient.KubernetesInterface().CoreV1().Services(namespace).Apply(ctx, cfg, opts)
@@ -463,11 +479,9 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
463479
op.lister.CoreV1().RegisterPodLister(metav1.NamespaceAll, csPodInformer.Lister())
464480
sharedIndexInformers = append(sharedIndexInformers, csPodInformer.Informer())
465481

466-
if err := labelObjects(corev1.SchemeGroupVersion.WithResource("pods"), csPodInformer.Informer(), labeller.ObjectLabeler[*corev1.Pod, *corev1applyconfigurations.PodApplyConfiguration](
467-
ctx, op.logger, func(object metav1.Object) bool {
468-
_, ok := object.GetLabels()[reconciler.CatalogSourceLabelKey]
469-
return ok
470-
},
482+
podsgvk := corev1.SchemeGroupVersion.WithResource("pods")
483+
if err := labelObjects(podsgvk, csPodInformer.Informer(), labeller.ObjectLabeler[*corev1.Pod, *corev1applyconfigurations.PodApplyConfiguration](
484+
ctx, op.logger, labeller.Filter(podsgvk),
471485
corev1applyconfigurations.Pod,
472486
func(namespace string, ctx context.Context, cfg *corev1applyconfigurations.PodApplyConfiguration, opts metav1.ApplyOptions) (*corev1.Pod, error) {
473487
return op.opClient.KubernetesInterface().CoreV1().Pods(namespace).Apply(ctx, cfg, opts)
@@ -500,19 +514,11 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
500514
jobInformer := k8sInformerFactory.Batch().V1().Jobs()
501515
sharedIndexInformers = append(sharedIndexInformers, jobInformer.Informer())
502516

503-
if err := labelObjects(batchv1.SchemeGroupVersion.WithResource("jobs"), jobInformer.Informer(), labeller.ObjectLabeler[*batchv1.Job, *batchv1applyconfigurations.JobApplyConfiguration](
504-
ctx, op.logger, func(object metav1.Object) bool {
505-
for _, ownerRef := range object.GetOwnerReferences() {
506-
if ownerRef.APIVersion == corev1.SchemeGroupVersion.String() && ownerRef.Kind == "ConfigMap" {
507-
cm, err := configMapInformer.Lister().ConfigMaps(object.GetNamespace()).Get(ownerRef.Name)
508-
if err != nil {
509-
return false
510-
}
511-
return labeller.HasOLMOwnerRef(cm)
512-
}
513-
}
514-
return false
515-
},
517+
jobsgvk := batchv1.SchemeGroupVersion.WithResource("jobs")
518+
if err := labelObjects(jobsgvk, jobInformer.Informer(), labeller.ObjectLabeler[*batchv1.Job, *batchv1applyconfigurations.JobApplyConfiguration](
519+
ctx, op.logger, labeller.JobFilter(func(namespace, name string) (metav1.Object, error) {
520+
return configMapInformer.Lister().ConfigMaps(namespace).Get(name)
521+
}),
516522
batchv1applyconfigurations.Job,
517523
func(namespace string, ctx context.Context, cfg *batchv1applyconfigurations.JobApplyConfiguration, opts metav1.ApplyOptions) (*batchv1.Job, error) {
518524
return op.opClient.KubernetesInterface().BatchV1().Jobs(namespace).Apply(ctx, cfg, opts)
@@ -585,15 +591,9 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
585591
return nil, err
586592
}
587593

588-
if err := labelObjects(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions"), crdInformer, labeller.ObjectPatchLabeler(
589-
ctx, op.logger, func(object metav1.Object) bool {
590-
for key := range object.GetAnnotations() {
591-
if strings.HasPrefix(key, alongside.AnnotationPrefix) {
592-
return true
593-
}
594-
}
595-
return false
596-
},
594+
customresourcedefinitionsgvk := apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")
595+
if err := labelObjects(customresourcedefinitionsgvk, crdInformer, labeller.ObjectPatchLabeler(
596+
ctx, op.logger, labeller.Filter(customresourcedefinitionsgvk),
597597
op.opClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Patch,
598598
)); err != nil {
599599
return nil, err

‎pkg/controller/operators/labeller/filters.go

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
package labeller
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
"sync"
8+
9+
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry/reconciler"
10+
"github.com/sirupsen/logrus"
11+
"golang.org/x/sync/errgroup"
12+
appsv1 "k8s.io/api/apps/v1"
13+
batchv1 "k8s.io/api/batch/v1"
14+
corev1 "k8s.io/api/core/v1"
15+
rbacv1 "k8s.io/api/rbac/v1"
16+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
17+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
18+
"k8s.io/apimachinery/pkg/runtime/schema"
19+
"k8s.io/client-go/metadata"
20+
21+
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/internal/alongside"
22+
)
23+
24+
func Filter(gvr schema.GroupVersionResource) func(metav1.Object) bool {
25+
if f, ok := filters[gvr]; ok {
26+
return f
27+
}
28+
return func(object metav1.Object) bool {
29+
return false
30+
}
31+
}
32+
33+
func JobFilter(getConfigMap func(namespace, name string) (metav1.Object, error)) func(object metav1.Object) bool {
34+
return func(object metav1.Object) bool {
35+
for _, ownerRef := range object.GetOwnerReferences() {
36+
if ownerRef.APIVersion == corev1.SchemeGroupVersion.String() && ownerRef.Kind == "ConfigMap" {
37+
cm, err := getConfigMap(object.GetNamespace(), ownerRef.Name)
38+
if err != nil {
39+
return false
40+
}
41+
return HasOLMOwnerRef(cm)
42+
}
43+
}
44+
return false
45+
}
46+
}
47+
48+
var filters = map[schema.GroupVersionResource]func(metav1.Object) bool{
49+
corev1.SchemeGroupVersion.WithResource("services"): HasOLMOwnerRef,
50+
corev1.SchemeGroupVersion.WithResource("pods"): func(object metav1.Object) bool {
51+
_, ok := object.GetLabels()[reconciler.CatalogSourceLabelKey]
52+
return ok
53+
},
54+
corev1.SchemeGroupVersion.WithResource("serviceaccounts"): func(object metav1.Object) bool {
55+
return HasOLMOwnerRef(object) || HasOLMLabel(object)
56+
},
57+
appsv1.SchemeGroupVersion.WithResource("deployments"): HasOLMOwnerRef,
58+
rbacv1.SchemeGroupVersion.WithResource("roles"): HasOLMOwnerRef,
59+
rbacv1.SchemeGroupVersion.WithResource("rolebindings"): HasOLMOwnerRef,
60+
rbacv1.SchemeGroupVersion.WithResource("clusterroles"): HasOLMOwnerRef,
61+
rbacv1.SchemeGroupVersion.WithResource("clusterrolebindings"): HasOLMOwnerRef,
62+
apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions"): func(object metav1.Object) bool {
63+
for key := range object.GetAnnotations() {
64+
if strings.HasPrefix(key, alongside.AnnotationPrefix) {
65+
return true
66+
}
67+
}
68+
return false
69+
},
70+
}
71+
72+
func Validate(ctx context.Context, logger *logrus.Logger, metadataClient metadata.Interface) (bool, error) {
73+
okLock := sync.Mutex{}
74+
var ok bool
75+
g, ctx := errgroup.WithContext(ctx)
76+
allFilters := map[schema.GroupVersionResource]func(metav1.Object) bool{}
77+
for gvr, filter := range filters {
78+
allFilters[gvr] = filter
79+
}
80+
allFilters[batchv1.SchemeGroupVersion.WithResource("jobs")] = JobFilter(func(namespace, name string) (metav1.Object, error) {
81+
return metadataClient.Resource(corev1.SchemeGroupVersion.WithResource("configmaps")).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
82+
})
83+
for gvr, filter := range allFilters {
84+
gvr, filter := gvr, filter
85+
g.Go(func() error {
86+
list, err := metadataClient.Resource(gvr).List(ctx, metav1.ListOptions{})
87+
if err != nil {
88+
return fmt.Errorf("failed to list %s: %w", gvr.String(), err)
89+
}
90+
var count int
91+
for _, item := range list.Items {
92+
if filter(&item) && !hasLabel(&item) {
93+
count++
94+
}
95+
}
96+
if count > 0 {
97+
logger.WithFields(logrus.Fields{
98+
"gvr": gvr.String(),
99+
"nonconforming": count,
100+
}).Info("found nonconforming items")
101+
}
102+
okLock.Lock()
103+
ok = ok && count == 0
104+
okLock.Unlock()
105+
return nil
106+
})
107+
}
108+
if err := g.Wait(); err != nil {
109+
return false, err
110+
}
111+
return ok, nil
112+
}

‎pkg/controller/operators/labeller/labels.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,8 @@ import (
77
"strings"
88

99
jsonpatch "github.com/evanphx/json-patch"
10-
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
11-
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/queueinformer"
1210
"github.com/sirupsen/logrus"
1311
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
14-
1512
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1613
"k8s.io/apimachinery/pkg/runtime/schema"
1714
"k8s.io/apimachinery/pkg/types"
@@ -22,6 +19,8 @@ import (
2219

2320
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
2421
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/decorators"
22+
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
23+
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/queueinformer"
2524
)
2625

2726
type ApplyConfig[T any] interface {

0 commit comments

Comments
 (0)