Skip to content

Commit 97071af

Browse files
operators/olm: use a partial object metadata watch for copied CSVs
All we ever ned to know about copied CSVs is their metadata. No need to prune objects in memory, it's better to never allocate the memory to deserilize them in the first place. Signed-off-by: Steve Kuznetsov <[email protected]>
1 parent a7e3f3f commit 97071af

File tree

15 files changed

+559
-79
lines changed

15 files changed

+559
-79
lines changed

Diff for: cmd/olm/main.go

+6
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/sirupsen/logrus"
1515
"github.com/spf13/pflag"
1616
corev1 "k8s.io/api/core/v1"
17+
"k8s.io/client-go/metadata"
1718
"k8s.io/klog"
1819
ctrl "sigs.k8s.io/controller-runtime"
1920

@@ -154,6 +155,10 @@ func main() {
154155
if err != nil {
155156
logger.WithError(err).Fatal("error configuring custom resource client")
156157
}
158+
metadataClient, err := metadata.NewForConfig(config)
159+
if err != nil {
160+
logger.WithError(err).Fatal("error configuring metadata client")
161+
}
157162

158163
// Create a new instance of the operator.
159164
op, err := olm.NewOperator(
@@ -162,6 +167,7 @@ func main() {
162167
olm.WithWatchedNamespaces(namespaces...),
163168
olm.WithResyncPeriod(queueinformer.ResyncWithJitter(*wakeupInterval, 0.2)),
164169
olm.WithExternalClient(crClient),
170+
olm.WithMetadataClient(metadataClient),
165171
olm.WithOperatorClient(opClient),
166172
olm.WithRestConfig(config),
167173
olm.WithConfigClient(versionedConfigClient),

Diff for: go.mod

+3-3
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,11 @@ require (
4040
google.golang.org/grpc v1.53.0
4141
gopkg.in/yaml.v2 v2.4.0
4242
helm.sh/helm/v3 v3.12.2
43-
k8s.io/api v0.27.2
43+
k8s.io/api v0.27.4
4444
k8s.io/apiextensions-apiserver v0.27.2
45-
k8s.io/apimachinery v0.27.2
45+
k8s.io/apimachinery v0.27.4
4646
k8s.io/apiserver v0.27.2
47-
k8s.io/client-go v0.27.2
47+
k8s.io/client-go v0.27.4
4848
k8s.io/code-generator v0.27.2
4949
k8s.io/component-base v0.27.2
5050
k8s.io/klog v1.0.0

Diff for: go.sum

+7-7
Original file line numberDiff line numberDiff line change
@@ -736,7 +736,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
736736
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
737737
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
738738
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
739-
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
739+
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
740740
github.com/rubenv/sql-migrate v1.3.1 h1:Vx+n4Du8X8VTYuXbhNxdEUoh6wiJERA0GlWocR5FrbA=
741741
github.com/rubenv/sql-migrate v1.3.1/go.mod h1:YzG/Vh82CwyhTFXy+Mf5ahAiiEOpAlHurg+23VEzcsk=
742742
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -1343,18 +1343,18 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
13431343
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
13441344
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
13451345
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
1346-
k8s.io/api v0.27.2 h1:+H17AJpUMvl+clT+BPnKf0E3ksMAzoBBg7CntpSuADo=
1347-
k8s.io/api v0.27.2/go.mod h1:ENmbocXfBT2ADujUXcBhHV55RIT31IIEvkntP6vZKS4=
1346+
k8s.io/api v0.27.4 h1:0pCo/AN9hONazBKlNUdhQymmnfLRbSZjd5H5H3f0bSs=
1347+
k8s.io/api v0.27.4/go.mod h1:O3smaaX15NfxjzILfiln1D8Z3+gEYpjEpiNA/1EVK1Y=
13481348
k8s.io/apiextensions-apiserver v0.27.2 h1:iwhyoeS4xj9Y7v8YExhUwbVuBhMr3Q4bd/laClBV6Bo=
13491349
k8s.io/apiextensions-apiserver v0.27.2/go.mod h1:Oz9UdvGguL3ULgRdY9QMUzL2RZImotgxvGjdWRq6ZXQ=
1350-
k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg=
1351-
k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=
1350+
k8s.io/apimachinery v0.27.4 h1:CdxflD4AF61yewuid0fLl6bM4a3q04jWel0IlP+aYjs=
1351+
k8s.io/apimachinery v0.27.4/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=
13521352
k8s.io/apiserver v0.27.2 h1:p+tjwrcQEZDrEorCZV2/qE8osGTINPuS5ZNqWAvKm5E=
13531353
k8s.io/apiserver v0.27.2/go.mod h1:EsOf39d75rMivgvvwjJ3OW/u9n1/BmUMK5otEOJrb1Y=
13541354
k8s.io/cli-runtime v0.27.2 h1:9HI8gfReNujKXt16tGOAnb8b4NZ5E+e0mQQHKhFGwYw=
13551355
k8s.io/cli-runtime v0.27.2/go.mod h1:9UecpyPDTkhiYY4d9htzRqN+rKomJgyb4wi0OfrmCjw=
1356-
k8s.io/client-go v0.27.2 h1:vDLSeuYvCHKeoQRhCXjxXO45nHVv2Ip4Fe0MfioMrhE=
1357-
k8s.io/client-go v0.27.2/go.mod h1:tY0gVmUsHrAmjzHX9zs7eCjxcBsf8IiNe7KQ52biTcQ=
1356+
k8s.io/client-go v0.27.4 h1:vj2YTtSJ6J4KxaC88P4pMPEQECWMY8gqPqsTgUKzvjk=
1357+
k8s.io/client-go v0.27.4/go.mod h1:ragcly7lUlN0SRPk5/ZkGnDjPknzb37TICq07WhI6Xc=
13581358
k8s.io/code-generator v0.27.2 h1:RmK0CnU5qRaK6WRtSyWNODmfTZNoJbrizpVcsgbtrvI=
13591359
k8s.io/code-generator v0.27.2/go.mod h1:DPung1sI5vBgn4AGKtlPRQAyagj/ir/4jI55ipZHVww=
13601360
k8s.io/component-base v0.27.2 h1:neju+7s/r5O4x4/txeUONNTS9r1HsPbyoPBAtHsDCpo=

Diff for: pkg/controller/operators/olm/config.go

+8
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"time"
66

77
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/queueinformer"
8+
"k8s.io/client-go/metadata"
89

910
"github.com/pkg/errors"
1011
"github.com/sirupsen/logrus"
@@ -29,6 +30,7 @@ type operatorConfig struct {
2930
clock utilclock.Clock
3031
logger *logrus.Logger
3132
operatorClient operatorclient.ClientInterface
33+
metadataClient metadata.Interface
3234
externalClient versioned.Interface
3335
strategyResolver install.StrategyResolverInterface
3436
apiReconciler APIIntersectionReconciler
@@ -159,6 +161,12 @@ func WithOperatorClient(operatorClient operatorclient.ClientInterface) OperatorO
159161
}
160162
}
161163

164+
func WithMetadataClient(metadataClient metadata.Interface) OperatorOption {
165+
return func(config *operatorConfig) {
166+
config.metadataClient = metadataClient
167+
}
168+
}
169+
162170
func WithExternalClient(externalClient versioned.Interface) OperatorOption {
163171
return func(config *operatorConfig) {
164172
config.externalClient = externalClient

Diff for: pkg/controller/operators/olm/operator.go

+25-44
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ import (
2424
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
2525
"k8s.io/client-go/informers"
2626
k8sscheme "k8s.io/client-go/kubernetes/scheme"
27+
"k8s.io/client-go/metadata/metadatainformer"
28+
"k8s.io/client-go/metadata/metadatalister"
2729
"k8s.io/client-go/tools/cache"
2830
"k8s.io/client-go/tools/record"
2931
"k8s.io/client-go/util/workqueue"
@@ -35,12 +37,10 @@ import (
3537
"github.com/operator-framework/api/pkg/operators/v1alpha1"
3638
"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned"
3739
"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/informers/externalversions"
38-
operatorsv1alpha1listers "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/listers/operators/v1alpha1"
3940
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/certs"
4041
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
41-
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/internal/pruning"
4242
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm/overrides"
43-
resolver "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry/resolver"
43+
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry/resolver"
4444
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/clients"
4545
csvutility "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/csv"
4646
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/event"
@@ -75,7 +75,7 @@ type Operator struct {
7575
client versioned.Interface
7676
lister operatorlister.OperatorLister
7777
protectedCopiedCSVNamespaces map[string]struct{}
78-
copiedCSVLister operatorsv1alpha1listers.ClusterServiceVersionLister
78+
copiedCSVLister metadatalister.Lister
7979
ogQueueSet *queueinformer.ResourceQueueSet
8080
csvQueueSet *queueinformer.ResourceQueueSet
8181
olmConfigQueue workqueue.RateLimitingInterface
@@ -211,51 +211,28 @@ func newOperatorWithConfig(ctx context.Context, config *operatorConfig) (*Operat
211211
// A separate informer solely for CSV copies. Fields
212212
// are pruned from local copies of the objects managed
213213
// by this informer in order to reduce cached size.
214-
copiedCSVInformer := cache.NewSharedIndexInformer(
215-
pruning.NewListerWatcher(
216-
op.client,
217-
namespace,
218-
func(opts *metav1.ListOptions) {
219-
opts.LabelSelector = v1alpha1.CopiedLabelKey
220-
},
221-
pruning.PrunerFunc(func(csv *v1alpha1.ClusterServiceVersion) {
222-
nonstatus, status := copyableCSVHash(csv)
223-
*csv = v1alpha1.ClusterServiceVersion{
224-
TypeMeta: csv.TypeMeta,
225-
ObjectMeta: csv.ObjectMeta,
226-
Status: v1alpha1.ClusterServiceVersionStatus{
227-
Phase: csv.Status.Phase,
228-
Reason: csv.Status.Reason,
229-
},
230-
}
231-
if csv.Annotations == nil {
232-
csv.Annotations = make(map[string]string, 2)
233-
}
234-
// These annotation keys are
235-
// intentionally invalid -- all writes
236-
// to copied CSVs are regenerated from
237-
// the corresponding non-copied CSV,
238-
// so it should never be transmitted
239-
// back to the API server.
240-
csv.Annotations["$copyhash-nonstatus"] = nonstatus
241-
csv.Annotations["$copyhash-status"] = status
242-
}),
243-
),
244-
&v1alpha1.ClusterServiceVersion{},
214+
gvr := v1alpha1.SchemeGroupVersion.WithResource("clusterserviceversions")
215+
copiedCSVInformer := metadatainformer.NewFilteredMetadataInformer(
216+
config.metadataClient,
217+
gvr,
218+
namespace,
245219
config.resyncPeriod(),
246220
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
221+
func(options *metav1.ListOptions) {
222+
options.LabelSelector = v1alpha1.CopiedLabelKey
223+
},
247224
)
248-
op.copiedCSVLister = operatorsv1alpha1listers.NewClusterServiceVersionLister(copiedCSVInformer.GetIndexer())
225+
op.copiedCSVLister = metadatalister.New(copiedCSVInformer.Informer().GetIndexer(), gvr)
249226

250227
// Register separate queue for gcing copied csvs
251228
copiedCSVGCQueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("%s/csv-gc", namespace))
252229
op.copiedCSVGCQueueSet.Set(namespace, copiedCSVGCQueue)
253230
copiedCSVGCQueueInformer, err := queueinformer.NewQueueInformer(
254231
ctx,
255-
queueinformer.WithInformer(copiedCSVInformer),
232+
queueinformer.WithInformer(copiedCSVInformer.Informer()),
256233
queueinformer.WithLogger(op.logger),
257234
queueinformer.WithQueue(copiedCSVGCQueue),
258-
queueinformer.WithIndexer(copiedCSVInformer.GetIndexer()),
235+
queueinformer.WithIndexer(copiedCSVInformer.Informer().GetIndexer()),
259236
queueinformer.WithSyncer(queueinformer.LegacySyncHandler(op.syncGcCsv).ToSyncer()),
260237
)
261238
if err != nil {
@@ -1195,17 +1172,16 @@ func (a *Operator) handleClusterServiceVersionDeletion(obj interface{}) {
11951172
}
11961173
}
11971174

1198-
func (a *Operator) removeDanglingChildCSVs(csv *v1alpha1.ClusterServiceVersion) error {
1175+
func (a *Operator) removeDanglingChildCSVs(csv *metav1.PartialObjectMetadata) error {
11991176
logger := a.logger.WithFields(logrus.Fields{
12001177
"id": queueinformer.NewLoopID(),
12011178
"csv": csv.GetName(),
12021179
"namespace": csv.GetNamespace(),
1203-
"phase": csv.Status.Phase,
12041180
"labels": csv.GetLabels(),
12051181
"annotations": csv.GetAnnotations(),
12061182
})
12071183

1208-
if !csv.IsCopied() {
1184+
if !IsCopied(csv) {
12091185
logger.Warning("removeDanglingChild called on a parent. this is a no-op but should be avoided.")
12101186
return nil
12111187
}
@@ -1244,7 +1220,7 @@ func (a *Operator) removeDanglingChildCSVs(csv *v1alpha1.ClusterServiceVersion)
12441220
return nil
12451221
}
12461222

1247-
func (a *Operator) deleteChild(csv *v1alpha1.ClusterServiceVersion, logger *logrus.Entry) error {
1223+
func (a *Operator) deleteChild(csv *metav1.PartialObjectMetadata, logger *logrus.Entry) error {
12481224
logger.Debug("gcing csv")
12491225
return a.client.OperatorsV1alpha1().ClusterServiceVersions(csv.GetNamespace()).Delete(context.TODO(), csv.GetName(), metav1.DeleteOptions{})
12501226
}
@@ -1683,18 +1659,23 @@ func (a *Operator) createCSVCopyingDisabledEvent(csv *v1alpha1.ClusterServiceVer
16831659
}
16841660

16851661
func (a *Operator) syncGcCsv(obj interface{}) (syncError error) {
1686-
clusterServiceVersion, ok := obj.(*v1alpha1.ClusterServiceVersion)
1662+
clusterServiceVersion, ok := obj.(*metav1.PartialObjectMetadata)
16871663
if !ok {
16881664
a.logger.Debugf("wrong type: %#v", obj)
16891665
return fmt.Errorf("casting ClusterServiceVersion failed")
16901666
}
1691-
if clusterServiceVersion.IsCopied() {
1667+
if IsCopied(clusterServiceVersion) {
16921668
syncError = a.removeDanglingChildCSVs(clusterServiceVersion)
16931669
return
16941670
}
16951671
return
16961672
}
16971673

1674+
func IsCopied(o metav1.Object) bool {
1675+
_, ok := o.GetLabels()[v1alpha1.CopiedLabelKey]
1676+
return ok
1677+
}
1678+
16981679
// operatorGroupFromAnnotations returns the OperatorGroup for the CSV only if the CSV is active one in the group
16991680
func (a *Operator) operatorGroupFromAnnotations(logger *logrus.Entry, csv *v1alpha1.ClusterServiceVersion) *operatorsv1.OperatorGroup {
17001681
annotations := csv.GetAnnotations()

Diff for: pkg/controller/operators/olm/operatorgroup.go

+12-11
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
corev1 "k8s.io/api/core/v1"
1313
rbacv1 "k8s.io/api/rbac/v1"
1414
apierrors "k8s.io/apimachinery/pkg/api/errors"
15-
meta "k8s.io/apimachinery/pkg/api/meta"
15+
"k8s.io/apimachinery/pkg/api/meta"
1616
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1717
"k8s.io/apimachinery/pkg/labels"
1818
"k8s.io/apimachinery/pkg/util/errors"
@@ -797,7 +797,7 @@ func (a *Operator) copyToNamespace(prototype *v1alpha1.ClusterServiceVersion, ns
797797
prototype.ResourceVersion = ""
798798
prototype.UID = ""
799799

800-
existing, err := a.copiedCSVLister.ClusterServiceVersions(nsTo).Get(prototype.GetName())
800+
existing, err := a.copiedCSVLister.Namespace(nsTo).Get(prototype.GetName())
801801
if apierrors.IsNotFound(err) {
802802
created, err := a.client.OperatorsV1alpha1().ClusterServiceVersions(nsTo).Create(context.TODO(), prototype, metav1.CreateOptions{})
803803
if err != nil {
@@ -824,38 +824,39 @@ func (a *Operator) copyToNamespace(prototype *v1alpha1.ClusterServiceVersion, ns
824824
existingNonStatus := existing.Annotations["$copyhash-nonstatus"]
825825
existingStatus := existing.Annotations["$copyhash-status"]
826826

827+
var updated *v1alpha1.ClusterServiceVersion
827828
if existingNonStatus != nonstatus {
828-
if existing, err = a.client.OperatorsV1alpha1().ClusterServiceVersions(nsTo).Update(context.TODO(), prototype, metav1.UpdateOptions{}); err != nil {
829+
if updated, err = a.client.OperatorsV1alpha1().ClusterServiceVersions(nsTo).Update(context.TODO(), prototype, metav1.UpdateOptions{}); err != nil {
829830
return nil, err
830831
}
831832
} else {
832833
// Avoid mutating cached copied CSV.
833-
existing = prototype
834+
updated = prototype
834835
}
835836

836837
if existingStatus != status {
837-
existing.Status = prototype.Status
838-
if _, err = a.client.OperatorsV1alpha1().ClusterServiceVersions(nsTo).UpdateStatus(context.TODO(), existing, metav1.UpdateOptions{}); err != nil {
838+
updated.Status = prototype.Status
839+
if _, err = a.client.OperatorsV1alpha1().ClusterServiceVersions(nsTo).UpdateStatus(context.TODO(), updated, metav1.UpdateOptions{}); err != nil {
839840
return nil, err
840841
}
841842
}
842843
return &v1alpha1.ClusterServiceVersion{
843844
ObjectMeta: metav1.ObjectMeta{
844-
Name: existing.Name,
845-
Namespace: existing.Namespace,
846-
UID: existing.UID,
845+
Name: updated.Name,
846+
Namespace: updated.Namespace,
847+
UID: updated.UID,
847848
},
848849
}, nil
849850
}
850851

851852
func (a *Operator) pruneFromNamespace(operatorGroupName, namespace string) error {
852-
fetchedCSVs, err := a.copiedCSVLister.ClusterServiceVersions(namespace).List(labels.Everything())
853+
fetchedCSVs, err := a.copiedCSVLister.Namespace(namespace).List(labels.Everything())
853854
if err != nil {
854855
return err
855856
}
856857

857858
for _, csv := range fetchedCSVs {
858-
if csv.IsCopied() && csv.GetAnnotations()[operatorsv1.OperatorGroupAnnotationKey] == operatorGroupName {
859+
if IsCopied(csv) && csv.GetAnnotations()[operatorsv1.OperatorGroupAnnotationKey] == operatorGroupName {
859860
a.logger.Debugf("Found CSV '%v' in namespace %v to delete", csv.GetName(), namespace)
860861
if err := a.copiedCSVGCQueueSet.Requeue(csv.GetNamespace(), csv.GetName()); err != nil {
861862
return err

Diff for: vendor/k8s.io/apimachinery/pkg/runtime/converter.go

+2-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: vendor/k8s.io/apimachinery/pkg/util/wait/loop.go

+15-4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)