Skip to content

Commit 9566f98

Browse files
awgreeneawgreene
committed
Introduce protectedCopiedCSVNamespaces flag
Problem: Users rely on Copied CSVs in order to understand which operators are available in a given namespace. When installing All Namespace operators, a Copied CSV is created in every namespace which can place a huge performance strain on clusters with many namespaces. OLM introduced the ability to disable Copied CSVs for All Namespace mode operators in an effort to resolve the performance issues on large clusters, unfortunately removing the ability for users to identify which operators are available in a given namespace. Solution: The protectedCopiedCSVNamespaces runtime flag can be used to prevent Copied CSVs from being deleted even when Copied CSVs are disabled. An admin can then provide users with the proper RBAC to view which operators are running in All Namespace mode. Signed-off-by: Alexander Greene <[email protected]>
1 parent 83e3ebf commit 9566f98

File tree

4 files changed

+199
-88
lines changed

4 files changed

+199
-88
lines changed

Diff for: ‎cmd/olm/main.go

+4
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ var (
6060
tlsKeyPath = pflag.String(
6161
"tls-key", "", "Path to use for private key (requires tls-cert)")
6262

63+
protectedCopiedCSVNamespaces = pflag.String("protectedCopiedCSVNamespaces",
64+
"", "A comma-delimited set of namespaces where global Copied CSVs will always appear, even if Copied CSVs are disabled")
65+
6366
tlsCertPath = pflag.String(
6467
"tls-cert", "", "Path to use for certificate key (requires tls-key)")
6568

@@ -162,6 +165,7 @@ func main() {
162165
olm.WithOperatorClient(opClient),
163166
olm.WithRestConfig(config),
164167
olm.WithConfigClient(versionedConfigClient),
168+
olm.WithProtectedCopiedCSVNamespaces(*protectedCopiedCSVNamespaces),
165169
)
166170
if err != nil {
167171
logger.WithError(err).Fatal("error configuring operator")

Diff for: ‎pkg/controller/operators/olm/config.go

+31-20
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package olm
22

33
import (
4+
"strings"
45
"time"
56

67
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/queueinformer"
@@ -21,18 +22,19 @@ import (
2122
type OperatorOption func(*operatorConfig)
2223

2324
type operatorConfig struct {
24-
resyncPeriod func() time.Duration
25-
operatorNamespace string
26-
watchedNamespaces []string
27-
clock utilclock.Clock
28-
logger *logrus.Logger
29-
operatorClient operatorclient.ClientInterface
30-
externalClient versioned.Interface
31-
strategyResolver install.StrategyResolverInterface
32-
apiReconciler APIIntersectionReconciler
33-
apiLabeler labeler.Labeler
34-
restConfig *rest.Config
35-
configClient configv1client.Interface
25+
protectedCopiedCSVNamespaces map[string]struct{}
26+
resyncPeriod func() time.Duration
27+
operatorNamespace string
28+
watchedNamespaces []string
29+
clock utilclock.Clock
30+
logger *logrus.Logger
31+
operatorClient operatorclient.ClientInterface
32+
externalClient versioned.Interface
33+
strategyResolver install.StrategyResolverInterface
34+
apiReconciler APIIntersectionReconciler
35+
apiLabeler labeler.Labeler
36+
restConfig *rest.Config
37+
configClient configv1client.Interface
3638
}
3739

3840
func (o *operatorConfig) apply(options []OperatorOption) {
@@ -77,14 +79,15 @@ func (o *operatorConfig) validate() (err error) {
7779

7880
func defaultOperatorConfig() *operatorConfig {
7981
return &operatorConfig{
80-
resyncPeriod: queueinformer.ResyncWithJitter(30*time.Second, 0.2),
81-
operatorNamespace: "default",
82-
watchedNamespaces: []string{metav1.NamespaceAll},
83-
clock: utilclock.RealClock{},
84-
logger: logrus.New(),
85-
strategyResolver: &install.StrategyResolver{},
86-
apiReconciler: APIIntersectionReconcileFunc(ReconcileAPIIntersection),
87-
apiLabeler: labeler.Func(LabelSetsFor),
82+
resyncPeriod: queueinformer.ResyncWithJitter(30*time.Second, 0.2),
83+
operatorNamespace: "default",
84+
watchedNamespaces: []string{metav1.NamespaceAll},
85+
clock: utilclock.RealClock{},
86+
logger: logrus.New(),
87+
strategyResolver: &install.StrategyResolver{},
88+
apiReconciler: APIIntersectionReconcileFunc(ReconcileAPIIntersection),
89+
apiLabeler: labeler.Func(LabelSetsFor),
90+
protectedCopiedCSVNamespaces: map[string]struct{}{},
8891
}
8992
}
9093

@@ -112,6 +115,14 @@ func WithLogger(logger *logrus.Logger) OperatorOption {
112115
}
113116
}
114117

118+
func WithProtectedCopiedCSVNamespaces(namespaces string) OperatorOption {
119+
return func(config *operatorConfig) {
120+
for _, ns := range strings.Split(namespaces, ",") {
121+
config.protectedCopiedCSVNamespaces[ns] = struct{}{}
122+
}
123+
}
124+
}
125+
115126
func WithClock(clock utilclock.Clock) OperatorOption {
116127
return func(config *operatorConfig) {
117128
config.clock = clock

Diff for: ‎pkg/controller/operators/olm/operator.go

+119-63
Original file line numberDiff line numberDiff line change
@@ -63,32 +63,33 @@ var (
6363
type Operator struct {
6464
queueinformer.Operator
6565

66-
clock utilclock.Clock
67-
logger *logrus.Logger
68-
opClient operatorclient.ClientInterface
69-
client versioned.Interface
70-
lister operatorlister.OperatorLister
71-
copiedCSVLister operatorsv1alpha1listers.ClusterServiceVersionLister
72-
ogQueueSet *queueinformer.ResourceQueueSet
73-
csvQueueSet *queueinformer.ResourceQueueSet
74-
olmConfigQueue workqueue.RateLimitingInterface
75-
csvCopyQueueSet *queueinformer.ResourceQueueSet
76-
copiedCSVGCQueueSet *queueinformer.ResourceQueueSet
77-
objGCQueueSet *queueinformer.ResourceQueueSet
78-
nsQueueSet workqueue.RateLimitingInterface
79-
apiServiceQueue workqueue.RateLimitingInterface
80-
csvIndexers map[string]cache.Indexer
81-
recorder record.EventRecorder
82-
resolver install.StrategyResolverInterface
83-
apiReconciler APIIntersectionReconciler
84-
apiLabeler labeler.Labeler
85-
csvSetGenerator csvutility.SetGenerator
86-
csvReplaceFinder csvutility.ReplaceFinder
87-
csvNotification csvutility.WatchNotification
88-
serviceAccountSyncer *scoped.UserDefinedServiceAccountSyncer
89-
clientAttenuator *scoped.ClientAttenuator
90-
serviceAccountQuerier *scoped.UserDefinedServiceAccountQuerier
91-
clientFactory clients.Factory
66+
clock utilclock.Clock
67+
logger *logrus.Logger
68+
opClient operatorclient.ClientInterface
69+
client versioned.Interface
70+
lister operatorlister.OperatorLister
71+
protectedCopiedCSVNamespaces map[string]struct{}
72+
copiedCSVLister operatorsv1alpha1listers.ClusterServiceVersionLister
73+
ogQueueSet *queueinformer.ResourceQueueSet
74+
csvQueueSet *queueinformer.ResourceQueueSet
75+
olmConfigQueue workqueue.RateLimitingInterface
76+
csvCopyQueueSet *queueinformer.ResourceQueueSet
77+
copiedCSVGCQueueSet *queueinformer.ResourceQueueSet
78+
objGCQueueSet *queueinformer.ResourceQueueSet
79+
nsQueueSet workqueue.RateLimitingInterface
80+
apiServiceQueue workqueue.RateLimitingInterface
81+
csvIndexers map[string]cache.Indexer
82+
recorder record.EventRecorder
83+
resolver install.StrategyResolverInterface
84+
apiReconciler APIIntersectionReconciler
85+
apiLabeler labeler.Labeler
86+
csvSetGenerator csvutility.SetGenerator
87+
csvReplaceFinder csvutility.ReplaceFinder
88+
csvNotification csvutility.WatchNotification
89+
serviceAccountSyncer *scoped.UserDefinedServiceAccountSyncer
90+
clientAttenuator *scoped.ClientAttenuator
91+
serviceAccountQuerier *scoped.UserDefinedServiceAccountQuerier
92+
clientFactory clients.Factory
9293
}
9394

9495
func NewOperator(ctx context.Context, options ...OperatorOption) (*Operator, error) {
@@ -121,30 +122,31 @@ func newOperatorWithConfig(ctx context.Context, config *operatorConfig) (*Operat
121122
}
122123

123124
op := &Operator{
124-
Operator: queueOperator,
125-
clock: config.clock,
126-
logger: config.logger,
127-
opClient: config.operatorClient,
128-
client: config.externalClient,
129-
ogQueueSet: queueinformer.NewEmptyResourceQueueSet(),
130-
csvQueueSet: queueinformer.NewEmptyResourceQueueSet(),
131-
olmConfigQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "olmConfig"),
132-
csvCopyQueueSet: queueinformer.NewEmptyResourceQueueSet(),
133-
copiedCSVGCQueueSet: queueinformer.NewEmptyResourceQueueSet(),
134-
objGCQueueSet: queueinformer.NewEmptyResourceQueueSet(),
135-
apiServiceQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "apiservice"),
136-
resolver: config.strategyResolver,
137-
apiReconciler: config.apiReconciler,
138-
lister: lister,
139-
recorder: eventRecorder,
140-
apiLabeler: config.apiLabeler,
141-
csvIndexers: map[string]cache.Indexer{},
142-
csvSetGenerator: csvutility.NewSetGenerator(config.logger, lister),
143-
csvReplaceFinder: csvutility.NewReplaceFinder(config.logger, config.externalClient),
144-
serviceAccountSyncer: scoped.NewUserDefinedServiceAccountSyncer(config.logger, scheme, config.operatorClient, config.externalClient),
145-
clientAttenuator: scoped.NewClientAttenuator(config.logger, config.restConfig, config.operatorClient),
146-
serviceAccountQuerier: scoped.NewUserDefinedServiceAccountQuerier(config.logger, config.externalClient),
147-
clientFactory: clients.NewFactory(config.restConfig),
125+
Operator: queueOperator,
126+
clock: config.clock,
127+
logger: config.logger,
128+
opClient: config.operatorClient,
129+
client: config.externalClient,
130+
ogQueueSet: queueinformer.NewEmptyResourceQueueSet(),
131+
csvQueueSet: queueinformer.NewEmptyResourceQueueSet(),
132+
olmConfigQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "olmConfig"),
133+
csvCopyQueueSet: queueinformer.NewEmptyResourceQueueSet(),
134+
copiedCSVGCQueueSet: queueinformer.NewEmptyResourceQueueSet(),
135+
objGCQueueSet: queueinformer.NewEmptyResourceQueueSet(),
136+
apiServiceQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "apiservice"),
137+
resolver: config.strategyResolver,
138+
apiReconciler: config.apiReconciler,
139+
lister: lister,
140+
recorder: eventRecorder,
141+
apiLabeler: config.apiLabeler,
142+
csvIndexers: map[string]cache.Indexer{},
143+
csvSetGenerator: csvutility.NewSetGenerator(config.logger, lister),
144+
csvReplaceFinder: csvutility.NewReplaceFinder(config.logger, config.externalClient),
145+
serviceAccountSyncer: scoped.NewUserDefinedServiceAccountSyncer(config.logger, scheme, config.operatorClient, config.externalClient),
146+
clientAttenuator: scoped.NewClientAttenuator(config.logger, config.restConfig, config.operatorClient),
147+
serviceAccountQuerier: scoped.NewUserDefinedServiceAccountQuerier(config.logger, config.externalClient),
148+
clientFactory: clients.NewFactory(config.restConfig),
149+
protectedCopiedCSVNamespaces: config.protectedCopiedCSVNamespaces,
148150
}
149151

150152
// Set up syncing for namespace-scoped resources
@@ -1299,23 +1301,40 @@ func (a *Operator) syncOLMConfig(obj interface{}) (syncError error) {
12991301
return err
13001302
}
13011303

1302-
// Filter to unique copies
1303-
uniqueCopiedCSVs := map[string]struct{}{}
1304+
copiedCSVNamespaces := map[string]map[string]struct{}{}
13041305
for _, copiedCSV := range copiedCSVs {
1305-
uniqueCopiedCSVs[copiedCSV.GetName()] = struct{}{}
1306+
if _, ok := copiedCSVNamespaces[copiedCSV.GetName()]; !ok {
1307+
copiedCSVNamespaces[copiedCSV.GetName()] = map[string]struct{}{}
1308+
}
1309+
copiedCSVNamespaces[copiedCSV.GetName()][copiedCSV.GetNamespace()] = struct{}{}
13061310
}
13071311

13081312
csvs, err := a.lister.OperatorsV1alpha1().ClusterServiceVersionLister().ClusterServiceVersions(og.GetNamespace()).List(labels.NewSelector().Add(*nonCopiedCSVRequirement))
13091313
if err != nil {
13101314
return err
13111315
}
13121316

1317+
namespaces, err := a.lister.CoreV1().NamespaceLister().List(labels.Everything())
1318+
if err != nil {
1319+
return err
1320+
}
1321+
1322+
var copiedCSVEvaluator func(m map[string]struct{}) bool
1323+
if olmConfig.CopiedCSVsAreEnabled() {
1324+
copiedCSVEvaluator = func(m map[string]struct{}) bool {
1325+
return areCopiedCSVsAvailableInAllNamespaces(namespaces, m)
1326+
}
1327+
} else {
1328+
copiedCSVEvaluator = func(m map[string]struct{}) bool {
1329+
return areCopiedCSVsAvailableOnlyInProtectedNamespaces(a.protectedCopiedCSVNamespaces, m)
1330+
}
1331+
}
1332+
13131333
for _, csv := range csvs {
1314-
// If the correct number of copied CSVs were found, continue
1315-
if _, ok := uniqueCopiedCSVs[csv.GetName()]; ok == olmConfig.CopiedCSVsAreEnabled() {
1334+
// Ignore NS where actual CSV is installed
1335+
if copiedCSVEvaluator(copiedCSVNamespaces[csv.GetName()]) {
13161336
continue
13171337
}
1318-
13191338
if err := a.csvQueueSet.Requeue(csv.GetNamespace(), csv.GetName()); err != nil {
13201339
a.logger.WithError(err).Warn("unable to requeue")
13211340
}
@@ -1324,7 +1343,7 @@ func (a *Operator) syncOLMConfig(obj interface{}) (syncError error) {
13241343
}
13251344

13261345
// Update the olmConfig status if it has changed.
1327-
condition := getCopiedCSVsCondition(!olmConfig.CopiedCSVsAreEnabled(), csvIsRequeued)
1346+
condition := getCopiedCSVsCondition(olmConfig.CopiedCSVsAreEnabled(), csvIsRequeued)
13281347
if !isStatusConditionPresentAndAreTypeReasonMessageStatusEqual(olmConfig.Status.Conditions, condition) {
13291348
meta.SetStatusCondition(&olmConfig.Status.Conditions, condition)
13301349
if _, err := a.client.OperatorsV1().OLMConfigs().UpdateStatus(context.TODO(), olmConfig, metav1.UpdateOptions{}); err != nil {
@@ -1335,6 +1354,26 @@ func (a *Operator) syncOLMConfig(obj interface{}) (syncError error) {
13351354
return nil
13361355
}
13371356

1357+
func areCopiedCSVsAvailableInAllNamespaces(namespaces []*corev1.Namespace, m map[string]struct{}) bool {
1358+
// Start at -1 to account for the namespace the original CSV is in
1359+
expectedCopiedCSVCount := -1
1360+
for _, ns := range namespaces {
1361+
if ns.Status.Phase == corev1.NamespaceActive {
1362+
expectedCopiedCSVCount++
1363+
}
1364+
}
1365+
return len(m) == expectedCopiedCSVCount
1366+
}
1367+
1368+
func areCopiedCSVsAvailableOnlyInProtectedNamespaces(protectedNamespaces map[string]struct{}, m map[string]struct{}) bool {
1369+
for protectedNS := range protectedNamespaces {
1370+
if _, ok := m[protectedNS]; !ok {
1371+
return false
1372+
}
1373+
}
1374+
return true
1375+
}
1376+
13381377
func isStatusConditionPresentAndAreTypeReasonMessageStatusEqual(conditions []metav1.Condition, condition metav1.Condition) bool {
13391378
foundCondition := meta.FindStatusCondition(conditions, condition.Type)
13401379
if foundCondition == nil {
@@ -1346,13 +1385,13 @@ func isStatusConditionPresentAndAreTypeReasonMessageStatusEqual(conditions []met
13461385
foundCondition.Status == condition.Status
13471386
}
13481387

1349-
func getCopiedCSVsCondition(isDisabled, csvIsRequeued bool) metav1.Condition {
1388+
func getCopiedCSVsCondition(enabled, csvIsRequeued bool) metav1.Condition {
13501389
condition := metav1.Condition{
13511390
Type: operatorsv1.DisabledCopiedCSVsConditionType,
13521391
LastTransitionTime: metav1.Now(),
13531392
Status: metav1.ConditionFalse,
13541393
}
1355-
if !isDisabled {
1394+
if enabled {
13561395
condition.Reason = "CopiedCSVsEnabled"
13571396
condition.Message = "Copied CSVs are enabled and present across the cluster"
13581397
if csvIsRequeued {
@@ -1361,15 +1400,14 @@ func getCopiedCSVsCondition(isDisabled, csvIsRequeued bool) metav1.Condition {
13611400
return condition
13621401
}
13631402

1403+
condition.Reason = "CopiedCSVsDisabled"
13641404
if csvIsRequeued {
1365-
condition.Reason = "CopiedCSVsFound"
1366-
condition.Message = "Copied CSVs are disabled and at least one copied CSV was found for an operator installed in AllNamespace mode"
1405+
condition.Message = "Copied CSVs are disabled and at least one unexpected copied CSV was found for an operator installed in AllNamespace mode"
13671406
return condition
13681407
}
13691408

13701409
condition.Status = metav1.ConditionTrue
1371-
condition.Reason = "NoCopiedCSVsFound"
1372-
condition.Message = "Copied CSVs are disabled and none were found for operators installed in AllNamespace mode"
1410+
condition.Message = "Copied CSVs are disabled and no unexpected copied CSVs were found for operators installed in AllNamespace mode"
13731411

13741412
return condition
13751413
}
@@ -1444,7 +1482,25 @@ func (a *Operator) syncCopyCSV(obj interface{}) (syncError error) {
14441482
return err
14451483
}
14461484

1485+
// Ensure that the Copied CSVs exist in the protected namespaces.
1486+
protectedNamespaces := []string{}
1487+
for ns := range a.protectedCopiedCSVNamespaces {
1488+
if ns == clusterServiceVersion.GetNamespace() {
1489+
continue
1490+
}
1491+
protectedNamespaces = append(protectedNamespaces, ns)
1492+
}
1493+
1494+
if err := a.ensureCSVsInNamespaces(clusterServiceVersion, operatorGroup, NewNamespaceSet(protectedNamespaces)); err != nil {
1495+
logger.WithError(err).Info("couldn't copy CSV to protected Copied CSV namespaces")
1496+
syncError = err
1497+
}
1498+
1499+
// Delete Copied CSVs in namespaces that are not protected.
14471500
for _, copiedCSV := range copiedCSVs {
1501+
if _, ok := a.protectedCopiedCSVNamespaces[copiedCSV.Namespace]; ok {
1502+
continue
1503+
}
14481504
err := a.client.OperatorsV1alpha1().ClusterServiceVersions(copiedCSV.Namespace).Delete(context.TODO(), copiedCSV.Name, metav1.DeleteOptions{})
14491505
if err != nil && !apierrors.IsNotFound(err) {
14501506
return err

0 commit comments

Comments
 (0)