Apply Subscription.Spec.Config.Annotations to Deployments, Pods, ApiServices (#3169)

* Apply Subscription.Spec.Config.Annotations to deployments

Signed-off-by: Jacob Floyd <[email protected]>

* Apply Subscription.Spec.Config.Annotations to Deployment Pods

Signed-off-by: Jacob Floyd <[email protected]>

* Apply Subscription.Spec.Config.Annotations to ApiServices

Signed-off-by: Jacob Floyd <[email protected]>

* Refactor and Test InjectAnnotationsIntoDeployment

Signed-off-by: Jacob Floyd <[email protected]>

---------

Signed-off-by: Jacob Floyd <[email protected]>

Author: cognifloyd

deploy/chart/crds/0000_50_olm_00-subscriptions.crd.yaml

@@ -524,6 +524,11 @@ spec:
                                   topologyKey:
                                     description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
                                     type: string
+                    annotations:
+                      description: Annotations is an unstructured key value map stored with each Deployment, Pod, APIService in the Operator. Typically, annotations may be set by external tools to store and retrieve arbitrary metadata. Use this field to pre-define annotations that OLM should add to each of the Subscription's deployments, pods, and apiservices.
+                      type: object
+                      additionalProperties:
+                        type: string
                     env:
                       description: Env is a list of environment variables to set in the container. Cannot be updated.
                       type: array

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/onsi/gomega v1.29.0
 	github.com/openshift/api v3.9.0+incompatible
 	github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
-	github.com/operator-framework/api v0.21.0
+	github.com/operator-framework/api v0.22.0
 	github.com/operator-framework/operator-registry v1.35.0
 	github.com/otiai10/copy v1.14.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -672,8 +672,8 @@ github.com/openshift/api v0.0.0-20221021112143-4226c2167e40 h1:PxjGCA72RtsdHWToZ
 github.com/openshift/api v0.0.0-20221021112143-4226c2167e40/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A=
 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg=
 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE=
-github.com/operator-framework/api v0.21.0 h1:89LhqGTLskxpPR4siEaorkF9PY3KLI51S5mFxP6q1G8=
-github.com/operator-framework/api v0.21.0/go.mod h1:3tsDLxXChMY1KgxO5v1CZQogHNQCIMy14YXkXqA5lT4=
+github.com/operator-framework/api v0.22.0 h1:UZSn+iaQih4rCReezOnWTTJkMyawwV5iLnIItaOzytY=
+github.com/operator-framework/api v0.22.0/go.mod h1:p/7YDbr+n4fmESfZ47yLAV1SvkfE6NU2aX8KhcfI0GA=
 github.com/operator-framework/operator-registry v1.35.0 h1:BvytqLwhgb0QiAkEODEKXq3vc2vWiHQq0IlofvFA+OI=
 github.com/operator-framework/operator-registry v1.35.0/go.mod h1:foC+NO1V9JuDIOk3pjjlrPE0KVkq09m8oDVRz/a/nFA=
 github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=

pkg/controller/install/apiservice.go

@@ -8,6 +8,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
@@ -40,6 +41,16 @@ func (i *StrategyDeploymentInstaller) createOrUpdateAPIService(caPEM []byte, des
 			},
 		}
 		apiService.SetName(apiServiceName)
+
+		ownerSubscription, err := i.findOwnerSubscription()
+		if err != nil {
+			return err
+		} else if ownerSubscription == nil {
+			// This is not an error. For example, the PackageServer CSV in OLM is created without a Subscription.
+			logger.Debugf("failed to get the owner subscription csv=%s", i.owner.GetName())
+		} else if ownerSubscription.Spec.Config != nil {
+			apiService.SetAnnotations(ownerSubscription.Spec.Config.Annotations)
+		}
 	} else {
 		apiService = apiService.DeepCopy()
 		csv, ok := i.owner.(*v1alpha1.ClusterServiceVersion)
@@ -101,10 +112,10 @@ func IsAPIServiceAdoptable(opLister operatorlister.OperatorLister, target *v1alp
 		return
 	}
 
-	labels := apiService.GetLabels()
-	ownerKind := labels[ownerutil.OwnerKind]
-	ownerName := labels[ownerutil.OwnerKey]
-	ownerNamespace := labels[ownerutil.OwnerNamespaceKey]
+	apiServiceLabels := apiService.GetLabels()
+	ownerKind := apiServiceLabels[ownerutil.OwnerKind]
+	ownerName := apiServiceLabels[ownerutil.OwnerKey]
+	ownerNamespace := apiServiceLabels[ownerutil.OwnerNamespaceKey]
 
 	if ownerKind == "" || ownerNamespace == "" || ownerName == "" {
 		return
@@ -285,3 +296,20 @@ func legacyAPIServiceNameToServiceName(apiServiceName string) string {
 	// Replace all '.'s with "-"s to convert to a DNS-1035 label
 	return strings.Replace(apiServiceName, ".", "-", -1)
 }
+
+func (i *StrategyDeploymentInstaller) findOwnerSubscription() (*v1alpha1.Subscription, error) {
+	list, listErr := i.strategyClient.GetOpLister().OperatorsV1alpha1().SubscriptionLister().Subscriptions(i.owner.GetNamespace()).List(labels.Everything())
+	if listErr != nil {
+		err := fmt.Errorf("failed to list subscription namespace=%s - %v", i.owner.GetNamespace(), listErr)
+		return nil, err
+	}
+
+	for idx := range list {
+		sub := list[idx]
+		if sub.Status.InstalledCSV == i.owner.GetName() {
+			return sub, nil
+		}
+	}
+
+	return nil, nil
+}

pkg/controller/install/deployment.go

@@ -144,7 +144,7 @@ func (i *StrategyDeploymentInstaller) deploymentForSpec(name string, spec appsv1
 	for k, v := range dep.Spec.Template.GetAnnotations() {
 		annotations[k] = v
 	}
-	for k, v := range i.templateAnnotations {
+	for k, v := range i.templateAnnotations { // templateAnnotations comes from CSV.Annotations
 		annotations[k] = v
 	}
 	dep.Spec.Template.SetAnnotations(annotations)
@@ -159,6 +159,7 @@ func (i *StrategyDeploymentInstaller) deploymentForSpec(name string, spec appsv1
 	ownerutil.AddNonBlockingOwner(dep, i.owner)
 	ownerutil.AddOwnerLabelsForKind(dep, i.owner, v1alpha1.ClusterServiceVersionKind)
 
+	// Any admin-provided config (Subscription.Spec.Config) gets injected here.
 	if applyErr := i.initializers.Apply(dep); applyErr != nil {
 		err = applyErr
 		return

pkg/controller/operators/olm/overrides/config.go

@@ -16,7 +16,7 @@ type operatorConfig struct {
 	logger *logrus.Logger
 }
 
-func (o *operatorConfig) GetConfigOverrides(ownerCSV ownerutil.Owner) (envVarOverrides []corev1.EnvVar, volumeOverrides []corev1.Volume, volumeMountOverrides []corev1.VolumeMount, tolerationOverrides []corev1.Toleration, resourcesOverride *corev1.ResourceRequirements, nodeSelectorOverride map[string]string, affinity *corev1.Affinity, err error) {
+func (o *operatorConfig) GetConfigOverrides(ownerCSV ownerutil.Owner) (envVarOverrides []corev1.EnvVar, volumeOverrides []corev1.Volume, volumeMountOverrides []corev1.VolumeMount, tolerationOverrides []corev1.Toleration, resourcesOverride *corev1.ResourceRequirements, nodeSelectorOverride map[string]string, affinity *corev1.Affinity, annotations map[string]string, err error) {
 	list, listErr := o.lister.OperatorsV1alpha1().SubscriptionLister().Subscriptions(ownerCSV.GetNamespace()).List(labels.Everything())
 	if listErr != nil {
 		err = fmt.Errorf("failed to list subscription namespace=%s - %v", ownerCSV.GetNamespace(), listErr)
@@ -41,6 +41,7 @@ func (o *operatorConfig) GetConfigOverrides(ownerCSV ownerutil.Owner) (envVarOve
 	resourcesOverride = owner.Spec.Config.Resources
 	nodeSelectorOverride = owner.Spec.Config.NodeSelector
 	affinity = owner.Spec.Config.Affinity
+	annotations = owner.Spec.Config.Annotations
 
 	return
 }

pkg/controller/operators/olm/overrides/initializer.go

@@ -45,7 +45,7 @@ func (d *DeploymentInitializer) initialize(ownerCSV ownerutil.Owner, deployment
 	var envVarOverrides, proxyEnvVar, merged []corev1.EnvVar
 	var err error
 
-	envVarOverrides, volumeOverrides, volumeMountOverrides, tolerationOverrides, resourcesOverride, nodeSelectorOverride, affinity, err := d.config.GetConfigOverrides(ownerCSV)
+	envVarOverrides, volumeOverrides, volumeMountOverrides, tolerationOverrides, resourcesOverride, nodeSelectorOverride, affinity, annotations, err := d.config.GetConfigOverrides(ownerCSV)
 	if err != nil {
 		err = fmt.Errorf("failed to get subscription pod configuration - %v", err)
 		return err
@@ -96,6 +96,10 @@ func (d *DeploymentInitializer) initialize(ownerCSV ownerutil.Owner, deployment
 		return fmt.Errorf("failed to inject affinity into deployment spec name=%s - %s", deployment.Name, err)
 	}
 
+	if err = inject.InjectAnnotationsIntoDeployment(deployment, annotations); err != nil {
+		return fmt.Errorf("failed to inject annotations into deployment spec name=%s - %s", deployment.Name, err)
+	}
+
 	return nil
 }
 

pkg/controller/operators/olm/overrides/inject/inject.go

@@ -4,14 +4,15 @@ import (
 	"errors"
 	"reflect"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
 // InjectEnvIntoDeployment injects the proxy env variables specified in
 // proxyEnvVar into the container(s) of the given PodSpec.
 //
 // If any Container in PodSpec already defines an env variable of the same name
-// as any of the proxy env variables then it
+// as any of the proxy env variables then it will be overwritten.
 func InjectEnvIntoDeployment(podSpec *corev1.PodSpec, envVars []corev1.EnvVar) error {
 	if podSpec == nil {
 		return errors.New("no pod spec provided")
@@ -217,8 +218,7 @@ func InjectResourcesIntoDeployment(podSpec *corev1.PodSpec, resources *corev1.Re
 // InjectNodeSelectorIntoDeployment injects the provided NodeSelector
 // into the container(s) of the given PodSpec.
 //
-// If any Container in PodSpec already defines a NodeSelector it will
-// be overwritten.
+// If the PodSpec already defines a NodeSelector it will be overwritten.
 func InjectNodeSelectorIntoDeployment(podSpec *corev1.PodSpec, nodeSelector map[string]string) error {
 	if podSpec == nil {
 		return errors.New("no pod spec provided")
@@ -300,3 +300,40 @@ func OverrideDeploymentAffinity(podSpec *corev1.PodSpec, affinity *corev1.Affini
 
 	return nil
 }
+
+// InjectAnnotationsIntoDeployment injects the provided Annotations
+// into the container(s) of the given PodSpec.
+//
+// If the Deployment already defines any Annotations they will NOT be overwritten.
+func InjectAnnotationsIntoDeployment(deployment *appsv1.Deployment, newAnnotations map[string]string) error {
+	if deployment == nil {
+		return errors.New("no deployment provided")
+	}
+
+	// do not override existing annotations
+	if newAnnotations != nil {
+		mergedDeploymentAnnotations := map[string]string{}
+		mergedPodAnnotations := map[string]string{}
+
+		// add newAnnotations first to prevent them from overwriting the defaults
+		for k, v := range newAnnotations {
+			mergedDeploymentAnnotations[k] = v
+			mergedPodAnnotations[k] = v
+		}
+
+		// then replace any duplicate annotations with the default annotation
+		for k, v := range deployment.Annotations {
+			mergedDeploymentAnnotations[k] = v
+		}
+		for k, v := range deployment.Spec.Template.GetAnnotations() {
+			mergedPodAnnotations[k] = v
+		}
+
+		// Inject Into Deployment
+		deployment.SetAnnotations(mergedDeploymentAnnotations)
+		// Inject Into Pod Spec
+		deployment.Spec.Template.SetAnnotations(mergedPodAnnotations)
+	}
+
+	return nil
+}