Skip to content

Commit ed03be3

Browse files
stevekuznetsovstevekuznetsov
authored
controller/registry: implement content extraction for catalog sources (#3029)
* go.mod: update the api dependency Signed-off-by: Steve Kuznetsov <[email protected]> * controller/registry: implement content extraction for catalog sources Signed-off-by: Steve Kuznetsov <[email protected]> --------- Signed-off-by: Steve Kuznetsov <[email protected]>
1 parent 03d2bf0 commit ed03be3

19 files changed

+431
-33
lines changed

‎deploy/chart/crds/0000_50_olm_00-catalogsources.crd.yaml

+11-1
Original file line numberDiff line numberDiff line change
@@ -532,8 +532,18 @@ spec:
532532
topologyKey:
533533
description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
534534
type: string
535+
extractContent:
536+
description: ExtractContent configures the gRPC catalog Pod to extract catalog metadata from the provided index image and use a well-known version of the `opm` server to expose it. The catalog index image that this CatalogSource is configured to use *must* be using the file-based catalogs in order to utilize this feature.
537+
type: object
538+
properties:
539+
cacheDir:
540+
description: CacheDir is the directory storing the pre-calculated API cache.
541+
type: string
542+
catalogDir:
543+
description: CatalogDir is the directory storing the file-based catalog contents.
544+
type: string
535545
memoryTarget:
536-
description: "MemoryTarget configures the $GOMEMLIMIT value for the gRPC catalog Pod. This is a soft memory limit for the server, which the runtime will attempt to meet but makes no guarantees that it will do so. If this value is set, the Pod will have the following modifications made to the container running the server: - the $GOMEMLIMIT environment variable will be set to this value in bytes - the memory request will be set to this value - the memory limit will be set to 200% of this value \n This field should be set if it's desired to reduce the footprint of a catalog server as much as possible, or if a catalog being served is very large and needs more than the default allocation. If your index image has a file- system cache, determine a good approximation for this value by doubling the size of the package cache at /tmp/cache/cache/packages.json in the index image. \n This field is best-effort; if unset, no default will be used and no Pod memory limit or $GOMEMLIMIT value will be set."
546+
description: "MemoryTarget configures the $GOMEMLIMIT value for the gRPC catalog Pod. This is a soft memory limit for the server, which the runtime will attempt to meet but makes no guarantees that it will do so. If this value is set, the Pod will have the following modifications made to the container running the server: - the $GOMEMLIMIT environment variable will be set to this value in bytes - the memory request will be set to this value \n This field should be set if it's desired to reduce the footprint of a catalog server as much as possible, or if a catalog being served is very large and needs more than the default allocation. If your index image has a file- system cache, determine a good approximation for this value by doubling the size of the package cache at /tmp/cache/cache/packages.json in the index image. \n This field is best-effort; if unset, no default will be used and no Pod memory limit or $GOMEMLIMIT value will be set."
537547
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
538548
anyOf:
539549
- type: integer

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ require (
2525
github.com/onsi/gomega v1.27.7
2626
github.com/openshift/api v3.9.0+incompatible
2727
github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
28-
github.com/operator-framework/api v0.17.8-0.20230803152844-704ae942c4a9
28+
github.com/operator-framework/api v0.17.8-0.20230907172037-bb012a3b9b25
2929
github.com/operator-framework/operator-registry v1.29.0
3030
github.com/otiai10/copy v1.2.0
3131
github.com/pkg/errors v0.9.1

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -604,8 +604,8 @@ github.com/openshift/api v0.0.0-20221021112143-4226c2167e40 h1:PxjGCA72RtsdHWToZ
604604
github.com/openshift/api v0.0.0-20221021112143-4226c2167e40/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A=
605605
github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg=
606606
github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE=
607-
github.com/operator-framework/api v0.17.8-0.20230803152844-704ae942c4a9 h1:0rOIH8rbWsjhiZb14DD7V8Ohl0CjPkTYIbm/0G/nv1s=
608-
github.com/operator-framework/api v0.17.8-0.20230803152844-704ae942c4a9/go.mod h1:lnurXgadLnoZ7pufKMHkErr2BVOIZSpHtvEkHBcKvdk=
607+
github.com/operator-framework/api v0.17.8-0.20230907172037-bb012a3b9b25 h1:nY4VZQbe/gtCd+7MZK+ai6N0JMgJzb/S89uOhB61IbY=
608+
github.com/operator-framework/api v0.17.8-0.20230907172037-bb012a3b9b25/go.mod h1:Wbg136l1Po6zqG2QcTN1QZ8dbT4BQvNlQDM9tmQYvz0=
609609
github.com/operator-framework/operator-registry v1.29.0 h1:HMmVTiuOAGoHLzYqR9Lr2QSOqbVzA50++ojNl2mu9f4=
610610
github.com/operator-framework/operator-registry v1.29.0/go.mod h1:4rVQu/cOuCtVt3JzKsAmwyq2lsiu9uPaH9nYNfnqj9o=
611611
github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=

‎pkg/controller/operators/catalog/operator.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ func NewOperator(ctx context.Context, kubeconfigPath string, clock utilclock.Clo
217217
op.sources = grpc.NewSourceStore(logger, 10*time.Second, 10*time.Minute, op.syncSourceState)
218218
op.sourceInvalidator = resolver.SourceProviderFromRegistryClientProvider(op.sources, logger)
219219
resolverSourceProvider := NewOperatorGroupToggleSourceProvider(op.sourceInvalidator, logger, op.lister.OperatorsV1().OperatorGroupLister())
220-
op.reconciler = reconciler.NewRegistryReconcilerFactory(lister, opClient, configmapRegistryImage, op.now, ssaClient, workloadUserID)
220+
op.reconciler = reconciler.NewRegistryReconcilerFactory(lister, opClient, configmapRegistryImage, op.now, ssaClient, workloadUserID, opmImage)
221221
res := resolver.NewOperatorStepResolver(lister, crClient, operatorNamespace, resolverSourceProvider, logger)
222222
op.resolver = resolver.NewInstrumentedResolver(res, metrics.RegisterDependencyResolutionSuccess, metrics.RegisterDependencyResolutionFailure)
223223

‎pkg/controller/operators/catalog/operator_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -1773,7 +1773,7 @@ func NewFakeOperator(ctx context.Context, namespace string, namespaces []string,
17731773
}
17741774
applier := controllerclient.NewFakeApplier(s, "testowner")
17751775

1776-
op.reconciler = reconciler.NewRegistryReconcilerFactory(lister, op.opClient, "test:pod", op.now, applier, 1001)
1776+
op.reconciler = reconciler.NewRegistryReconcilerFactory(lister, op.opClient, "test:pod", op.now, applier, 1001, "")
17771777
}
17781778

17791779
op.RunInformers(ctx)
@@ -1929,7 +1929,7 @@ func toManifest(t *testing.T, obj runtime.Object) string {
19291929
}
19301930

19311931
func pod(s v1alpha1.CatalogSource) *corev1.Pod {
1932-
pod := reconciler.Pod(&s, "registry-server", s.Spec.Image, s.GetName(), s.GetLabels(), s.GetAnnotations(), 5, 10, 1001)
1932+
pod := reconciler.Pod(&s, "registry-server", "central-opm", s.Spec.Image, s.GetName(), s.GetLabels(), s.GetAnnotations(), 5, 10, 1001)
19331933
ownerutil.AddOwner(pod, &s, false, true)
19341934
return pod
19351935
}

‎pkg/controller/registry/reconciler/configmap.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func (s *configMapCatalogSourceDecorator) Service() *corev1.Service {
106106
}
107107

108108
func (s *configMapCatalogSourceDecorator) Pod(image string) *corev1.Pod {
109-
pod := Pod(s.CatalogSource, "configmap-registry-server", image, "", s.Labels(), s.Annotations(), 5, 5, s.runAsUser)
109+
pod := Pod(s.CatalogSource, "configmap-registry-server", "", image, "", s.Labels(), s.Annotations(), 5, 5, s.runAsUser)
110110
pod.Spec.ServiceAccountName = s.GetName() + ConfigMapServerPostfix
111111
pod.Spec.Containers[0].Command = []string{"configmap-server", "-c", s.Spec.ConfigMap, "-n", s.GetNamespace()}
112112
ownerutil.AddOwner(pod, s.CatalogSource, false, true)

‎pkg/controller/registry/reconciler/configmap_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ func objectsForCatalogSource(catsrc *v1alpha1.CatalogSource) []runtime.Object {
198198
)
199199
case v1alpha1.SourceTypeGrpc:
200200
if catsrc.Spec.Image != "" {
201-
decorated := grpcCatalogSourceDecorator{catsrc, runAsUser}
201+
decorated := grpcCatalogSourceDecorator{CatalogSource: catsrc, createPodAsUser: runAsUser, opmImage: ""}
202202
objs = clientfake.AddSimpleGeneratedNames(
203203
decorated.Pod(catsrc.GetName()),
204204
decorated.Service(),

‎pkg/controller/registry/reconciler/grpc.go

+15-4
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ const (
3535
type grpcCatalogSourceDecorator struct {
3636
*v1alpha1.CatalogSource
3737
createPodAsUser int64
38+
opmImage string
3839
}
3940

4041
type UpdateNotReadyErr struct {
@@ -128,7 +129,7 @@ func (s *grpcCatalogSourceDecorator) ServiceAccount() *corev1.ServiceAccount {
128129
}
129130

130131
func (s *grpcCatalogSourceDecorator) Pod(saName string) *corev1.Pod {
131-
pod := Pod(s.CatalogSource, "registry-server", s.Spec.Image, saName, s.Labels(), s.Annotations(), 5, 10, s.createPodAsUser)
132+
pod := Pod(s.CatalogSource, "registry-server", s.opmImage, s.Spec.Image, saName, s.Labels(), s.Annotations(), 5, 10, s.createPodAsUser)
132133
ownerutil.AddOwner(pod, s.CatalogSource, false, true)
133134
return pod
134135
}
@@ -139,6 +140,7 @@ type GrpcRegistryReconciler struct {
139140
OpClient operatorclient.ClientInterface
140141
SSAClient *controllerclient.ServerSideApplier
141142
createPodAsUser int64
143+
opmImage string
142144
}
143145

144146
var _ RegistryReconciler = &GrpcRegistryReconciler{}
@@ -196,16 +198,25 @@ func (c *GrpcRegistryReconciler) currentPodsWithCorrectImageAndSpec(source grpcC
196198
found := []*corev1.Pod{}
197199
newPod := source.Pod(saName)
198200
for _, p := range pods {
199-
if p.Spec.Containers[0].Image == source.Spec.Image && podHashMatch(p, newPod) {
201+
if correctImages(source, p) && podHashMatch(p, newPod) {
200202
found = append(found, p)
201203
}
202204
}
203205
return found
204206
}
205207

208+
func correctImages(source grpcCatalogSourceDecorator, pod *corev1.Pod) bool {
209+
if source.CatalogSource.Spec.GrpcPodConfig != nil && source.CatalogSource.Spec.GrpcPodConfig.ExtractContent != nil {
210+
return pod.Spec.InitContainers[0].Image == source.opmImage &&
211+
pod.Spec.InitContainers[1].Image == source.CatalogSource.Spec.Image &&
212+
pod.Spec.Containers[0].Image == source.opmImage
213+
}
214+
return pod.Spec.Containers[0].Image == source.CatalogSource.Spec.Image
215+
}
216+
206217
// EnsureRegistryServer ensures that all components of registry server are up to date.
207218
func (c *GrpcRegistryReconciler) EnsureRegistryServer(catalogSource *v1alpha1.CatalogSource) error {
208-
source := grpcCatalogSourceDecorator{catalogSource, c.createPodAsUser}
219+
source := grpcCatalogSourceDecorator{CatalogSource: catalogSource, createPodAsUser: c.createPodAsUser, opmImage: c.opmImage}
209220

210221
// if service status is nil, we force create every object to ensure they're created the first time
211222
overwrite := source.Status.RegistryServiceStatus == nil || !isRegistryServiceStatusValid(&source)
@@ -454,7 +465,7 @@ func (c *GrpcRegistryReconciler) removePods(pods []*corev1.Pod, namespace string
454465

455466
// CheckRegistryServer returns true if the given CatalogSource is considered healthy; false otherwise.
456467
func (c *GrpcRegistryReconciler) CheckRegistryServer(catalogSource *v1alpha1.CatalogSource) (healthy bool, err error) {
457-
source := grpcCatalogSourceDecorator{catalogSource, c.createPodAsUser}
468+
source := grpcCatalogSourceDecorator{CatalogSource: catalogSource, createPodAsUser: c.createPodAsUser, opmImage: c.opmImage}
458469
// Check on registry resources
459470
// TODO: add gRPC health check
460471
if len(c.currentPodsWithCorrectImageAndSpec(source, source.ServiceAccount().GetName())) < 1 ||

‎pkg/controller/registry/reconciler/grpc_test.go

+7-2
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@ package reconciler
22

33
import (
44
"context"
5+
"fmt"
56
"testing"
67
"time"
78

9+
"github.com/google/go-cmp/cmp"
810
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
911
"github.com/stretchr/testify/require"
1012
corev1 "k8s.io/api/core/v1"
@@ -355,7 +357,7 @@ func TestGrpcRegistryReconciler(t *testing.T) {
355357
}
356358

357359
// Check for resource existence
358-
decorated := grpcCatalogSourceDecorator{tt.in.catsrc, runAsUser}
360+
decorated := grpcCatalogSourceDecorator{CatalogSource: tt.in.catsrc, createPodAsUser: runAsUser}
359361
pod := decorated.Pod(tt.in.catsrc.GetName())
360362
service := decorated.Service()
361363
sa := decorated.ServiceAccount()
@@ -367,6 +369,9 @@ func TestGrpcRegistryReconciler(t *testing.T) {
367369
case *GrpcRegistryReconciler:
368370
// Should be created by a GrpcRegistryReconciler
369371
require.NoError(t, podErr)
372+
if diff := cmp.Diff(outPods.Items, []corev1.Pod{*pod}); diff != "" {
373+
fmt.Printf("incorrect pods: %s\n", diff)
374+
}
370375
require.Len(t, outPods.Items, 1)
371376
outPod := outPods.Items[0]
372377
require.Equal(t, pod.GetGenerateName(), outPod.GetGenerateName())
@@ -445,7 +450,7 @@ func TestRegistryPodPriorityClass(t *testing.T) {
445450
require.NoError(t, err)
446451

447452
// Check for resource existence
448-
decorated := grpcCatalogSourceDecorator{tt.in.catsrc, runAsUser}
453+
decorated := grpcCatalogSourceDecorator{CatalogSource: tt.in.catsrc, createPodAsUser: runAsUser}
449454
pod := decorated.Pod(tt.in.catsrc.GetName())
450455
listOptions := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{CatalogSourceLabelKey: tt.in.catsrc.GetName()}).String()}
451456
outPods, podErr := client.KubernetesInterface().CoreV1().Pods(pod.GetNamespace()).List(context.TODO(), listOptions)

‎pkg/controller/registry/reconciler/reconciler.go

+58-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ package reconciler
44
import (
55
"fmt"
66
"hash/fnv"
7+
"path/filepath"
78

89
"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
910
corev1 "k8s.io/api/core/v1"
@@ -64,6 +65,7 @@ type registryReconcilerFactory struct {
6465
ConfigMapServerImage string
6566
SSAClient *controllerclient.ServerSideApplier
6667
createPodAsUser int64
68+
opmImage string
6769
}
6870

6971
// ReconcilerForSource returns a RegistryReconciler based on the configuration of the given CatalogSource.
@@ -86,6 +88,7 @@ func (r *registryReconcilerFactory) ReconcilerForSource(source *operatorsv1alpha
8688
OpClient: r.OpClient,
8789
SSAClient: r.SSAClient,
8890
createPodAsUser: r.createPodAsUser,
91+
opmImage: r.opmImage,
8992
}
9093
} else if source.Spec.Address != "" {
9194
return &GrpcAddressRegistryReconciler{
@@ -97,18 +100,19 @@ func (r *registryReconcilerFactory) ReconcilerForSource(source *operatorsv1alpha
97100
}
98101

99102
// NewRegistryReconcilerFactory returns an initialized RegistryReconcilerFactory.
100-
func NewRegistryReconcilerFactory(lister operatorlister.OperatorLister, opClient operatorclient.ClientInterface, configMapServerImage string, now nowFunc, ssaClient *controllerclient.ServerSideApplier, createPodAsUser int64) RegistryReconcilerFactory {
103+
func NewRegistryReconcilerFactory(lister operatorlister.OperatorLister, opClient operatorclient.ClientInterface, configMapServerImage string, now nowFunc, ssaClient *controllerclient.ServerSideApplier, createPodAsUser int64, opmImage string) RegistryReconcilerFactory {
101104
return &registryReconcilerFactory{
102105
now: now,
103106
Lister: lister,
104107
OpClient: opClient,
105108
ConfigMapServerImage: configMapServerImage,
106109
SSAClient: ssaClient,
107110
createPodAsUser: createPodAsUser,
111+
opmImage: opmImage,
108112
}
109113
}
110114

111-
func Pod(source *operatorsv1alpha1.CatalogSource, name string, img string, saName string, labels map[string]string, annotations map[string]string, readinessDelay int32, livenessDelay int32, runAsUser int64) *corev1.Pod {
115+
func Pod(source *operatorsv1alpha1.CatalogSource, name, opmImg, img, saName string, labels, annotations map[string]string, readinessDelay, livenessDelay int32, runAsUser int64) *corev1.Pod {
112116
// make a copy of the labels and annotations to avoid mutating the input parameters
113117
podLabels := make(map[string]string)
114118
podAnnotations := make(map[string]string)
@@ -236,6 +240,58 @@ func Pod(source *operatorsv1alpha1.CatalogSource, name string, img string, saNam
236240
Value: grpcPodConfig.MemoryTarget.String() + "B", // k8s resources use Mi, GOMEMLIMIT wants MiB
237241
})
238242
}
243+
244+
// Reconfigure pod to extract content
245+
if grpcPodConfig.ExtractContent != nil {
246+
pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
247+
Name: "utilities",
248+
VolumeSource: corev1.VolumeSource{
249+
EmptyDir: &corev1.EmptyDirVolumeSource{},
250+
},
251+
}, corev1.Volume{
252+
Name: "catalog-content",
253+
VolumeSource: corev1.VolumeSource{
254+
EmptyDir: &corev1.EmptyDirVolumeSource{},
255+
},
256+
})
257+
const utilitiesPath = "/utilities"
258+
utilitiesVolumeMount := corev1.VolumeMount{
259+
Name: "utilities",
260+
MountPath: utilitiesPath,
261+
}
262+
const catalogPath = "/extracted-catalog"
263+
contentVolumeMount := corev1.VolumeMount{
264+
Name: "catalog-content",
265+
MountPath: catalogPath,
266+
}
267+
pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{
268+
Name: "extract-utilities",
269+
Image: opmImg,
270+
Command: []string{"sh", "-c"},
271+
Args: []string{fmt.Sprintf("cp $( command -v sh ) %s/sh && cp $( command -v cp ) %s/cp",
272+
utilitiesPath, utilitiesPath,
273+
)},
274+
VolumeMounts: []corev1.VolumeMount{utilitiesVolumeMount},
275+
}, corev1.Container{
276+
Name: "extract-content",
277+
Image: img,
278+
Command: []string{utilitiesPath + "/sh", "-c"},
279+
Args: []string{fmt.Sprintf("%s/cp -r %s %s/catalog && %s/cp -r %s %s/cache",
280+
utilitiesPath, grpcPodConfig.ExtractContent.CatalogDir, catalogPath,
281+
utilitiesPath, grpcPodConfig.ExtractContent.CacheDir, catalogPath,
282+
)},
283+
VolumeMounts: []corev1.VolumeMount{utilitiesVolumeMount, contentVolumeMount},
284+
})
285+
286+
pod.Spec.Containers[0].Image = opmImg
287+
pod.Spec.Containers[0].Command = []string{"/bin/opm"}
288+
pod.Spec.Containers[0].Args = []string{
289+
"serve",
290+
filepath.Join(catalogPath, "catalog"),
291+
"--cache-dir=" + filepath.Join(catalogPath, "cache"),
292+
}
293+
pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, contentVolumeMount)
294+
}
239295
}
240296

241297
// Set priorityclass if its annotation exists

0 commit comments

Comments
 (0)