Skip to content

Commit 8b80f26

Browse files
joelanfordjoelanford
authoredJul 29, 2021
installplans: retry crd updates on conflicts (#2293)
Signed-off-by: Joe Lanford <[email protected]>
·
v0.31.0v0.19.0
1 parent 939bf94 commit 8b80f26

File tree

2 files changed

+114
-96
lines changed

2 files changed

+114
-96
lines changed
 

‎pkg/controller/install/webhook.go

+64-58
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,17 @@ import (
55
"fmt"
66
"hash/fnv"
77

8-
"github.com/operator-framework/api/pkg/operators/v1alpha1"
9-
hashutil "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/kubernetes/pkg/util/hash"
10-
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
11-
128
log "github.com/sirupsen/logrus"
139
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
1410
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1511
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1612
"k8s.io/apimachinery/pkg/labels"
1713
"k8s.io/apimachinery/pkg/util/rand"
14+
"k8s.io/client-go/util/retry"
15+
16+
"github.com/operator-framework/api/pkg/operators/v1alpha1"
17+
hashutil "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/kubernetes/pkg/util/hash"
18+
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
1819
)
1920

2021
func ValidWebhookRules(rules []admissionregistrationv1.RuleWithOperations) error {
@@ -200,69 +201,74 @@ func (i *StrategyDeploymentInstaller) createOrUpdateConversionWebhook(caPEM []by
200201

201202
// iterate over all the ConversionCRDs
202203
for _, conversionCRD := range desc.ConversionCRDs {
203-
// Get existing CRD on cluster
204-
crd, err := i.strategyClient.GetOpClient().ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), conversionCRD, metav1.GetOptions{})
205-
if err != nil {
206-
return fmt.Errorf("Unable to get CRD %s specified in Conversion Webhook: %v", conversionCRD, err)
207-
}
204+
if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
205+
// Get existing CRD on cluster
206+
crd, err := i.strategyClient.GetOpClient().ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), conversionCRD, metav1.GetOptions{})
207+
if err != nil {
208+
return fmt.Errorf("Unable to get CRD %s specified in Conversion Webhook: %v", conversionCRD, err)
209+
}
208210

209-
// check if this CRD is an owned CRD
210-
foundCRD := false
211-
for _, ownedCRD := range csv.Spec.CustomResourceDefinitions.Owned {
212-
if ownedCRD.Name == conversionCRD {
213-
foundCRD = true
214-
break
211+
// check if this CRD is an owned CRD
212+
foundCRD := false
213+
for _, ownedCRD := range csv.Spec.CustomResourceDefinitions.Owned {
214+
if ownedCRD.Name == conversionCRD {
215+
foundCRD = true
216+
break
217+
}
218+
}
219+
if !foundCRD {
220+
return fmt.Errorf("CSV %s does not own CRD %s", csv.GetName(), conversionCRD)
215221
}
216-
}
217-
if !foundCRD {
218-
return fmt.Errorf("CSV %s does not own CRD %s", csv.GetName(), conversionCRD)
219-
}
220222

221-
// crd.Spec.Conversion.Strategy specifies how custom resources are converted between versions.
222-
// Allowed values are:
223-
// - None: The converter only change the apiVersion and would not touch any other field in the custom resource.
224-
// - Webhook: API Server will call to an external webhook to do the conversion. This requires crd.Spec.preserveUnknownFields to be false.
225-
// References:
226-
// - https://docs.openshift.com/container-platform/4.5/rest_api/extension_apis/customresourcedefinition-apiextensions-k8s-io-v1.html
227-
// - https://kubernetes.io/blog/2019/06/20/crd-structural-schema/#pruning-don-t-preserve-unknown-fields
228-
// By default the strategy is none
229-
// Reference:
230-
// - https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#specify-multiple-versions
231-
if crd.Spec.PreserveUnknownFields != false {
232-
return fmt.Errorf("crd.Spec.PreserveUnknownFields must be false to let API Server call webhook to do the conversion")
233-
}
223+
// crd.Spec.Conversion.Strategy specifies how custom resources are converted between versions.
224+
// Allowed values are:
225+
// - None: The converter only change the apiVersion and would not touch any other field in the custom resource.
226+
// - Webhook: API Server will call to an external webhook to do the conversion. This requires crd.Spec.preserveUnknownFields to be false.
227+
// References:
228+
// - https://docs.openshift.com/container-platform/4.5/rest_api/extension_apis/customresourcedefinition-apiextensions-k8s-io-v1.html
229+
// - https://kubernetes.io/blog/2019/06/20/crd-structural-schema/#pruning-don-t-preserve-unknown-fields
230+
// By default the strategy is none
231+
// Reference:
232+
// - https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#specify-multiple-versions
233+
if crd.Spec.PreserveUnknownFields != false {
234+
return fmt.Errorf("crd.Spec.PreserveUnknownFields must be false to let API Server call webhook to do the conversion")
235+
}
234236

235-
// Conversion WebhookClientConfig should not be set when Strategy is None
236-
// https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#specify-multiple-versions
237-
// Conversion WebhookClientConfig needs to be set when Strategy is None
238-
// https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#configure-customresourcedefinition-to-use-conversion-webhooks
237+
// Conversion WebhookClientConfig should not be set when Strategy is None
238+
// https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#specify-multiple-versions
239+
// Conversion WebhookClientConfig needs to be set when Strategy is None
240+
// https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#configure-customresourcedefinition-to-use-conversion-webhooks
239241

240-
// use user defined path for CRD conversion webhook, else set default value
241-
conversionWebhookPath := "/"
242-
if desc.WebhookPath != nil {
243-
conversionWebhookPath = *desc.WebhookPath
244-
}
242+
// use user defined path for CRD conversion webhook, else set default value
243+
conversionWebhookPath := "/"
244+
if desc.WebhookPath != nil {
245+
conversionWebhookPath = *desc.WebhookPath
246+
}
245247

246-
// Override Name, Namespace, and CABundle
247-
crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
248-
Strategy: "Webhook",
249-
Webhook: &apiextensionsv1.WebhookConversion{
250-
ClientConfig: &apiextensionsv1.WebhookClientConfig{
251-
Service: &apiextensionsv1.ServiceReference{
252-
Namespace: i.owner.GetNamespace(),
253-
Name: desc.DomainName() + "-service",
254-
Path: &conversionWebhookPath,
255-
Port: &desc.ContainerPort,
248+
// Override Name, Namespace, and CABundle
249+
crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
250+
Strategy: "Webhook",
251+
Webhook: &apiextensionsv1.WebhookConversion{
252+
ClientConfig: &apiextensionsv1.WebhookClientConfig{
253+
Service: &apiextensionsv1.ServiceReference{
254+
Namespace: i.owner.GetNamespace(),
255+
Name: desc.DomainName() + "-service",
256+
Path: &conversionWebhookPath,
257+
Port: &desc.ContainerPort,
258+
},
259+
CABundle: caPEM,
256260
},
257-
CABundle: caPEM,
261+
ConversionReviewVersions: desc.AdmissionReviewVersions,
258262
},
259-
ConversionReviewVersions: desc.AdmissionReviewVersions,
260-
},
261-
}
263+
}
262264

263-
// update CRD conversion Specs
264-
if _, err = i.strategyClient.GetOpClient().ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{}); err != nil {
265-
return fmt.Errorf("Error updating CRD with Conversion info: %v", err)
265+
// update CRD conversion Specs
266+
if _, err = i.strategyClient.GetOpClient().ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{}); err != nil {
267+
return fmt.Errorf("Error updating CRD with Conversion info: %w", err)
268+
}
269+
return nil
270+
}); err != nil {
271+
return err
266272
}
267273
}
268274

‎pkg/controller/operators/catalog/step.go

+50-38
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@ import (
66

77
"github.com/pkg/errors"
88
"github.com/sirupsen/logrus"
9-
109
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1110
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
1211
apiextensionsv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
1312
apiextensionsv1beta1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
1413
k8serrors "k8s.io/apimachinery/pkg/api/errors"
1514
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1615
"k8s.io/client-go/dynamic"
16+
"k8s.io/client-go/util/retry"
1717

1818
"github.com/operator-framework/api/pkg/operators/v1alpha1"
1919
listersv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/listers/operators/v1alpha1"
@@ -132,27 +132,33 @@ func (b *builder) NewCRDV1Step(client apiextensionsv1client.ApiextensionsV1Inter
132132

133133
_, createError := client.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
134134
if k8serrors.IsAlreadyExists(createError) {
135-
currentCRD, _ := client.CustomResourceDefinitions().Get(context.TODO(), crd.GetName(), metav1.GetOptions{})
136-
crd.SetResourceVersion(currentCRD.GetResourceVersion())
137-
if err = validateV1CRDCompatibility(b.dynamicClient, currentCRD, crd); err != nil {
138-
return v1alpha1.StepStatusUnknown, errors.Wrapf(err, "error validating existing CRs against new CRD's schema: %s", step.Resource.Name)
139-
}
135+
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
136+
currentCRD, _ := client.CustomResourceDefinitions().Get(context.TODO(), crd.GetName(), metav1.GetOptions{})
137+
crd.SetResourceVersion(currentCRD.GetResourceVersion())
138+
if err = validateV1CRDCompatibility(b.dynamicClient, currentCRD, crd); err != nil {
139+
return fmt.Errorf("error validating existing CRs against new CRD's schema for %q: %w", step.Resource.Name, err)
140+
}
140141

141-
// check to see if stored versions changed and whether the upgrade could cause potential data loss
142-
safe, err := crdlib.SafeStorageVersionUpgrade(currentCRD, crd)
143-
if !safe {
144-
b.logger.Errorf("risk of data loss updating %s: %s", step.Resource.Name, err)
145-
return v1alpha1.StepStatusUnknown, errors.Wrapf(err, "risk of data loss updating %s", step.Resource.Name)
146-
}
147-
if err != nil {
148-
return v1alpha1.StepStatusUnknown, errors.Wrapf(err, "checking CRD for potential data loss updating %s", step.Resource.Name)
149-
}
142+
// check to see if stored versions changed and whether the upgrade could cause potential data loss
143+
safe, err := crdlib.SafeStorageVersionUpgrade(currentCRD, crd)
144+
if !safe {
145+
b.logger.Errorf("risk of data loss updating %q: %s", step.Resource.Name, err)
146+
return fmt.Errorf("risk of data loss updating %q: %w", step.Resource.Name, err)
147+
}
148+
if err != nil {
149+
return fmt.Errorf("checking CRD for potential data loss updating %q: %w", step.Resource.Name, err)
150+
}
150151

151-
// Update CRD to new version
152-
setInstalledAlongsideAnnotation(b.annotator, crd, b.plan.GetNamespace(), step.Resolving, b.csvLister, crd, currentCRD)
153-
_, err = client.CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
152+
// Update CRD to new version
153+
setInstalledAlongsideAnnotation(b.annotator, crd, b.plan.GetNamespace(), step.Resolving, b.csvLister, crd, currentCRD)
154+
_, err = client.CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
155+
if err != nil {
156+
return fmt.Errorf("error updating CRD %q: %w", step.Resource.Name, err)
157+
}
158+
return nil
159+
})
154160
if err != nil {
155-
return v1alpha1.StepStatusUnknown, errors.Wrapf(err, "error updating CRD: %s", step.Resource.Name)
161+
return v1alpha1.StepStatusUnknown, err
156162
}
157163
// If it already existed, mark the step as Present.
158164
// they were equal - mark CRD as present
@@ -181,7 +187,7 @@ func (b *builder) NewCRDV1Beta1Step(client apiextensionsv1beta1client.Apiextensi
181187
if k8serrors.IsNotFound(err) {
182188
return v1alpha1.StepStatusNotPresent, nil
183189
} else {
184-
return v1alpha1.StepStatusNotPresent, errors.Wrapf(err, "error finding the %s CRD", crd.Name)
190+
return v1alpha1.StepStatusNotPresent, fmt.Errorf("error finding the %q CRD: %w", crd.Name, err)
185191
}
186192
}
187193
established, namesAccepted := false, false
@@ -210,28 +216,34 @@ func (b *builder) NewCRDV1Beta1Step(client apiextensionsv1beta1client.Apiextensi
210216

211217
_, createError := client.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
212218
if k8serrors.IsAlreadyExists(createError) {
213-
currentCRD, _ := client.CustomResourceDefinitions().Get(context.TODO(), crd.GetName(), metav1.GetOptions{})
214-
crd.SetResourceVersion(currentCRD.GetResourceVersion())
219+
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
220+
currentCRD, _ := client.CustomResourceDefinitions().Get(context.TODO(), crd.GetName(), metav1.GetOptions{})
221+
crd.SetResourceVersion(currentCRD.GetResourceVersion())
215222

216-
if err = validateV1Beta1CRDCompatibility(b.dynamicClient, currentCRD, crd); err != nil {
217-
return v1alpha1.StepStatusUnknown, errors.Wrapf(err, "error validating existing CRs against new CRD's schema: %s", step.Resource.Name)
218-
}
223+
if err = validateV1Beta1CRDCompatibility(b.dynamicClient, currentCRD, crd); err != nil {
224+
return fmt.Errorf("error validating existing CRs against new CRD's schema for %q: %w", step.Resource.Name, err)
225+
}
219226

220-
// check to see if stored versions changed and whether the upgrade could cause potential data loss
221-
safe, err := crdlib.SafeStorageVersionUpgrade(currentCRD, crd)
222-
if !safe {
223-
b.logger.Errorf("risk of data loss updating %s: %s", step.Resource.Name, err)
224-
return v1alpha1.StepStatusUnknown, errors.Wrapf(err, "risk of data loss updating %s", step.Resource.Name)
225-
}
226-
if err != nil {
227-
return v1alpha1.StepStatusUnknown, errors.Wrapf(err, "checking CRD for potential data loss updating %s", step.Resource.Name)
228-
}
227+
// check to see if stored versions changed and whether the upgrade could cause potential data loss
228+
safe, err := crdlib.SafeStorageVersionUpgrade(currentCRD, crd)
229+
if !safe {
230+
b.logger.Errorf("risk of data loss updating %q: %s", step.Resource.Name, err)
231+
return fmt.Errorf("risk of data loss updating %q: %w", step.Resource.Name, err)
232+
}
233+
if err != nil {
234+
return fmt.Errorf("checking CRD for potential data loss updating %q: %w", step.Resource.Name, err)
235+
}
229236

230-
// Update CRD to new version
231-
setInstalledAlongsideAnnotation(b.annotator, crd, b.plan.GetNamespace(), step.Resolving, b.csvLister, crd, currentCRD)
232-
_, err = client.CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
237+
// Update CRD to new version
238+
setInstalledAlongsideAnnotation(b.annotator, crd, b.plan.GetNamespace(), step.Resolving, b.csvLister, crd, currentCRD)
239+
_, err = client.CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
240+
if err != nil {
241+
return fmt.Errorf("error updating CRD %q: %w", step.Resource.Name, err)
242+
}
243+
return nil
244+
})
233245
if err != nil {
234-
return v1alpha1.StepStatusUnknown, errors.Wrapf(err, "error updating CRD: %s", step.Resource.Name)
246+
return v1alpha1.StepStatusUnknown, err
235247
}
236248
// If it already existed, mark the step as Present.
237249
// they were equal - mark CRD as present

0 commit comments

Comments
 (0)
Please sign in to comment.