Skip to content

Commit 0dd03c9

Browse files
xing-yangxing-yang
committed
Add original retries back
1 parent c5d6e81 commit 0dd03c9

File tree

2 files changed

+62
-31
lines changed

2 files changed

+62
-31
lines changed

‎pkg/common-controller/snapshot_controller.go

+42-21
Original file line numberDiff line numberDiff line change
@@ -337,10 +337,16 @@ func (ctrl *csiSnapshotCommonController) syncUnreadySnapshot(snapshot *crdv1.Vol
337337
}
338338

339339
// update snapshot status
340-
klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot))
341-
_, err = ctrl.updateSnapshotStatus(snapshot, newContent)
342-
if err != nil {
340+
for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ {
341+
klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot))
342+
_, err = ctrl.updateSnapshotStatus(snapshot, newContent)
343+
if err == nil {
344+
break
345+
}
343346
klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err)
347+
}
348+
349+
if err != nil {
344350
// update snapshot status failed
345351
ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err))
346352
return err
@@ -393,10 +399,16 @@ func (ctrl *csiSnapshotCommonController) syncUnreadySnapshot(snapshot *crdv1.Vol
393399
}
394400

395401
// Update snapshot status with BoundVolumeSnapshotContentName
396-
klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot))
397-
_, err = ctrl.updateSnapshotStatus(snapshot, content)
398-
if err != nil {
402+
for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ {
403+
klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot))
404+
_, err = ctrl.updateSnapshotStatus(snapshot, content)
405+
if err == nil {
406+
break
407+
}
399408
klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err)
409+
}
410+
411+
if err != nil {
400412
// update snapshot status failed
401413
ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err))
402414
return err
@@ -489,17 +501,23 @@ func (ctrl *csiSnapshotCommonController) createSnapshotContent(snapshot *crdv1.V
489501

490502
var updateContent *crdv1.VolumeSnapshotContent
491503
klog.V(3).Infof("volume snapshot content %#v", snapshotContent)
492-
// Try to create the VolumeSnapshotContent object
493-
klog.V(5).Infof("createSnapshotContent [%s]: trying to save volume snapshot content %s", utils.SnapshotKey(snapshot), snapshotContent.Name)
494-
if updateContent, err = ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Create(snapshotContent); err == nil || apierrs.IsAlreadyExists(err) {
495-
// Save succeeded.
496-
if err != nil {
497-
klog.V(3).Infof("volume snapshot content %q for snapshot %q already exists, reusing", snapshotContent.Name, utils.SnapshotKey(snapshot))
498-
err = nil
499-
updateContent = snapshotContent
500-
} else {
501-
klog.V(3).Infof("volume snapshot content %q for snapshot %q saved, %v", snapshotContent.Name, utils.SnapshotKey(snapshot), snapshotContent)
504+
// Try to create the VolumeSnapshotContent object several times
505+
for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ {
506+
klog.V(5).Infof("createSnapshotContent [%s]: trying to save volume snapshot content %s", utils.SnapshotKey(snapshot), snapshotContent.Name)
507+
if updateContent, err = ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Create(snapshotContent); err == nil || apierrs.IsAlreadyExists(err) {
508+
// Save succeeded.
509+
if err != nil {
510+
klog.V(3).Infof("volume snapshot content %q for snapshot %q already exists, reusing", snapshotContent.Name, utils.SnapshotKey(snapshot))
511+
err = nil
512+
updateContent = snapshotContent
513+
} else {
514+
klog.V(3).Infof("volume snapshot content %q for snapshot %q saved, %v", snapshotContent.Name, utils.SnapshotKey(snapshot), snapshotContent)
515+
}
516+
break
502517
}
518+
// Save failed, try again after a while.
519+
klog.V(3).Infof("failed to save volume snapshot content %q for snapshot %q: %v", snapshotContent.Name, utils.SnapshotKey(snapshot), err)
520+
time.Sleep(ctrl.createSnapshotContentInterval)
503521
}
504522

505523
if err != nil {
@@ -849,14 +867,17 @@ func (ctrl *csiSnapshotCommonController) bindandUpdateVolumeSnapshot(snapshotCon
849867
snapshotCopy := snapshotObj.DeepCopy()
850868

851869
// update snapshot status
852-
klog.V(5).Infof("bindandUpdateVolumeSnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshotCopy))
853-
updateSnapshot, err := ctrl.updateSnapshotStatus(snapshotCopy, snapshotContent)
854-
if err == nil {
855-
snapshotCopy = updateSnapshot
870+
for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ {
871+
klog.V(5).Infof("bindandUpdateVolumeSnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshotCopy))
872+
updateSnapshot, err := ctrl.updateSnapshotStatus(snapshotCopy, snapshotContent)
873+
if err == nil {
874+
snapshotCopy = updateSnapshot
875+
break
876+
}
877+
klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err)
856878
}
857879

858880
if err != nil {
859-
klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err)
860881
// update snapshot status failed
861882
ctrl.updateSnapshotErrorStatusWithEvent(snapshotCopy, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err))
862883
return nil, err

‎pkg/common-controller/snapshot_controller_base.go

+20-10
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ import (
4242
"k8s.io/kubernetes/pkg/util/goroutinemap"
4343
)
4444

45+
// Number of retries when we create a VolumeSnapshotContent object
46+
const createSnapshotContentRetryCount = 5
47+
48+
// Interval between retries when we create a VolumeSnapshotContent object
49+
const createSnapshotContentInterval = 10 * time.Second
50+
4551
type csiSnapshotCommonController struct {
4652
clientset clientset.Interface
4753
client kubernetes.Interface
@@ -64,7 +70,9 @@ type csiSnapshotCommonController struct {
6470
// Map of scheduled/running operations.
6571
runningOperations goroutinemap.GoRoutineMap
6672

67-
resyncPeriod time.Duration
73+
createSnapshotContentRetryCount int
74+
createSnapshotContentInterval time.Duration
75+
resyncPeriod time.Duration
6876
}
6977

7078
// NewCSISnapshotController returns a new *csiSnapshotCommonController
@@ -84,15 +92,17 @@ func NewCSISnapshotCommonController(
8492
eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("snapshot-controller")})
8593

8694
ctrl := &csiSnapshotCommonController{
87-
clientset: clientset,
88-
client: client,
89-
eventRecorder: eventRecorder,
90-
runningOperations: goroutinemap.NewGoRoutineMap(true),
91-
resyncPeriod: resyncPeriod,
92-
snapshotStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),
93-
contentStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),
94-
snapshotQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "snapshot-controller-snapshot"),
95-
contentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "snapshot-controller-content"),
95+
clientset: clientset,
96+
client: client,
97+
eventRecorder: eventRecorder,
98+
runningOperations: goroutinemap.NewGoRoutineMap(true),
99+
createSnapshotContentRetryCount: createSnapshotContentRetryCount,
100+
createSnapshotContentInterval: createSnapshotContentInterval,
101+
resyncPeriod: resyncPeriod,
102+
snapshotStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),
103+
contentStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),
104+
snapshotQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "snapshot-controller-snapshot"),
105+
contentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "snapshot-controller-content"),
96106
}
97107

98108
ctrl.pvcLister = pvcInformer.Lister()

0 commit comments

Comments
 (0)