Add state to snapshot create and configurable retry logic

Signed-off-by: Grant Griffiths <[email protected]>

Author: ggriffiths

cmd/csi-snapshotter/csi-snapshotter

@@ -63,6 +63,9 @@ var (
 	csiAddress                      = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.")
 	createSnapshotContentRetryCount = flag.Int("create-snapshotcontent-retrycount", 5, "Number of retries when we create a snapshot content object for a snapshot.")
 	createSnapshotContentInterval   = flag.Duration("create-snapshotcontent-interval", 10*time.Second, "Interval between retries when we create a snapshot content object for a snapshot.")
+	retryIntervalStart              = flag.Duration("retry-interval-start", time.Second, "Initial retry interval of failed snapshot creation. It doubles with each failure, up to retry-interval-max.")
+	retryIntervalMax                = flag.Duration("retry-interval-max", 5*time.Minute, "Maximum retry interval of failed snapshot creation.")
+	failedSnapshotThreshold         = flag.Int("failed-snapshot-threshold", 15, "The maximum number of retries on snapshot failures. Set 0 to retry indefinitely. Default is 15.")
 	resyncPeriod                    = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.")
 	snapshotNamePrefix              = flag.String("snapshot-name-prefix", "snapshot", "Prefix to apply to the name of a created snapshot")
 	snapshotNameUUIDLength          = flag.Int("snapshot-name-uuid-length", -1, "Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate.")
@@ -191,6 +194,9 @@ func main() {
 		coreFactory.Core().V1().PersistentVolumeClaims(),
 		*createSnapshotContentRetryCount,
 		*createSnapshotContentInterval,
+		*retryIntervalStart,
+		*retryIntervalMax,
+		*failedSnapshotThreshold,
 		snapShotter,
 		*csiTimeout,
 		*resyncPeriod,

cmd/csi-snapshotter/main.go

@@ -30,7 +30,7 @@ import (
 
 // Handler is responsible for handling VolumeSnapshot events from informer.
 type Handler interface {
-	CreateSnapshot(snapshot *crdv1.VolumeSnapshot, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error)
+	CreateSnapshot(snapshot *crdv1.VolumeSnapshot, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, snapshotter.SnapshottingState, error)
 	DeleteSnapshot(content *crdv1.VolumeSnapshotContent, snapshotterCredentials map[string]string) error
 	GetSnapshotStatus(content *crdv1.VolumeSnapshotContent) (bool, time.Time, int64, error)
 }
@@ -58,19 +58,20 @@ func NewCSIHandler(
 	}
 }
 
-func (handler *csiHandler) CreateSnapshot(snapshot *crdv1.VolumeSnapshot, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) {
+func (handler *csiHandler) CreateSnapshot(snapshot *crdv1.VolumeSnapshot, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, snapshotter.SnapshottingState, error) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), handler.timeout)
 	defer cancel()
 
 	snapshotName, err := makeSnapshotName(handler.snapshotNamePrefix, string(snapshot.UID), handler.snapshotNameUUIDLength)
 	if err != nil {
-		return "", "", time.Time{}, 0, false, err
+		return "", "", time.Time{}, 0, false, snapshotter.SnapshottingFinished, err
 	}
 	newParameters, err := removePrefixedParameters(parameters)
 	if err != nil {
-		return "", "", time.Time{}, 0, false, fmt.Errorf("failed to remove CSI Parameters of prefixed keys: %v", err)
+		return "", "", time.Time{}, 0, false, snapshotter.SnapshottingFinished, fmt.Errorf("failed to remove CSI Parameters of prefixed keys: %v", err)
 	}
+
 	return handler.snapshotter.CreateSnapshot(ctx, snapshotName, volume, newParameters, snapshotterCredentials)
 }
 

pkg/controller/csi_handler.go

@@ -35,6 +35,7 @@ import (
 	snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme"
 	informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions"
 	storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1alpha1"
+	"github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter"
 	"k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	storagev1beta1 "k8s.io/api/storage/v1beta1"
@@ -772,6 +773,9 @@ func newTestController(kubeClient kubernetes.Interface, clientset clientset.Inte
 		coreFactory.Core().V1().PersistentVolumeClaims(),
 		3,
 		5*time.Millisecond,
+		5*time.Millisecond,
+		10*time.Second,
+		5,
 		fakeSnapshot,
 		5*time.Millisecond,
 		60*time.Second,
@@ -1329,10 +1333,10 @@ type fakeSnapshotter struct {
 	t                 *testing.T
 }
 
-func (f *fakeSnapshotter) CreateSnapshot(ctx context.Context, snapshotName string, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) {
+func (f *fakeSnapshotter) CreateSnapshot(ctx context.Context, snapshotName string, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, snapshotter.SnapshottingState, error) {
 	if f.createCallCounter >= len(f.createCalls) {
 		f.t.Errorf("Unexpected CSI Create Snapshot call: snapshotName=%s, volume=%v, index: %d, calls: %+v", snapshotName, volume.Name, f.createCallCounter, f.createCalls)
-		return "", "", time.Time{}, 0, false, fmt.Errorf("unexpected call")
+		return "", "", time.Time{}, 0, false, snapshotter.SnapshottingFinished, fmt.Errorf("unexpected call")
 	}
 	call := f.createCalls[f.createCallCounter]
 	f.createCallCounter++
@@ -1359,10 +1363,10 @@ func (f *fakeSnapshotter) CreateSnapshot(ctx context.Context, snapshotName strin
 	}
 
 	if err != nil {
-		return "", "", time.Time{}, 0, false, fmt.Errorf("unexpected call")
+		return "", "", time.Time{}, 0, false, snapshotter.SnapshottingFinished, fmt.Errorf("unexpected call")
 	}
 
-	return call.driverName, call.snapshotId, call.creationTime, call.size, call.readyToUse, call.err
+	return call.driverName, call.snapshotId, call.creationTime, call.size, call.readyToUse, snapshotter.SnapshottingFinished, call.err
 }
 
 func (f *fakeSnapshotter) DeleteSnapshot(ctx context.Context, snapshotID string, snapshotterCredentials map[string]string) error {

pkg/controller/framework_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
+	"github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter"
 	"k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	storage "k8s.io/api/storage/v1beta1"
@@ -364,8 +365,8 @@ func (ctrl *csiSnapshotController) createSnapshot(snapshot *crdv1.VolumeSnapshot
 	klog.V(5).Infof("createSnapshot[%s]: started", snapshotKey(snapshot))
 	opName := fmt.Sprintf("create-%s[%s]", snapshotKey(snapshot), string(snapshot.UID))
 	ctrl.scheduleOperation(opName, func() error {
-		snapshotObj, err := ctrl.createSnapshotOperation(snapshot)
-		if err != nil {
+		snapshotObj, state, err := ctrl.createSnapshotOperation(snapshot)
+		if err != nil && state == snapshotter.SnapshottingFinished {
 			ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot: %v", err))
 			klog.Errorf("createSnapshot [%s]: error occurred in createSnapshotOperation: %v", opName, err)
 			return err
@@ -375,7 +376,19 @@ func (ctrl *csiSnapshotController) createSnapshot(snapshot *crdv1.VolumeSnapshot
 			// We will get an "snapshot update" event soon, this is not a big error
 			klog.V(4).Infof("createSnapshot [%s]: cannot update internal cache: %v", snapshotKey(snapshotObj), updateErr)
 		}
-		return nil
+		if state == snapshotter.SnapshottingFinished {
+			// Snapshotting finished, remove obj from the queue.
+			ctrl.snapshotQueue.Done(snapshotObj)
+			return nil
+
+		} else if state == snapshotter.SnapshottingInBackground {
+			klog.V(4).Infof("createSnapshot [%s]: Temporary error received, adding Snapshot back in queue: %v", snapshotKey(snapshotObj), updateErr)
+			// Snapshotting still in progress.
+			return nil
+		} else {
+			// State is SnapshottingNoChange. Don't change the snapshot queue.
+			return nil
+		}
 	})
 	return nil
 }
@@ -584,7 +597,7 @@ func (ctrl *csiSnapshotController) checkandUpdateBoundSnapshotStatusOperation(sn
 		if err != nil {
 			return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", snapshot.Name, err)
 		}
-		driverName, snapshotID, creationTime, size, readyToUse, err = ctrl.handler.CreateSnapshot(snapshot, volume, class.Parameters, snapshotterCredentials)
+		driverName, snapshotID, creationTime, size, readyToUse, _, err = ctrl.handler.CreateSnapshot(snapshot, volume, class.Parameters, snapshotterCredentials)
 		if err != nil {
 			klog.Errorf("checkandUpdateBoundSnapshotStatusOperation: failed to call create snapshot to check whether the snapshot is ready to use %q", err)
 			return nil, err
@@ -611,30 +624,30 @@ func (ctrl *csiSnapshotController) checkandUpdateBoundSnapshotStatusOperation(sn
 // 2. Update VolumeSnapshot status with creationtimestamp information
 // 3. Create the VolumeSnapshotContent object with the snapshot id information.
 // 4. Bind the VolumeSnapshot and VolumeSnapshotContent object
-func (ctrl *csiSnapshotController) createSnapshotOperation(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, error) {
+func (ctrl *csiSnapshotController) createSnapshotOperation(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, snapshotter.SnapshottingState, error) {
 	klog.Infof("createSnapshot: Creating snapshot %s through the plugin ...", snapshotKey(snapshot))
 
 	if snapshot.Status.Error != nil && !isControllerUpdateFailError(snapshot.Status.Error) {
 		klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", snapshot.Status.Error.Message)
-		return snapshot, nil
+		return snapshot, snapshotter.SnapshottingFinished, nil
 	}
 
 	// If PVC is not being deleted and finalizer is not added yet, a finalizer should be added.
 	klog.V(5).Infof("createSnapshotOperation: Check if PVC is not being deleted and add Finalizer for source of snapshot [%s] if needed", snapshot.Name)
 	err := ctrl.ensureSnapshotSourceFinalizer(snapshot)
 	if err != nil {
 		klog.Errorf("createSnapshotOperation failed to add finalizer for source of snapshot %s", err)
-		return nil, err
+		return nil, snapshotter.SnapshottingFinished, err
 	}
 
 	class, volume, contentName, snapshotterCredentials, err := ctrl.getCreateSnapshotInput(snapshot)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", snapshot.Name, err)
+		return nil, snapshotter.SnapshottingFinished, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", snapshot.Name, err)
 	}
 
-	driverName, snapshotID, creationTime, size, readyToUse, err := ctrl.handler.CreateSnapshot(snapshot, volume, class.Parameters, snapshotterCredentials)
+	driverName, snapshotID, creationTime, size, readyToUse, state, err := ctrl.handler.CreateSnapshot(snapshot, volume, class.Parameters, snapshotterCredentials)
 	if err != nil {
-		return nil, fmt.Errorf("failed to take snapshot of the volume, %s: %q", volume.Name, err)
+		return nil, state, fmt.Errorf("failed to take snapshot of the volume, %s: %q", volume.Name, err)
 	}
 
 	klog.V(5).Infof("Created snapshot: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse)
@@ -651,16 +664,16 @@ func (ctrl *csiSnapshotController) createSnapshotOperation(snapshot *crdv1.Volum
 	}
 
 	if err != nil {
-		return nil, err
+		return nil, snapshotter.SnapshottingFinished, err
 	}
 	// Create VolumeSnapshotContent in the database
 	volumeRef, err := ref.GetReference(scheme.Scheme, volume)
 	if err != nil {
-		return nil, err
+		return nil, snapshotter.SnapshottingFinished, err
 	}
 	snapshotRef, err := ref.GetReference(scheme.Scheme, snapshot)
 	if err != nil {
-		return nil, err
+		return nil, snapshotter.SnapshottingFinished, err
 	}
 
 	if class.DeletionPolicy == nil {
@@ -713,15 +726,15 @@ func (ctrl *csiSnapshotController) createSnapshotOperation(snapshot *crdv1.Volum
 		strerr := fmt.Sprintf("Error creating volume snapshot content object for snapshot %s: %v.", snapshotKey(snapshot), err)
 		klog.Error(strerr)
 		ctrl.eventRecorder.Event(newSnapshot, v1.EventTypeWarning, "CreateSnapshotContentFailed", strerr)
-		return nil, newControllerUpdateError(snapshotKey(snapshot), err.Error())
+		return nil, snapshotter.SnapshottingInBackground, newControllerUpdateError(snapshotKey(snapshot), err.Error())
 	}
 
 	// save succeeded, bind and update status for snapshot.
 	result, err := ctrl.bindandUpdateVolumeSnapshot(snapshotContent, newSnapshot)
 	if err != nil {
-		return nil, err
+		return nil, snapshotter.SnapshottingFinished, err
 	}
-	return result, nil
+	return result, snapshotter.SnapshottingFinished, nil
 }
 
 // Delete a snapshot

pkg/controller/snapshot_controller.go

@@ -68,6 +68,9 @@ type csiSnapshotController struct {
 
 	createSnapshotContentRetryCount int
 	createSnapshotContentInterval   time.Duration
+	retryIntervalStart              time.Duration
+	retryIntervalMax                time.Duration
+	failedSnapshotThreshold         int
 	resyncPeriod                    time.Duration
 }
 
@@ -82,6 +85,9 @@ func NewCSISnapshotController(
 	pvcInformer coreinformers.PersistentVolumeClaimInformer,
 	createSnapshotContentRetryCount int,
 	createSnapshotContentInterval time.Duration,
+	retryIntervalStart time.Duration,
+	retryIntervalMax time.Duration,
+	failedSnapshotThreshold int,
 	snapshotter snapshotter.Snapshotter,
 	timeout time.Duration,
 	resyncPeriod time.Duration,
@@ -103,10 +109,13 @@ func NewCSISnapshotController(
 		runningOperations:               goroutinemap.NewGoRoutineMap(true),
 		createSnapshotContentRetryCount: createSnapshotContentRetryCount,
 		createSnapshotContentInterval:   createSnapshotContentInterval,
+		retryIntervalStart:              retryIntervalStart,
+		retryIntervalMax:                retryIntervalMax,
+		failedSnapshotThreshold:         failedSnapshotThreshold,
 		resyncPeriod:                    resyncPeriod,
 		snapshotStore:                   cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),
 		contentStore:                    cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),
-		snapshotQueue:                   workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csi-snapshotter-snapshot"),
+		snapshotQueue:                   workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(retryIntervalStart, retryIntervalMax), "csi-snapshotter-snapshot"),
 		contentQueue:                    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csi-snapshotter-content"),
 	}
 
@@ -205,7 +214,6 @@ func (ctrl *csiSnapshotController) snapshotWorker() {
 		if quit {
 			return true
 		}
-		defer ctrl.snapshotQueue.Done(keyObj)
 		key := keyObj.(string)
 		klog.V(5).Infof("snapshotWorker[%s]", key)
 
@@ -393,12 +401,18 @@ func (ctrl *csiSnapshotController) updateSnapshot(snapshot *crdv1.VolumeSnapshot
 	}
 	err = ctrl.syncSnapshot(snapshot)
 	if err != nil {
+		sKey := snapshotKey(snapshot)
+		if ctrl.failedSnapshotThreshold == 0 {
+			ctrl.snapshotQueue.AddRateLimited(sKey)
+		} else if ctrl.snapshotQueue.NumRequeues(sKey) < ctrl.failedSnapshotThreshold {
+			ctrl.snapshotQueue.AddRateLimited(sKey)
+		}
 		if errors.IsConflict(err) {
 			// Version conflict error happens quite often and the controller
 			// recovers from it easily.
-			klog.V(3).Infof("could not sync claim %q: %+v", snapshotKey(snapshot), err)
+			klog.V(3).Infof("could not sync claim %q: %+v", sKey, err)
 		} else {
-			klog.Errorf("could not sync volume %q: %+v", snapshotKey(snapshot), err)
+			klog.Errorf("could not sync volume %q: %+v", sKey, err)
 		}
 	}
 }