Skip to content

Commit ed82822

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #210 from yuxiangqian/sidecar
sidecar controller update, fix potential snapshot leaking
2 parents 67c2e57 + b747de4 commit ed82822

File tree

5 files changed

+162
-268
lines changed

5 files changed

+162
-268
lines changed

‎pkg/sidecar-controller/content_create_test.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,13 @@ func TestSyncContent(t *testing.T) {
2525
var tests []controllerTest
2626

2727
tests = append(tests, controllerTest{
28-
name: "Basic content create ready to use",
29-
initialContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "", retainPolicy, nil, &defaultSize, &False, true),
30-
expectedContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "", retainPolicy, nil, &defaultSize, &True, true),
28+
name: "Basic content update ready to use",
29+
initialContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &False, true),
30+
expectedContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "volume-handle-1-1", retainPolicy, nil, &defaultSize, &True, true),
3131
expectedEvents: noevents,
3232
expectedCreateCalls: []createCall{
3333
{
34+
volumeHandle: "volume-handle-1-1",
3435
snapshotName: "snapshot-snapuid1-1",
3536
driverName: mockDriverName,
3637
snapshotId: "snapuid1-1",

‎pkg/sidecar-controller/framework_test.go

-4
Original file line numberDiff line numberDiff line change
@@ -553,10 +553,6 @@ func newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHa
553553
Spec: crdv1.VolumeSnapshotContentSpec{
554554
Driver: mockDriverName,
555555
DeletionPolicy: deletionPolicy,
556-
Source: crdv1.VolumeSnapshotContentSource{
557-
SnapshotHandle: &snapshotHandle,
558-
VolumeHandle: &volumeHandle,
559-
},
560556
},
561557
Status: &crdv1.VolumeSnapshotContentStatus{
562558
CreationTime: creationTime,

‎pkg/sidecar-controller/snapshot_controller.go

+73-59
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import (
2323

2424
crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1"
2525
"github.com/kubernetes-csi/external-snapshotter/pkg/utils"
26-
"k8s.io/api/core/v1"
26+
v1 "k8s.io/api/core/v1"
2727
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2828
"k8s.io/klog"
2929
"k8s.io/kubernetes/pkg/util/goroutinemap"
@@ -34,20 +34,20 @@ import (
3434
// Design:
3535
//
3636
// This is the sidecar controller that is responsible for creating and deleting a
37-
// snapshot on the storage infrastructure through a csi volume driver. It watches
38-
// the VolumeSnapshotContent object which is either created/deleted by the
37+
// snapshot on the storage system through a csi volume driver. It watches
38+
// VolumeSnapshotContent objects which have been either created/deleted by the
3939
// common snapshot controller in the case of dynamic provisioning or by the admin
4040
// in the case of pre-provisioned snapshots.
4141

42-
// The snapshot creation through csi volume driver should return a snapshot after
43-
// it is created successfully (however, the snapshot might not be ready to use yet if
44-
// there is an uploading phase). The creationTime will be updated accordingly
42+
// The snapshot creation through csi driver should return a snapshot after
43+
// it is created successfully(however, the snapshot might not be ready to use yet
44+
// if there is an uploading phase). The creationTime will be updated accordingly
4545
// on the status of VolumeSnapshotContent.
4646
// After that, the sidecar controller will keep checking the snapshot status
4747
// through csi snapshot calls. When the snapshot is ready to use, the sidecar
4848
// controller set the status "ReadyToUse" to true on the VolumeSnapshotContent object
49-
// to indicate the snapshot is ready to use. If the creation failed for any reason,
50-
// the Error status is set accordingly.
49+
// to indicate the snapshot is ready to be used to restore a volume.
50+
// If the creation failed for any reason, the Error status is set accordingly.
5151

5252
const controllerUpdateFailMsg = "snapshot controller failed to update"
5353

@@ -57,73 +57,50 @@ func (ctrl *csiSnapshotSideCarController) syncContent(content *crdv1.VolumeSnaps
5757

5858
var err error
5959
if ctrl.shouldDelete(content) {
60-
switch content.Spec.DeletionPolicy {
61-
case crdv1.VolumeSnapshotContentRetain:
62-
klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Retain. Keep physical snapshot and remove content finalizer", content.Name)
63-
// It is a deletion candidate if DeletionTimestamp is not nil and
64-
// VolumeSnapshotContentFinalizer is set.
65-
if utils.IsContentDeletionCandidate(content) {
66-
// Volume snapshot content is a deletion candidate.
67-
// Remove the content finalizer.
68-
klog.V(5).Infof("syncContent: Content [%s] is a deletion candidate. Remove finalizer.", content.Name)
69-
return ctrl.removeContentFinalizer(content)
70-
}
71-
72-
case crdv1.VolumeSnapshotContentDelete:
73-
klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Delete. Delete physical snapshot", content.Name)
74-
err = ctrl.deleteCSISnapshot(content)
75-
if err != nil {
76-
return err
77-
}
78-
klog.V(5).Infof("syncContent: check if we should remove Finalizer for VolumeSnapshotContent[%s]", content.Name)
79-
// It is a deletion candidate if DeletionTimestamp is not nil and
80-
// VolumeSnapshotContentFinalizer is set.
81-
if utils.IsContentDeletionCandidate(content) {
82-
// Volume snapshot content is a deletion candidate.
83-
// Remove the content finalizer.
84-
klog.V(5).Infof("syncContent: Content [%s] is a deletion candidate. Remove finalizer.", content.Name)
85-
return ctrl.removeContentFinalizer(content)
86-
}
87-
88-
default:
89-
// Unknown VolumeSnapshotDeletionPolicy
90-
ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotUnknownDeletionPolicy", "Volume Snapshot Content has unrecognized deletion policy")
91-
}
9260
klog.V(4).Infof("VolumeSnapshotContent[%s]: the policy is %s", content.Name, content.Spec.DeletionPolicy)
61+
if content.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete &&
62+
content.Status != nil && content.Status.SnapshotHandle != nil {
63+
// issue a CSI deletion call if the snapshot has not been deleted yet from
64+
// underlying storage system. Note that the deletion snapshot operation will
65+
// update content SnapshotHandle to nil upon a successful deletion. At this
66+
// point, the finalizer on content should NOT be removed to avoid leaking.
67+
return ctrl.deleteCSISnapshot(content)
68+
} else {
69+
// otherwise, either the snapshot has been deleted from the underlying
70+
// storage system, or the deletion policy is Retain, remove the finalizer
71+
// if there is one so that API server could delete the object if there is
72+
// no other finalizer.
73+
return ctrl.removeContentFinalizer(content)
74+
}
9375
} else {
94-
var err error
95-
klog.V(5).Infof("syncContent: Call CreateSnapshot for content %s", content.Name)
9676
if content.Spec.Source.VolumeHandle != nil && content.Status == nil {
77+
klog.V(5).Infof("syncContent: Call CreateSnapshot for content %s", content.Name)
9778
if err = ctrl.createSnapshot(content); err != nil {
9879
ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot with error %v", err))
9980
return err
10081
}
10182
} else {
102-
10383
if err = ctrl.checkandUpdateContentStatus(content); err != nil {
10484
ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotContentStatusUpdateFailed", fmt.Sprintf("Failed to update snapshot content status with error %v", err))
10585
return err
10686
}
10787
}
108-
109-
return nil
11088
}
111-
11289
return nil
11390
}
11491

11592
// deleteCSISnapshot starts delete action.
11693
func (ctrl *csiSnapshotSideCarController) deleteCSISnapshot(content *crdv1.VolumeSnapshotContent) error {
11794
operationName := fmt.Sprintf("delete-%s", content.Name)
118-
klog.V(5).Infof("Snapshotter is about to delete volume snapshot content and the operation named %s", operationName)
95+
klog.V(5).Infof("schedule to delete snapshot, operation named %s", operationName)
11996
ctrl.scheduleOperation(operationName, func() error {
12097
return ctrl.deleteCSISnapshotOperation(content)
12198
})
12299
return nil
123100
}
124101

125-
// scheduleOperation starts given asynchronous operation on given volume. It
126-
// makes sure the operation is already not running.
102+
// scheduleOperation starts given asynchronous operation on given snapshot. It
103+
// makes sure there is no running operation with the same operationName
127104
func (ctrl *csiSnapshotSideCarController) scheduleOperation(operationName string, operation func() error) {
128105
klog.V(5).Infof("scheduleOperation[%s]", operationName)
129106

@@ -199,17 +176,17 @@ func (ctrl *csiSnapshotSideCarController) updateContentErrorStatusWithEvent(cont
199176
if content.Status != nil && content.Status.Error != nil && *content.Status.Error.Message == message {
200177
klog.V(4).Infof("updateContentStatusWithEvent[%s]: the same error %v is already set", content.Name, content.Status.Error)
201178
return nil
202-
} else if content.Status == nil {
203-
content.Status = &crdv1.VolumeSnapshotContentStatus{}
204179
}
205180
contentClone := content.DeepCopy()
206-
statusError := &crdv1.VolumeSnapshotError{
181+
if contentClone.Status == nil {
182+
contentClone.Status = &crdv1.VolumeSnapshotContentStatus{}
183+
}
184+
contentClone.Status.Error = &crdv1.VolumeSnapshotError{
207185
Time: &metav1.Time{
208186
Time: time.Now(),
209187
},
210188
Message: &message,
211189
}
212-
contentClone.Status.Error = statusError
213190
ready := false
214191
contentClone.Status.ReadyToUse = &ready
215192
newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().UpdateStatus(contentClone)
@@ -233,7 +210,7 @@ func (ctrl *csiSnapshotSideCarController) updateContentErrorStatusWithEvent(cont
233210

234211
func (ctrl *csiSnapshotSideCarController) getCSISnapshotInput(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotClass, map[string]string, error) {
235212
className := content.Spec.VolumeSnapshotClassName
236-
klog.V(5).Infof("getCSISnapshotInput for content [%s]: VolumeSnapshotClassName [%s]", content.Name, *className)
213+
klog.V(5).Infof("getCSISnapshotInput for content [%s]", content.Name)
237214
var class *crdv1.VolumeSnapshotClass
238215
var err error
239216
if className != nil {
@@ -358,6 +335,7 @@ func (ctrl *csiSnapshotSideCarController) deleteCSISnapshotOperation(content *cr
358335

359336
_, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content)
360337
if err != nil {
338+
ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to get snapshot class or credentials")
361339
return fmt.Errorf("failed to get input parameters to delete snapshot for content %s: %q", content.Name, err)
362340
}
363341

@@ -366,10 +344,42 @@ func (ctrl *csiSnapshotSideCarController) deleteCSISnapshotOperation(content *cr
366344
ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to delete snapshot")
367345
return fmt.Errorf("failed to delete snapshot %#v, err: %v", content.Name, err)
368346
}
369-
347+
// the snapshot has been deleted from the underlying storage system, update
348+
// content status to remove snapshot handle etc.
349+
newContent, err := ctrl.clearVolumeContentStatus(content.Name)
350+
if err != nil {
351+
ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to clear content status")
352+
return err
353+
}
354+
// update local cache
355+
ctrl.updateContentInCacheStore(newContent)
370356
return nil
371357
}
372358

359+
// clearVolumeContentStatus resets all fields to nil related to a snapshot in
360+
// content.Status. On success, the latest version of the content object will be
361+
// returned.
362+
func (ctrl *csiSnapshotSideCarController) clearVolumeContentStatus(
363+
contentName string) (*crdv1.VolumeSnapshotContent, error) {
364+
klog.V(5).Infof("cleanVolumeSnapshotStatus content [%s]", contentName)
365+
// get the latest version from API server
366+
content, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Get(contentName, metav1.GetOptions{})
367+
if err != nil {
368+
return nil, fmt.Errorf("error get snapshot content %s from api server: %v", contentName, err)
369+
}
370+
if content.Status != nil {
371+
content.Status.SnapshotHandle = nil
372+
content.Status.ReadyToUse = nil
373+
content.Status.CreationTime = nil
374+
content.Status.RestoreSize = nil
375+
}
376+
newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().UpdateStatus(content)
377+
if err != nil {
378+
return nil, newControllerUpdateError(contentName, err.Error())
379+
}
380+
return newContent, nil
381+
}
382+
373383
func (ctrl *csiSnapshotSideCarController) updateSnapshotContentStatus(
374384
content *crdv1.VolumeSnapshotContent,
375385
snapshotHandle string,
@@ -497,8 +507,13 @@ func (ctrl *csiSnapshotSideCarController) GetCredentialsFromAnnotation(content *
497507
return snapshotterCredentials, nil
498508
}
499509

500-
// removeContentFinalizer removes a Finalizer for VolumeSnapshotContent.
510+
// removeContentFinalizer removes the VolumeSnapshotContentFinalizer from a
511+
// content if there exists one.
501512
func (ctrl csiSnapshotSideCarController) removeContentFinalizer(content *crdv1.VolumeSnapshotContent) error {
513+
if !slice.ContainsString(content.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer, nil) {
514+
// the finalizer does not exit, return directly
515+
return nil
516+
}
502517
contentClone := content.DeepCopy()
503518
contentClone.ObjectMeta.Finalizers = slice.RemoveString(contentClone.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer, nil)
504519

@@ -507,12 +522,11 @@ func (ctrl csiSnapshotSideCarController) removeContentFinalizer(content *crdv1.V
507522
return newControllerUpdateError(content.Name, err.Error())
508523
}
509524

525+
klog.V(5).Infof("Removed protection finalizer from volume snapshot content %s", content.Name)
510526
_, err = ctrl.storeContentUpdate(contentClone)
511527
if err != nil {
512528
klog.Errorf("failed to update content store %v", err)
513529
}
514-
515-
klog.V(5).Infof("Removed protection finalizer from volume snapshot content %s", content.Name)
516530
return nil
517531
}
518532

@@ -524,7 +538,7 @@ func (ctrl *csiSnapshotSideCarController) shouldDelete(content *crdv1.VolumeSnap
524538
if content.ObjectMeta.DeletionTimestamp == nil {
525539
return false
526540
}
527-
// 1) shouldDelete returns true if content is not bound
541+
// 1) shouldDelete returns true if a content is not bound
528542
// (VolumeSnapshotRef.UID == "") for pre-provisioned snapshot
529543
if content.Spec.Source.SnapshotHandle != nil && content.Spec.VolumeSnapshotRef.UID == "" {
530544
return true

0 commit comments

Comments
 (0)