use feature gate for toggling FsResizeAnnotation feature on/off

Author: sunpa93

cmd/csi-resizer/main.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/kubernetes-csi/csi-lib-utils/metrics"
@@ -38,7 +39,9 @@ import (
 	"github.com/kubernetes-csi/external-resizer/pkg/util"
 
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
+	cflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog/v2"
 )
 
@@ -68,12 +71,13 @@ var (
 
 	handleVolumeInUseError = flag.Bool("handle-volume-inuse-error", true, "Flag to turn on/off capability to handle volume in use error in resizer controller. Defaults to true if not set.")
 
-	enableFSResizeAnnotation = flag.Bool("enable-fs-resize-annotation", true, "Flag to turn on/off capability to add an annotation to PV to denote its size prior to resize and to remove it upon completion of filesystem resize.")
+	featureGates map[string]bool
 
 	version = "unknown"
 )
 
 func main() {
+	flag.Var(cflag.NewMapStringBool(&featureGates), "feature-gates", "A set of key=value paris that describe feature gates for alpha/experimental features for csi external resizer."+"Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n"))
 	klog.InitFlags(nil)
 	flag.Set("logtostderr", "true")
 	flag.Parse()
@@ -92,6 +96,9 @@ func main() {
 	if addr == "" {
 		addr = *httpEndpoint
 	}
+	if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(featureGates); err != nil {
+		klog.Fatal(err)
+	}
 
 	var config *rest.Config
 	var err error
@@ -155,7 +162,7 @@ func main() {
 	resizerName := csiResizer.Name()
 	rc := controller.NewResizeController(resizerName, csiResizer, kubeClient, *resyncPeriod, informerFactory,
 		workqueue.NewItemExponentialFailureRateLimiter(*retryIntervalStart, *retryIntervalMax),
-		*handleVolumeInUseError, *enableFSResizeAnnotation)
+		*handleVolumeInUseError)
 	run := func(ctx context.Context) {
 		informerFactory.Start(wait.NeverStop)
 		rc.Run(*workers, ctx)

pkg/controller/controller.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"time"
 
+	features "sigs.k8s.io/external-resizer/pkg/features" /* TO BE REPLACED WITH features "github.com/external-resizer/pkg/features once PR merged*/
+
 	"github.com/kubernetes-csi/external-resizer/pkg/resizer"
 	"github.com/kubernetes-csi/external-resizer/pkg/util"
 	"google.golang.org/grpc/codes"
@@ -31,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -66,9 +69,8 @@ type resizeController struct {
 	// a cache to store PersistentVolume objects
 	volumes cache.Store
 	// a cache to store PersistentVolumeClaim objects
-	claims                   cache.Store
-	handleVolumeInUseError   bool
-	enableFSResizeAnnotation bool
+	claims                 cache.Store
+	handleVolumeInUseError bool
 }
 
 // NewResizeController returns a ResizeController.
@@ -79,8 +81,7 @@ func NewResizeController(
 	resyncPeriod time.Duration,
 	informerFactory informers.SharedInformerFactory,
 	pvcRateLimiter workqueue.RateLimiter,
-	handleVolumeInUseError,
-	enableFSResizeAnnotation bool) ResizeController {
+	handleVolumeInUseError bool) ResizeController {
 	pvInformer := informerFactory.Core().V1().PersistentVolumes()
 	pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims()
 	eventBroadcaster := record.NewBroadcaster()
@@ -93,18 +94,17 @@ func NewResizeController(
 		pvcRateLimiter, fmt.Sprintf("%s-pvc", name))
 
 	ctrl := &resizeController{
-		name:                     name,
-		resizer:                  resizer,
-		kubeClient:               kubeClient,
-		pvSynced:                 pvInformer.Informer().HasSynced,
-		pvcSynced:                pvcInformer.Informer().HasSynced,
-		claimQueue:               claimQueue,
-		volumes:                  pvInformer.Informer().GetStore(),
-		claims:                   pvcInformer.Informer().GetStore(),
-		eventRecorder:            eventRecorder,
-		usedPVCs:                 newUsedPVCStore(),
-		handleVolumeInUseError:   handleVolumeInUseError,
-		enableFSResizeAnnotation: enableFSResizeAnnotation,
+		name:                   name,
+		resizer:                resizer,
+		kubeClient:             kubeClient,
+		pvSynced:               pvInformer.Informer().HasSynced,
+		pvcSynced:              pvcInformer.Informer().HasSynced,
+		claimQueue:             claimQueue,
+		volumes:                pvInformer.Informer().GetStore(),
+		claims:                 pvcInformer.Informer().GetStore(),
+		eventRecorder:          eventRecorder,
+		usedPVCs:               newUsedPVCStore(),
+		handleVolumeInUseError: handleVolumeInUseError,
 	}
 
 	// Add a resync period as the PVC's request size can be resized again when we handling
@@ -324,7 +324,7 @@ func (ctrl *resizeController) syncPVC(key string) error {
 		return fmt.Errorf("expected volume but got %+v", volumeObj)
 	}
 
-	if ctrl.enableFSResizeAnnotation && ctrl.isNodeExpandComplete(pvc, pv) && metav1.HasAnnotation(pv.ObjectMeta, util.AnnPreResizeCapacity) {
+	if utilfeature.DefaultFeatureGate.Enabled(features.AnnotateFsResize) && ctrl.isNodeExpandComplete(pvc, pv) && metav1.HasAnnotation(pv.ObjectMeta, util.AnnPreResizeCapacity) {
 		if err := ctrl.deletePreResizeCapAnnotation(pv); err != nil {
 			return fmt.Errorf("failed removing annotation %s from pv %q: %v", util.AnnPreResizeCapacity, pv.Name, err)
 		}
@@ -571,7 +571,7 @@ func (ctrl *resizeController) updatePVCapacity(pv *v1.PersistentVolume, oldCapac
 	newPV := pv.DeepCopy()
 	newPV.Spec.Capacity[v1.ResourceStorage] = newCapacity
 
-	if ctrl.enableFSResizeAnnotation && fsResizeRequired {
+	if utilfeature.DefaultFeatureGate.Enabled(features.AnnotateFsResize) && fsResizeRequired {
 		// only update annotation if there already isn't one
 		if !metav1.HasAnnotation(pv.ObjectMeta, util.AnnPreResizeCapacity) {
 			if newPV.ObjectMeta.Annotations == nil {

pkg/controller/controller_test.go

@@ -5,6 +5,8 @@ import (
 	"testing"
 	"time"
 
+	features "sigs.k8s.io/external-resizer/pkg/features" /* TO BE REPLACED WITH features "github.com/external-resizer/pkg/features once PR merged*/
+
 	"k8s.io/client-go/util/workqueue"
 
 	"github.com/kubernetes-csi/external-resizer/pkg/csi"
@@ -16,9 +18,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 )
 
 func TestController(t *testing.T) {
@@ -234,7 +238,8 @@ func TestController(t *testing.T) {
 			t.Fatalf("Test %s: Unable to create resizer: %v", test.Name, err)
 		}
 
-		controller := NewResizeController(driverName, csiResizer, kubeClient, time.Second, informerFactory, workqueue.DefaultControllerRateLimiter(), !test.disableVolumeInUseErrorHandler, test.enableFSResizeAnnotation)
+		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AnnotateFsResize, true)()
+		controller := NewResizeController(driverName, csiResizer, kubeClient, time.Second, informerFactory, workqueue.DefaultControllerRateLimiter(), !test.disableVolumeInUseErrorHandler)
 
 		ctrlInstance, _ := controller.(*resizeController)
 
@@ -365,7 +370,8 @@ func TestResizePVC(t *testing.T) {
 			t.Fatalf("Test %s: Unable to create resizer: %v", test.Name, err)
 		}
 
-		controller := NewResizeController(driverName, csiResizer, kubeClient, time.Second, informerFactory, workqueue.DefaultControllerRateLimiter(), true /* disableVolumeInUseErrorHandler*/, true /*enableFsResizeAnnotation*/)
+		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AnnotateFsResize, true)()
+		controller := NewResizeController(driverName, csiResizer, kubeClient, time.Second, informerFactory, workqueue.DefaultControllerRateLimiter(), true /* disableVolumeInUseErrorHandler*/)
 
 		ctrlInstance, _ := controller.(*resizeController)
 

pkg/features/features.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/component-base/featuregate"
+)
+
+const (
+	// owner: @sunpa93
+	// alpha: v1.22
+	AnnotateFsResize featuregate.Feature = "AnnotateFsResize"
+)
+
+func init() {
+	utilfeature.DefaultMutableFeatureGate.Add(defaultResizerFeatureGates)
+}
+
+var defaultResizerFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
+	AnnotateFsResize: {Default: false, PreRelease: featuregate.Alpha},
+}