hostpath: Add block volume support

This change adds block volume support to hostpath driver.

When a block volume request is received, a block file is created at
provisionRoot with the requested capacity as size and a loop device is
created associated with the block file.

At node publish, a bind mount of the loop device is created at the
publish target path.

At node unpublish, the target path is unmounted and deleted.

At volume delete, loop device is disassociated and the block file is
deleted.

Add plugins-dir to hostpath plugin daemonset
The volume publish target path for block devices are usually under
/var/lib/kubelet/plugins directory. Hence, adding plugins directory to
the pod volumes with bidirectional mount propagation.

Run the plugin as privileged to use loop devices
In order to share loop devices with the host, the plugin container must
be run as a privileged container.

Author: darkowlzz

Dockerfile

@@ -2,5 +2,7 @@ FROM alpine
 LABEL maintainers="Kubernetes Authors"
 LABEL description="HostPath Driver"
 
+# Add util-linux to get a new version of losetup.
+RUN apk add util-linux
 COPY ./bin/hostpathplugin /hostpathplugin
 ENTRYPOINT ["/hostpathplugin"]

Gopkg.lock

@@ -24,6 +24,8 @@ spec:
             - --v=5
             - --csi-address=/csi/csi.sock
             - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock
+          securityContext:
+            privileged: true
           env:
             - name: KUBE_NODE_NAME
               valueFrom:
@@ -59,6 +61,9 @@ spec:
             - mountPath: /var/lib/kubelet/pods
               mountPropagation: Bidirectional
               name: mountpoint-dir
+            - mountPath: /var/lib/kubelet/plugins
+              mountPropagation: Bidirectional
+              name: plugins-dir
       volumes:
         - hostPath:
             path: /var/lib/kubelet/plugins/csi-hostpath
@@ -72,3 +77,7 @@ spec:
             path: /var/lib/kubelet/plugins_registry
             type: Directory
           name: registration-dir
+        - hostPath:
+            path: /var/lib/kubelet/plugins
+            type: Directory
+          name: plugins-dir

deploy/hostpath/csi-hostpath-plugin.yaml

@@ -32,6 +32,7 @@ import (
 	"google.golang.org/grpc/status"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
+	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
 	utilexec "k8s.io/utils/exec"
 )
 
@@ -42,6 +43,13 @@ const (
 	maxStorageCapacity = tib
 )
 
+type accessType int
+
+const (
+	mountAccess accessType = iota
+	blockAccess
+)
+
 type controllerServer struct {
 	caps []*csi.ControllerServiceCapability
 }
@@ -67,9 +75,41 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	if len(req.GetName()) == 0 {
 		return nil, status.Error(codes.InvalidArgument, "Name missing in request")
 	}
-	if req.GetVolumeCapabilities() == nil {
+	caps := req.GetVolumeCapabilities()
+	if caps == nil {
 		return nil, status.Error(codes.InvalidArgument, "Volume Capabilities missing in request")
 	}
+
+	// Keep a record of the requested access types.
+	var accessTypeMount, accessTypeBlock bool
+
+	for _, cap := range caps {
+		if cap.GetBlock() != nil {
+			accessTypeBlock = true
+		}
+		if cap.GetMount() != nil {
+			accessTypeMount = true
+		}
+	}
+	// A real driver would also need to check that the other
+	// fields in VolumeCapabilities are sane. The check above is
+	// just enough to pass the "[Testpattern: Dynamic PV (block
+	// volmode)] volumeMode should fail in binding dynamic
+	// provisioned PV to PVC" storage E2E test.
+
+	if accessTypeBlock && accessTypeMount {
+		return nil, status.Error(codes.InvalidArgument, "cannot have both block and mount access type")
+	}
+
+	var requestedAccessType accessType
+
+	if accessTypeBlock {
+		requestedAccessType = blockAccess
+	} else {
+		// Default to mount.
+		requestedAccessType = mountAccess
+	}
+
 	// Need to check for already existing volume name, and if found
 	// check for the requested capacity and already allocated capacity
 	if exVol, err := getVolumeByName(req.GetName()); err == nil {
@@ -94,13 +134,35 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	if capacity >= maxStorageCapacity {
 		return nil, status.Errorf(codes.OutOfRange, "Requested capacity %d exceeds maximum allowed %d", capacity, maxStorageCapacity)
 	}
+
 	volumeID := uuid.NewUUID().String()
 	path := provisionRoot + volumeID
-	err := os.MkdirAll(path, 0777)
-	if err != nil {
-		glog.V(3).Infof("failed to create volume: %v", err)
-		return nil, err
+
+	switch requestedAccessType {
+	case blockAccess:
+		executor := utilexec.New()
+		size := fmt.Sprintf("%dM", capacity/mib)
+		// Create a block file.
+		out, err := executor.Command("fallocate", "-l", size, path).CombinedOutput()
+		if err != nil {
+			glog.V(3).Infof("failed to create block device: %v", string(out))
+			return nil, err
+		}
+
+		// Associate block file with the loop device.
+		volPathHandler := volumepathhandler.VolumePathHandler{}
+		_, err = volPathHandler.AttachFileDevice(path)
+		if err != nil {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("failed to attach device: %v", err))
+		}
+	case mountAccess:
+		err := os.MkdirAll(path, 0777)
+		if err != nil {
+			glog.V(3).Infof("failed to create volume: %v", err)
+			return nil, err
+		}
 	}
+
 	if req.GetVolumeContentSource() != nil {
 		contentSource := req.GetVolumeContentSource()
 		if contentSource.GetSnapshot() != nil {
@@ -127,6 +189,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	hostPathVol.VolID = volumeID
 	hostPathVol.VolSize = capacity
 	hostPathVol.VolPath = path
+	hostPathVol.VolAccessType = requestedAccessType
 	hostPathVolumes[volumeID] = hostPathVol
 	return &csi.CreateVolumeResponse{
 		Volume: &csi.Volume{
@@ -148,11 +211,34 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
 		glog.V(3).Infof("invalid delete volume req: %v", req)
 		return nil, err
 	}
-	volumeID := req.VolumeId
-	glog.V(4).Infof("deleting volume %s", volumeID)
-	path := provisionRoot + volumeID
-	os.RemoveAll(path)
-	delete(hostPathVolumes, volumeID)
+
+	vol, err := getVolumeByID(req.GetVolumeId())
+	if err != nil {
+		// Return OK if the volume is not found.
+		return &csi.DeleteVolumeResponse{}, nil
+	}
+	glog.V(4).Infof("deleting volume %s", vol.VolID)
+
+	if vol.VolAccessType == blockAccess {
+
+		volPathHandler := volumepathhandler.VolumePathHandler{}
+		// Get the associated loop device.
+		device, err := volPathHandler.GetLoopDevice(provisionRoot + vol.VolID)
+		if err != nil {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("failed to get the loop device: %v", err))
+		}
+
+		if device != "" {
+			// Remove any associated loop device.
+			glog.V(4).Infof("deleting loop device %s", device)
+			if err := volPathHandler.RemoveLoopDevice(device); err != nil {
+				return nil, status.Error(codes.Internal, fmt.Sprintf("failed to remove loop device: %v", err))
+			}
+		}
+	}
+
+	os.RemoveAll(vol.VolPath)
+	delete(hostPathVolumes, vol.VolID)
 	return &csi.DeleteVolumeResponse{}, nil
 }
 

pkg/hostpath/controllerserver.go

@@ -45,10 +45,11 @@ type hostPath struct {
 }
 
 type hostPathVolume struct {
-	VolName string `json:"volName"`
-	VolID   string `json:"volID"`
-	VolSize int64  `json:"volSize"`
-	VolPath string `json:"volPath"`
+	VolName       string     `json:"volName"`
+	VolID         string     `json:"volID"`
+	VolSize       int64      `json:"volSize"`
+	VolPath       string     `json:"volPath"`
+	VolAccessType accessType `json:"volAccessType"`
 }
 
 type hostPathSnapshot struct {