test: add blobfuse-proxy build & e2e test

.github/workflows/linux.yaml

@@ -41,7 +41,7 @@ jobs:
       run: |
         export PATH=$PATH:$HOME/.local/bin
         make blobfuse-proxy
-        sudo dpkg -i pkg/blobfuse-proxy/debpackage.deb
+        sudo dpkg -i deploy/blobfuse-proxy/blobfuse-proxy.deb
 
     - name: Get code coverage
       env:

Makefile

@@ -77,9 +77,6 @@ e2e-test:
 e2e-bootstrap: install-helm
 	# Only build and push the image if it does not exist in the registry
 	docker pull $(IMAGE_TAG) || make blob-container push
-	if [ ! -z "$(ENABLE_BLOBFUSE_PROXY)" ]; then \
-		make install-blobfuse-proxy;\
-	fi
 	helm install blob-csi-driver ./charts/latest/blob-csi-driver --namespace kube-system --wait --timeout=15m -v=5 --debug \
 		--set controller.runOnMaster=true \
 		--set controller.replicas=1 \
@@ -107,7 +104,7 @@ blob-darwin:
 	CGO_ENABLED=0 GOOS=darwin go build -a -ldflags ${LDFLAGS} -mod vendor -o _output/blobplugin ./pkg/blobplugin
 
 .PHONY: container
-container: blob
+container: blob blobfuse-proxy
 	docker build -t $(IMAGE_TAG) --output=type=docker -f ./pkg/blobplugin/Dockerfile .
 
 .PHONY: container-linux
@@ -116,7 +113,7 @@ container-linux:
 		-t $(IMAGE_TAG)-linux-$(ARCH) --build-arg ARCH=$(ARCH) -f ./pkg/blobplugin/Dockerfile .
 
 .PHONY: blob-container
-blob-container:
+blob-container: blobfuse-proxy
 	docker buildx rm container-builder || true
 	docker buildx create --use --name=container-builder
 
@@ -172,16 +169,8 @@ delete-metrics-svc:
 blobfuse-proxy:
 	mkdir -p ./pkg/blobfuse-proxy/debpackage/usr/bin/
 	CGO_ENABLED=0 GOOS=linux go build -mod vendor -ldflags="-s -w" -o ./pkg/blobfuse-proxy/debpackage/usr/bin/blobfuse-proxy ./pkg/blobfuse-proxy
-	dpkg-deb --build pkg/blobfuse-proxy/debpackage
+	dpkg-deb --build pkg/blobfuse-proxy/debpackage deploy/blobfuse-proxy/blobfuse-proxy.deb
 
 .PHONY: blobfuse-proxy-container
 blobfuse-proxy-container:
 	sudo docker build -t blobfuse-proxy -f pkg/blobfuse-proxy/Dockerfile .
-
-.PHONY: install-blobfuse-proxy
-install-blobfuse-proxy:
-	kubectl apply -f ./deploy/blobfuse-proxy/blobfuse-proxy.yaml
-
-.PHONY: uninstall-blobfuse-proxy
-uninstall-blobfuse-proxy:
-	kubectl delete -f ./deploy/blobfuse-proxy/blobfuse-proxy.yaml --ignore-not-found

charts/latest/blob-csi-driver/templates/blobfuse-proxy.yaml

@@ -0,0 +1,90 @@
+{{- if .Values.node.enableBlobfuseProxy -}}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: csi-blobfuse-proxy
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: csi-blobfuse-proxy
+  template:
+    metadata:
+      labels:
+        app: csi-blobfuse-proxy
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: type
+                    operator: NotIn
+                    values:
+                      - virtual-kubelet
+      initContainers:
+        - name: prepare-binaries
+          image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}"
+          command: ['sh', '-c', "cp /blobfuse-proxy/*.deb /tmp/"]
+          volumeMounts:
+            - mountPath: /tmp
+              name: tmp-dir
+      containers:
+        - command:
+            - nsenter
+            - '--target'
+            - '1'
+            - '--mount'
+            - '--uts'
+            - '--ipc'
+            - '--net'
+            - '--pid'
+            - '--'
+            - sh
+            - '-c'
+            - |
+              set -xe
+              dpkg -i /tmp/packages-microsoft-prod.deb
+              apt-get install -y blobfuse
+              dpkg -i /tmp/blobfuse-proxy.deb
+              rm -f /tmp/packages-microsoft-prod.deb /tmp/blobfuse-proxy.deb
+              mkdir -p /var/lib/kubelet/plugins/blob.csi.azure.com
+              echo "set max open file num"
+              sysctl -w fs.file-max=9000000
+              echo "Enabling blobfuse proxy systemctl service"
+              systemctl daemon-reload
+              systemctl enable blobfuse-proxy
+              systemctl start blobfuse-proxy
+              echo "waiting for blobfuse-proxy service to start"
+              sleep 3s
+              # tail blobfuse proxy logs
+              journalctl -u blobfuse-proxy -f
+          image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}"
+          imagePullPolicy: IfNotPresent
+          name: sysctl-install-blobfuse-proxy
+          env:
+            - name: "DEBIAN_FRONTEND"
+              value: "noninteractive"
+          resources:
+            limits:
+              cpu: 200m
+              memory: 200Mi
+            requests:
+              cpu: 10m
+              memory: 1Mi
+          securityContext:
+            privileged: true
+      hostNetwork: true
+      hostPID: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      tolerations:
+        - operator: Exists
+      volumes:
+        - hostPath:
+            path: /tmp
+            type: DirectoryOrCreate
+          name: tmp-dir
+{{ end }}

deploy/blobfuse-proxy.yaml

@@ -0,0 +1,89 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: csi-blobfuse-proxy
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: csi-blobfuse-proxy
+  template:
+    metadata:
+      labels:
+        app: csi-blobfuse-proxy
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: type
+                    operator: NotIn
+                    values:
+                      - virtual-kubelet
+      initContainers:
+        - name: prepare-binaries
+          image: mcr.microsoft.com/k8s/csi/blob-csi:latest
+          command: ['sh', '-c', "cp /blobfuse-proxy/*.deb /tmp/"]
+          volumeMounts:
+            - mountPath: /tmp
+              name: tmp-dir
+      containers:
+        - command:
+            - nsenter
+            - '--target'
+            - '1'
+            - '--mount'
+            - '--uts'
+            - '--ipc'
+            - '--net'
+            - '--pid'
+            - '--'
+            - sh
+            - '-c'
+            - |
+              set -xe
+              dpkg -i /tmp/packages-microsoft-prod.deb
+              apt-get install -y blobfuse
+              dpkg -i /tmp/blobfuse-proxy.deb
+              rm -f /tmp/packages-microsoft-prod.deb /tmp/blobfuse-proxy.deb
+              mkdir -p /var/lib/kubelet/plugins/blob.csi.azure.com
+              echo "set max open file num"
+              sysctl -w fs.file-max=9000000
+              echo "Enabling blobfuse proxy systemctl service"
+              systemctl daemon-reload
+              systemctl enable blobfuse-proxy
+              systemctl start blobfuse-proxy
+              echo "waiting for blobfuse-proxy service to start"
+              sleep 3s
+              # tail blobfuse proxy logs
+              journalctl -u blobfuse-proxy -f
+          image: mcr.microsoft.com/k8s/csi/blob-csi:latest
+          imagePullPolicy: IfNotPresent
+          name: sysctl-install-blobfuse-proxy
+          env:
+            - name: "DEBIAN_FRONTEND"
+              value: "noninteractive"
+          resources:
+            limits:
+              cpu: 200m
+              memory: 200Mi
+            requests:
+              cpu: 10m
+              memory: 1Mi
+          securityContext:
+            privileged: true
+      hostNetwork: true
+      hostPID: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      tolerations:
+        - operator: Exists
+      volumes:
+        - hostPath:
+            path: /tmp
+            type: DirectoryOrCreate
+          name: tmp-dir

deploy/blobfuse-proxy/README.md

@@ -1,20 +1,22 @@
 # Blobfuse Proxy
  - supported CSI driver version: v1.1.0 or later version
- - only available on debian based OS
+ - only available on debian based agent node
 
 By default, restart csi-blobfuse-node daemonset would make current blobfuse mounts unavailable. When fuse nodeserver restarts on the node, the fuse daemon also restarts, this results in breaking all connections FUSE daemon is maintaining. You could find more details here: [No easy way how to update CSI driver that uses fuse](https://github.com/kubernetes/kubernetes/issues/70013).
 
-This guide shows how to install a blobfuse proxy on all agent nodes and the proxy would mount volumes, maintain  FUSE connections.
+This guide shows how to install a blobfuse proxy on all agent nodes and the proxy would mount volumes, maintain FUSE connections.
 
-### Step#1. Install blobfuse-proxy on debian based agent nodes
-> following daemonset would also install latest [blobfuse](https://github.com/Azure/azure-storage-fuse) version on the node by default.
+### Install Blob CSI driver with `node.enableBlobfuseProxy=true`
+ - helm install
 ```console
-kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/deploy/blobfuse-proxy/blobfuse-proxy.yaml
+helm repo add blob-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/charts
+helm install blob-csi-driver blob-csi-driver/blob-csi-driver --namespace kube-system --version v1.6.0 --set node.enableBlobfuseProxy=true
 ```
 
-### Step#2. Install Blob CSI driver with `node.enableBlobfuseProxy=true` helm chart setting
-> another opition: set `enable-blobfuse-proxy=true` in existing `csi-blob-node` daemonset (default is `false`)
+### Enable blobfuse proxy on existing Blob CSI driver
+ - install blobfuse proxy
+> following config only works on debian based agent node
 ```console
-helm repo add blob-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/charts
-helm install blob-csi-driver blob-csi-driver/blob-csi-driver --namespace kube-system --version v1.4.0 --set node.enableBlobfuseProxy=true
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/deploy/blobfuse-proxy.yaml
 ```
+ - set `enable-blobfuse-proxy=true` in existing `csi-blob-node` daemonset (default is `false`)

deploy/install-driver.sh

@@ -42,7 +42,7 @@ kubectl apply -f $repo/csi-blob-controller.yaml
 if [[ "$#" -gt 1 ]]; then
   if [[ "$2" == *"blobfuse-proxy"* ]]; then
     echo "set enable-blobfuse-proxy as true ..."
-    kubectl apply -f ./deploy/blobfuse-proxy/blobfuse-proxy.yaml
+    kubectl apply -f ./deploy/blobfuse-proxy.yaml
     if [[ "$2" == *"local"* ]]; then
       cat $repo/csi-blob-node.yaml | sed 's/enable-blobfuse-proxy=false/enable-blobfuse-proxy=true/g' | kubectl apply -f -
     else
@@ -51,5 +51,7 @@ if [[ "$#" -gt 1 ]]; then
   else
     kubectl apply -f $repo/csi-blob-node.yaml
   fi
+else
+  kubectl apply -f $repo/csi-blob-node.yaml
 fi
 echo 'Azure Blob Storage CSI driver installed successfully.'

deploy/uninstall-driver.sh

@@ -36,7 +36,7 @@ fi
 if [[ "$#" -gt 1 ]]; then
   if [[ "$2" == *"blobfuse-proxy"* ]]; then
     echo "remove blobfuse-proxy deployment ..."
-    kubectl delete -f ./deploy/blobfuse-proxy/blobfuse-proxy.yaml
+    kubectl delete -f ./deploy/blobfuse-proxy.yaml
   fi
 fi
 

pkg/blobplugin/Dockerfile

@@ -19,7 +19,7 @@ RUN clean-install ca-certificates libfuse-dev libcurl4-gnutls-dev libgnutls28-de
 RUN mkdir /blobfuse-proxy/
 # for compatibility, remove this after v1.6.0 release (todo)
 COPY deploy/blobfuse-proxy/v0.1.0/blobfuse-proxy-v0.1.0.deb /blobfuse-proxy/
-COPY deploy/blobfuse-proxy/v0.1.1/blobfuse-proxy.deb /blobfuse-proxy/
+COPY deploy/blobfuse-proxy/blobfuse-proxy.deb /blobfuse-proxy/
 RUN wget -O /blobfuse-proxy/packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb
 RUN dpkg -i /blobfuse-proxy/packages-microsoft-prod.deb && apt update && apt install blobfuse fuse -y
 # this is a workaround to install nfs-kernel-server and don't quit with error