Skip to content

Commit c3f77d3

Browse files
andyzhangxandyzhangx
authored
Merge pull request #541 from andyzhangx/blobfuse-proxy-changedir
test: add blobfuse-proxy build & e2e test
2 parents 69c3767 + 7d07ff2 commit c3f77d3

File tree

10 files changed

+199
-112
lines changed

10 files changed

+199
-112
lines changed

‎.github/workflows/linux.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
run: |
4242
export PATH=$PATH:$HOME/.local/bin
4343
make blobfuse-proxy
44-
sudo dpkg -i pkg/blobfuse-proxy/debpackage.deb
44+
sudo dpkg -i deploy/blobfuse-proxy/blobfuse-proxy.deb
4545
4646
- name: Get code coverage
4747
env:

‎Makefile

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,6 @@ e2e-test:
7777
e2e-bootstrap: install-helm
7878
# Only build and push the image if it does not exist in the registry
7979
docker pull $(IMAGE_TAG) || make blob-container push
80-
if [ ! -z "$(ENABLE_BLOBFUSE_PROXY)" ]; then \
81-
make install-blobfuse-proxy;\
82-
fi
8380
helm install blob-csi-driver ./charts/latest/blob-csi-driver --namespace kube-system --wait --timeout=15m -v=5 --debug \
8481
--set controller.runOnMaster=true \
8582
--set controller.replicas=1 \
@@ -107,7 +104,7 @@ blob-darwin:
107104
CGO_ENABLED=0 GOOS=darwin go build -a -ldflags ${LDFLAGS} -mod vendor -o _output/blobplugin ./pkg/blobplugin
108105

109106
.PHONY: container
110-
container: blob
107+
container: blob blobfuse-proxy
111108
docker build -t $(IMAGE_TAG) --output=type=docker -f ./pkg/blobplugin/Dockerfile .
112109

113110
.PHONY: container-linux
@@ -116,7 +113,7 @@ container-linux:
116113
-t $(IMAGE_TAG)-linux-$(ARCH) --build-arg ARCH=$(ARCH) -f ./pkg/blobplugin/Dockerfile .
117114

118115
.PHONY: blob-container
119-
blob-container:
116+
blob-container: blobfuse-proxy
120117
docker buildx rm container-builder || true
121118
docker buildx create --use --name=container-builder
122119

@@ -172,16 +169,8 @@ delete-metrics-svc:
172169
blobfuse-proxy:
173170
mkdir -p ./pkg/blobfuse-proxy/debpackage/usr/bin/
174171
CGO_ENABLED=0 GOOS=linux go build -mod vendor -ldflags="-s -w" -o ./pkg/blobfuse-proxy/debpackage/usr/bin/blobfuse-proxy ./pkg/blobfuse-proxy
175-
dpkg-deb --build pkg/blobfuse-proxy/debpackage
172+
dpkg-deb --build pkg/blobfuse-proxy/debpackage deploy/blobfuse-proxy/blobfuse-proxy.deb
176173

177174
.PHONY: blobfuse-proxy-container
178175
blobfuse-proxy-container:
179176
sudo docker build -t blobfuse-proxy -f pkg/blobfuse-proxy/Dockerfile .
180-
181-
.PHONY: install-blobfuse-proxy
182-
install-blobfuse-proxy:
183-
kubectl apply -f ./deploy/blobfuse-proxy/blobfuse-proxy.yaml
184-
185-
.PHONY: uninstall-blobfuse-proxy
186-
uninstall-blobfuse-proxy:
187-
kubectl delete -f ./deploy/blobfuse-proxy/blobfuse-proxy.yaml --ignore-not-found

‎charts/latest/blob-csi-driver-v1.6.0.tgz

580 Bytes
Binary file not shown.

‎charts/latest/blob-csi-driver/templates/blobfuse-proxy.yaml

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
{{- if .Values.node.enableBlobfuseProxy -}}
2+
apiVersion: apps/v1
3+
kind: DaemonSet
4+
metadata:
5+
name: csi-blobfuse-proxy
6+
namespace: kube-system
7+
spec:
8+
selector:
9+
matchLabels:
10+
app: csi-blobfuse-proxy
11+
template:
12+
metadata:
13+
labels:
14+
app: csi-blobfuse-proxy
15+
spec:
16+
affinity:
17+
nodeAffinity:
18+
requiredDuringSchedulingIgnoredDuringExecution:
19+
nodeSelectorTerms:
20+
- matchExpressions:
21+
- key: type
22+
operator: NotIn
23+
values:
24+
- virtual-kubelet
25+
initContainers:
26+
- name: prepare-binaries
27+
image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}"
28+
command: ['sh', '-c', "cp /blobfuse-proxy/*.deb /tmp/"]
29+
volumeMounts:
30+
- mountPath: /tmp
31+
name: tmp-dir
32+
containers:
33+
- command:
34+
- nsenter
35+
- '--target'
36+
- '1'
37+
- '--mount'
38+
- '--uts'
39+
- '--ipc'
40+
- '--net'
41+
- '--pid'
42+
- '--'
43+
- sh
44+
- '-c'
45+
- |
46+
set -xe
47+
dpkg -i /tmp/packages-microsoft-prod.deb
48+
apt-get install -y blobfuse
49+
dpkg -i /tmp/blobfuse-proxy.deb
50+
rm -f /tmp/packages-microsoft-prod.deb /tmp/blobfuse-proxy.deb
51+
mkdir -p /var/lib/kubelet/plugins/blob.csi.azure.com
52+
echo "set max open file num"
53+
sysctl -w fs.file-max=9000000
54+
echo "Enabling blobfuse proxy systemctl service"
55+
systemctl daemon-reload
56+
systemctl enable blobfuse-proxy
57+
systemctl start blobfuse-proxy
58+
echo "waiting for blobfuse-proxy service to start"
59+
sleep 3s
60+
# tail blobfuse proxy logs
61+
journalctl -u blobfuse-proxy -f
62+
image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}"
63+
imagePullPolicy: IfNotPresent
64+
name: sysctl-install-blobfuse-proxy
65+
env:
66+
- name: "DEBIAN_FRONTEND"
67+
value: "noninteractive"
68+
resources:
69+
limits:
70+
cpu: 200m
71+
memory: 200Mi
72+
requests:
73+
cpu: 10m
74+
memory: 1Mi
75+
securityContext:
76+
privileged: true
77+
hostNetwork: true
78+
hostPID: true
79+
nodeSelector:
80+
kubernetes.io/os: linux
81+
priorityClassName: system-node-critical
82+
restartPolicy: Always
83+
tolerations:
84+
- operator: Exists
85+
volumes:
86+
- hostPath:
87+
path: /tmp
88+
type: DirectoryOrCreate
89+
name: tmp-dir
90+
{{ end }}

‎deploy/blobfuse-proxy.yaml

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
---
2+
apiVersion: apps/v1
3+
kind: DaemonSet
4+
metadata:
5+
name: csi-blobfuse-proxy
6+
namespace: kube-system
7+
spec:
8+
selector:
9+
matchLabels:
10+
app: csi-blobfuse-proxy
11+
template:
12+
metadata:
13+
labels:
14+
app: csi-blobfuse-proxy
15+
spec:
16+
affinity:
17+
nodeAffinity:
18+
requiredDuringSchedulingIgnoredDuringExecution:
19+
nodeSelectorTerms:
20+
- matchExpressions:
21+
- key: type
22+
operator: NotIn
23+
values:
24+
- virtual-kubelet
25+
initContainers:
26+
- name: prepare-binaries
27+
image: mcr.microsoft.com/k8s/csi/blob-csi:latest
28+
command: ['sh', '-c', "cp /blobfuse-proxy/*.deb /tmp/"]
29+
volumeMounts:
30+
- mountPath: /tmp
31+
name: tmp-dir
32+
containers:
33+
- command:
34+
- nsenter
35+
- '--target'
36+
- '1'
37+
- '--mount'
38+
- '--uts'
39+
- '--ipc'
40+
- '--net'
41+
- '--pid'
42+
- '--'
43+
- sh
44+
- '-c'
45+
- |
46+
set -xe
47+
dpkg -i /tmp/packages-microsoft-prod.deb
48+
apt-get install -y blobfuse
49+
dpkg -i /tmp/blobfuse-proxy.deb
50+
rm -f /tmp/packages-microsoft-prod.deb /tmp/blobfuse-proxy.deb
51+
mkdir -p /var/lib/kubelet/plugins/blob.csi.azure.com
52+
echo "set max open file num"
53+
sysctl -w fs.file-max=9000000
54+
echo "Enabling blobfuse proxy systemctl service"
55+
systemctl daemon-reload
56+
systemctl enable blobfuse-proxy
57+
systemctl start blobfuse-proxy
58+
echo "waiting for blobfuse-proxy service to start"
59+
sleep 3s
60+
# tail blobfuse proxy logs
61+
journalctl -u blobfuse-proxy -f
62+
image: mcr.microsoft.com/k8s/csi/blob-csi:latest
63+
imagePullPolicy: IfNotPresent
64+
name: sysctl-install-blobfuse-proxy
65+
env:
66+
- name: "DEBIAN_FRONTEND"
67+
value: "noninteractive"
68+
resources:
69+
limits:
70+
cpu: 200m
71+
memory: 200Mi
72+
requests:
73+
cpu: 10m
74+
memory: 1Mi
75+
securityContext:
76+
privileged: true
77+
hostNetwork: true
78+
hostPID: true
79+
nodeSelector:
80+
kubernetes.io/os: linux
81+
priorityClassName: system-node-critical
82+
restartPolicy: Always
83+
tolerations:
84+
- operator: Exists
85+
volumes:
86+
- hostPath:
87+
path: /tmp
88+
type: DirectoryOrCreate
89+
name: tmp-dir

‎deploy/blobfuse-proxy/README.md

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
11
# Blobfuse Proxy
22
- supported CSI driver version: v1.1.0 or later version
3-
- only available on debian based OS
3+
- only available on debian based agent node
44

55
By default, restart csi-blobfuse-node daemonset would make current blobfuse mounts unavailable. When fuse nodeserver restarts on the node, the fuse daemon also restarts, this results in breaking all connections FUSE daemon is maintaining. You could find more details here: [No easy way how to update CSI driver that uses fuse](https://github.com/kubernetes/kubernetes/issues/70013).
66

7-
This guide shows how to install a blobfuse proxy on all agent nodes and the proxy would mount volumes, maintain FUSE connections.
7+
This guide shows how to install a blobfuse proxy on all agent nodes and the proxy would mount volumes, maintain FUSE connections.
88

9-
### Step#1. Install blobfuse-proxy on debian based agent nodes
10-
> following daemonset would also install latest [blobfuse](https://github.com/Azure/azure-storage-fuse) version on the node by default.
9+
### Install Blob CSI driver with `node.enableBlobfuseProxy=true`
10+
- helm install
1111
```console
12-
kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/deploy/blobfuse-proxy/blobfuse-proxy.yaml
12+
helm repo add blob-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/charts
13+
helm install blob-csi-driver blob-csi-driver/blob-csi-driver --namespace kube-system --version v1.6.0 --set node.enableBlobfuseProxy=true
1314
```
1415

15-
### Step#2. Install Blob CSI driver with `node.enableBlobfuseProxy=true` helm chart setting
16-
> another opition: set `enable-blobfuse-proxy=true` in existing `csi-blob-node` daemonset (default is `false`)
16+
### Enable blobfuse proxy on existing Blob CSI driver
17+
- install blobfuse proxy
18+
> following config only works on debian based agent node
1719
```console
18-
helm repo add blob-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/charts
19-
helm install blob-csi-driver blob-csi-driver/blob-csi-driver --namespace kube-system --version v1.4.0 --set node.enableBlobfuseProxy=true
20+
kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/deploy/blobfuse-proxy.yaml
2021
```
22+
- set `enable-blobfuse-proxy=true` in existing `csi-blob-node` daemonset (default is `false`)

‎deploy/blobfuse-proxy/blobfuse-proxy.yaml

Lines changed: 0 additions & 85 deletions
This file was deleted.

‎deploy/install-driver.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ kubectl apply -f $repo/csi-blob-controller.yaml
4242
if [[ "$#" -gt 1 ]]; then
4343
if [[ "$2" == *"blobfuse-proxy"* ]]; then
4444
echo "set enable-blobfuse-proxy as true ..."
45-
kubectl apply -f ./deploy/blobfuse-proxy/blobfuse-proxy.yaml
45+
kubectl apply -f ./deploy/blobfuse-proxy.yaml
4646
if [[ "$2" == *"local"* ]]; then
4747
cat $repo/csi-blob-node.yaml | sed 's/enable-blobfuse-proxy=false/enable-blobfuse-proxy=true/g' | kubectl apply -f -
4848
else
@@ -51,5 +51,7 @@ if [[ "$#" -gt 1 ]]; then
5151
else
5252
kubectl apply -f $repo/csi-blob-node.yaml
5353
fi
54+
else
55+
kubectl apply -f $repo/csi-blob-node.yaml
5456
fi
5557
echo 'Azure Blob Storage CSI driver installed successfully.'

‎deploy/uninstall-driver.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ fi
3636
if [[ "$#" -gt 1 ]]; then
3737
if [[ "$2" == *"blobfuse-proxy"* ]]; then
3838
echo "remove blobfuse-proxy deployment ..."
39-
kubectl delete -f ./deploy/blobfuse-proxy/blobfuse-proxy.yaml
39+
kubectl delete -f ./deploy/blobfuse-proxy.yaml
4040
fi
4141
fi
4242

‎pkg/blobplugin/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ RUN clean-install ca-certificates libfuse-dev libcurl4-gnutls-dev libgnutls28-de
1919
RUN mkdir /blobfuse-proxy/
2020
# for compatibility, remove this after v1.6.0 release (todo)
2121
COPY deploy/blobfuse-proxy/v0.1.0/blobfuse-proxy-v0.1.0.deb /blobfuse-proxy/
22-
COPY deploy/blobfuse-proxy/v0.1.1/blobfuse-proxy.deb /blobfuse-proxy/
22+
COPY deploy/blobfuse-proxy/blobfuse-proxy.deb /blobfuse-proxy/
2323
RUN wget -O /blobfuse-proxy/packages-microsoft-prod.deb https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb
2424
RUN dpkg -i /blobfuse-proxy/packages-microsoft-prod.deb && apt update && apt install blobfuse fuse -y
2525
# this is a workaround to install nfs-kernel-server and don't quit with error

0 commit comments

Comments
 (0)