Skip to content

Commit e3559b7

Browse files
fengzixufengzixu
committed
feature: support external-heath-monitor
1 parent 123fa4e commit e3559b7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+27529
-277
lines changed

‎deploy/kubernetes-1.17/hostpath/csi-hostpath-driverinfo.yaml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,6 @@ metadata:
44
name: hostpath.csi.k8s.io
55
spec:
66
# Supports persistent and ephemeral inline volumes.
7-
volumeLifecycleModes:
8-
- Persistent
9-
- Ephemeral
107
# To determine at runtime which mode a volume uses, pod info and its
118
# "csi.storage.k8s.io/ephemeral" entry are needed.
129
podInfoOnMount: true

‎go.mod

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,19 @@ module github.com/kubernetes-csi/csi-driver-host-path
33
go 1.12
44

55
require (
6-
github.com/container-storage-interface/spec v1.2.0
6+
github.com/container-storage-interface/spec v1.3.0
7+
github.com/gogo/protobuf v1.3.1 // indirect
78
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
89
github.com/golang/protobuf v1.3.2
10+
github.com/google/gofuzz v1.2.0 // indirect
911
github.com/google/uuid v1.0.0 // indirect
1012
github.com/kubernetes-csi/csi-lib-utils v0.3.0
1113
github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709
1214
github.com/spf13/afero v1.2.2 // indirect
1315
github.com/stretchr/testify v1.4.0 // indirect
1416
golang.org/x/net v0.0.0-20190311183353-d8887717615a
1517
google.golang.org/grpc v1.26.0
18+
gopkg.in/inf.v0 v0.9.1 // indirect
1619
k8s.io/apimachinery v0.0.0-20181110190943-2a7c93004028 // indirect
1720
k8s.io/kubernetes v1.12.2
1821
k8s.io/utils v0.0.0-20181102055113-1bd4f387aa67

‎go.sum

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
22
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
33
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
44
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
5-
github.com/container-storage-interface/spec v1.2.0 h1:bD9KIVgaVKKkQ/UbVUY9kCaH/CJbhNxe0eeB4JeJV2s=
6-
github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
5+
github.com/container-storage-interface/spec v1.3.0 h1:wMH4UIoWnK/TXYw8mbcIHgZmB6kHOeIsYsiaTJwa6bc=
6+
github.com/container-storage-interface/spec v1.3.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
77
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
88
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
99
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
1010
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
11+
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
12+
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
1113
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
1214
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
1315
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -17,8 +19,12 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
1719
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
1820
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
1921
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
22+
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
23+
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
2024
github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
2125
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
26+
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
27+
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
2228
github.com/kubernetes-csi/csi-lib-utils v0.3.0 h1:iZ7zGJMqXXlixH24Hvc132rZ7lMYx7ggHsZPIqvhPIU=
2329
github.com/kubernetes-csi/csi-lib-utils v0.3.0/go.mod h1:GVmlUmxZ+SUjVLXicRFjqWUUvWez0g0Y78zNV9t7KfQ=
2430
github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs983HfUfpkw9OTFD9tbBfAViHE=
@@ -51,6 +57,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv
5157
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
5258
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
5359
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
60+
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
5461
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
5562
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
5663
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -66,6 +73,8 @@ google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
6673
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
6774
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
6875
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
76+
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
77+
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
6978
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
7079
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
7180
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

‎pkg/hostpath/controllerserver.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,10 @@ func NewControllerServer(ephemeral bool, nodeID string) *controllerServer {
6161
caps: getControllerServiceCapabilities(
6262
[]csi.ControllerServiceCapability_RPC_Type{
6363
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
64+
csi.ControllerServiceCapability_RPC_GET_VOLUME,
6465
csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
6566
csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
67+
csi.ControllerServiceCapability_RPC_LIST_VOLUMES,
6668
csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
6769
csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
6870
}),
@@ -277,6 +279,43 @@ func (cs *controllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacit
277279
}
278280

279281
func (cs *controllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) {
282+
volumeRes := &csi.ListVolumesResponse{
283+
Entries: []*csi.ListVolumesResponse_Entry{},
284+
}
285+
286+
var (
287+
startIdx, volumesLength, maxLength int64
288+
hpVolume hostPathVolume
289+
)
290+
volumeIds := getSortedVolumeIDs()
291+
startIdx, err := strconv.ParseInt(req.StartingToken, 10, 32)
292+
if err != nil {
293+
return nil, status.Error(codes.InvalidArgument, "The type of startingToken should be integer")
294+
}
295+
296+
volumesLength = int64(len(volumeIds))
297+
maxLength = int64(req.MaxEntries)
298+
for index := startIdx - 1; index < volumesLength || index < maxLength; index++ {
299+
hpVolume = hostPathVolumes[volumeIds[index]]
300+
healthy, msg := doHealthCheck(volumeIds[index])
301+
volumeRes.Entries = append(volumeRes.Entries, &csi.ListVolumesResponse_Entry{
302+
Volume: &csi.Volume{
303+
VolumeId: hpVolume.VolID,
304+
CapacityBytes: hpVolume.VolSize,
305+
},
306+
Status: &csi.ListVolumesResponse_VolumeStatus{
307+
PublishedNodeIds: []string{hpVolume.NodeID},
308+
VolumeCondition: &csi.VolumeCondition{
309+
Abnormal: !healthy,
310+
Message: msg,
311+
},
312+
},
313+
})
314+
}
315+
return volumeRes, nil
316+
}
317+
318+
func (cs *controllerServer) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) {
280319
return nil, status.Error(codes.Unimplemented, "")
281320
}
282321

‎pkg/hostpath/healthcheck.go

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
package hostpath
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
"strings"
8+
9+
fs "k8s.io/kubernetes/pkg/volume/util/fs"
10+
utilexec "k8s.io/utils/exec"
11+
)
12+
13+
const (
14+
// sourcePathPrefix should be specified by the csi-data-dir volume.
15+
// You can change it by modifying the volume path of csi-data-dir volume.
16+
sourcePathPrefix = "/var/lib/csi-hostpath-data/"
17+
)
18+
19+
func locateCommandPath(commandName string) string {
20+
// default to root
21+
binary := filepath.Join("/", commandName)
22+
for _, path := range []string{"/bin", "/usr/sbin", "/usr/bin"} {
23+
binPath := filepath.Join(path, binary)
24+
if _, err := os.Stat(binPath); err != nil {
25+
continue
26+
}
27+
28+
return binPath
29+
}
30+
31+
return ""
32+
}
33+
34+
func getSourcePath(volumeHandle string) string {
35+
return fmt.Sprintf("%s%s", sourcePathPrefix, volumeHandle)
36+
}
37+
38+
func checkSourcePathExist(volumeHandle string) (bool, error) {
39+
sourcePath := getSnapshotPath(volumeHandle)
40+
_, err := os.Stat(sourcePath)
41+
if err != nil {
42+
if os.IsNotExist(err) {
43+
return false, nil
44+
}
45+
46+
return false, err
47+
}
48+
49+
return true, nil
50+
}
51+
52+
func checkMountPointExist(sourcePath string) (bool, error) {
53+
exec := utilexec.New()
54+
args := []string{"-l", "-o TARGET,SOURCE"}
55+
out, err := exec.Command(locateCommandPath("findmnt"), args...).CombinedOutput()
56+
if err != nil {
57+
return false, err
58+
}
59+
60+
if len(out) < 1 {
61+
return false, fmt.Errorf("mount point info is nil")
62+
}
63+
64+
mountInfo := strings.Split(string(out), " ")
65+
if len(mountInfo) != 2 {
66+
return false, nil
67+
}
68+
69+
mountPoint := mountInfo[0]
70+
sp := mountInfo[1]
71+
72+
if !strings.Contains(sp, sourcePath) {
73+
return false, nil
74+
}
75+
76+
_, err = os.Stat(mountPoint)
77+
if err != nil {
78+
if os.IsNotExist(err) {
79+
return false, nil
80+
}
81+
82+
return false, err
83+
}
84+
85+
return true, nil
86+
}
87+
88+
func checkPVCapacityValid(volumeHandle string) (bool, error) {
89+
sourcePath := getSourcePath(volumeHandle)
90+
_, fscapacity, _, _, _, _, err := fs.FsInfo(sourcePath)
91+
if err != nil {
92+
return false, err
93+
}
94+
95+
volumeCapacity := hostPathVolumes[volumeHandle].VolSize
96+
return fscapacity >= volumeCapacity, nil
97+
}
98+
99+
func checkPVUsage(volumeHandle string) (bool, error) {
100+
sourcePath := getSourcePath(volumeHandle)
101+
fsavailable, _, _, _, _, _, err := fs.FsInfo(sourcePath)
102+
if err != nil {
103+
return false, err
104+
}
105+
106+
return fsavailable > 0, nil
107+
}
108+
109+
func doHealthCheck(volumeHandle string) (bool, string) {
110+
spExist, err := checkSourcePathExist(volumeHandle)
111+
if err != nil {
112+
return false, err.Error()
113+
}
114+
115+
if !spExist {
116+
return false, "The source path of the volume doesn't exist"
117+
}
118+
119+
mpExist, err := checkMountPointExist(volumeHandle)
120+
if err != nil {
121+
return false, err.Error()
122+
}
123+
124+
if !mpExist {
125+
return false, "The volume isn't mounted"
126+
}
127+
128+
capValid, err := checkPVCapacityValid(volumeHandle)
129+
if err != nil {
130+
return false, err.Error()
131+
}
132+
133+
if !capValid {
134+
return false, "The capacity of volume is greater than actual storage"
135+
}
136+
137+
available, err := checkPVUsage(volumeHandle)
138+
if err != nil {
139+
return false, err.Error()
140+
}
141+
142+
if !available {
143+
return false, "The free space of the volume is insufficient"
144+
}
145+
146+
return true, ""
147+
}

‎pkg/hostpath/hostpath.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"io/ioutil"
2424
"os"
2525
"path/filepath"
26+
"sort"
2627
"strings"
2728

2829
"github.com/golang/glog"
@@ -65,6 +66,7 @@ type hostPathVolume struct {
6566
ParentVolID string `json:"parentVolID,omitempty"`
6667
ParentSnapID string `json:"parentSnapID,omitempty"`
6768
Ephemeral bool `json:"ephemeral"`
69+
NodeID string `json:"nodeID"`
6870
}
6971

7072
type hostPathSnapshot struct {
@@ -395,3 +397,14 @@ func loadFromBlockVolume(hostPathVolume hostPathVolume, destPath string) error {
395397
}
396398
return nil
397399
}
400+
401+
func getSortedVolumeIDs() []string {
402+
ids := make([]string, len(hostPathVolumes))
403+
index := 0
404+
for volId := range hostPathVolumes {
405+
ids[index] = volId
406+
}
407+
408+
sort.Strings(ids)
409+
return ids
410+
}

‎pkg/hostpath/nodeserver.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,9 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
185185
}
186186
}
187187

188+
volume := hostPathVolumes[req.GetVolumeId()]
189+
volume.NodeID = ns.nodeID
190+
hostPathVolumes[req.GetVolumeId()] = volume
188191
return &csi.NodePublishVolumeResponse{}, nil
189192
}
190193

0 commit comments

Comments
 (0)