Merge pull request #8 from sbezverk/web_server_liveness_probe

Refactor of liveness probe

Author: sbezverk

cmd/main.go

@@ -19,7 +19,8 @@ package main
 import (
 	"context"
 	"flag"
-	"os"
+	"net"
+	"net/http"
 	"time"
 
 	"github.com/golang/glog"
@@ -37,51 +38,68 @@ var (
 	// kubeconfig        = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.")
 	connectionTimeout = flag.Duration("connection-timeout", 30*time.Second, "Timeout for waiting for CSI driver socket in seconds.")
 	csiAddress        = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.")
+	healthzPort       = flag.String("health-port", "9808", "TCP ports for listening healthz requests")
 )
 
-func main() {
-	flag.Set("logtostderr", "true")
-	flag.Parse()
+func runProbe(ctx context.Context) error {
 
 	// Connect to CSI.
-	glog.V(1).Infof("Attempting to open a gRPC connection with: %q", csiAddress)
+	glog.Infof("Attempting to open a gRPC connection with: %s", *csiAddress)
 	csiConn, err := connection.NewConnection(*csiAddress, *connectionTimeout)
 	if err != nil {
-		glog.Error(err.Error())
-		os.Exit(1)
+		return err
 	}
 
 	// Get CSI driver name.
-	glog.V(1).Infof("Calling CSI driver to discover driver name.")
-	ctx, cancel := context.WithTimeout(context.Background(), csiTimeout)
-	defer cancel()
+	glog.Infof("Calling CSI driver to discover driver name.")
 	csiDriverName, err := csiConn.GetDriverName(ctx)
 	if err != nil {
-		glog.Error(err.Error())
-		os.Exit(1)
+		return err
 	}
-	glog.V(2).Infof("CSI driver name: %q", csiDriverName)
+	glog.Infof("CSI driver name: %q", csiDriverName)
 
 	// Get CSI Driver Node ID
-	glog.V(1).Infof("Calling CSI driver to discover node ID.")
-	ctx, cancel = context.WithTimeout(context.Background(), csiTimeout)
-	defer cancel()
+	glog.Infof("Calling CSI driver to discover node ID.")
 	csiDriverNodeID, err := csiConn.NodeGetId(ctx)
 	if err != nil {
-		glog.Error(err.Error())
-		os.Exit(1)
+		return err
 	}
-	glog.V(2).Infof("CSI driver node ID: %q", csiDriverNodeID)
+	glog.Infof("CSI driver node ID: %q", csiDriverNodeID)
 
 	// Sending Probe request
-	glog.V(1).Infof("Sending probe request to CSI driver.")
-	ctx, cancel = context.WithTimeout(context.Background(), csiTimeout)
+	glog.Infof("Sending probe request to CSI driver.")
+	if err := csiConn.LivenessProbe(ctx); err != nil {
+		return err
+	}
+	return nil
+}
+
+func chekcHealth(w http.ResponseWriter, req *http.Request) {
+
+	glog.Infof("Request: %s from: %s\n", req.URL.Path, req.RemoteAddr)
+	ctx, cancel := context.WithTimeout(context.Background(), *connectionTimeout)
 	defer cancel()
-	err = csiConn.LivenessProbe(ctx)
+	err := runProbe(ctx)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		glog.Infof("Health check failed with: %v.", err)
+	} else {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`ok`))
+		glog.Infof("Health check succeeded.")
+	}
+}
+
+func main() {
+	flag.Set("logtostderr", "true")
+	flag.Parse()
+
+	addr := net.JoinHostPort("0.0.0.0", *healthzPort)
+	http.HandleFunc("/healthz", chekcHealth)
+	glog.Infof("Serving requests to /healthz on: %s", addr)
+	err := http.ListenAndServe(addr, nil)
 	if err != nil {
-		glog.Error(err.Error())
-		os.Exit(1)
+		glog.Fatalf("failed to start http server with error: %v", err)
 	}
-	// Liveness Probe was sucessfuly
-	os.Exit(0)
 }

deployment/kubernetes/hostpath-with-livenessprobe.yaml

@@ -102,22 +102,23 @@ kind: Pod
 metadata:
   labels:
     app: hostpath-driver
-  name: csi-pod
+  name: csi-hostpath-plugin
   namespace: default
 spec:
   serviceAccount: csi-service-account 
   containers:
-  - args:
+  - name: external-provisioner
+    args:
     - --v=5
     - --provisioner=csi-hostpath
     - --csi-address=/csi/csi.sock
     image: quay.io/k8scsi/csi-provisioner:v0.2.0
     imagePullPolicy: Always
-    name: external-provisioner
     volumeMounts:
     - mountPath: /csi
       name: socket-dir
-  - args:
+  - name: driver-registrar
+    args:
     - --v=5
     - --csi-address=/csi/csi.sock
     env:
@@ -128,23 +129,46 @@ spec:
           fieldPath: spec.nodeName
     image: quay.io/k8scsi/driver-registrar:v0.2.0
     imagePullPolicy: Always
-    name: driver-registrar
     volumeMounts:
     - mountPath: /csi
       name: socket-dir
-  - args:
+  - name: external-attacher
+    args:
     - --v=5
     - --csi-address=$(ADDRESS)
     env:
     - name: ADDRESS
       value: /csi/csi.sock
     image: quay.io/k8scsi/csi-attacher:v0.2.0 
     imagePullPolicy: Always
-    name: external-attacher
     volumeMounts:
     - mountPath: /csi
       name: socket-dir
-  - args:
+  - name: hostpath-driver
+    image: quay.io/k8scsi/hostpathplugin:v0.2.0
+    imagePullPolicy: Always
+    securityContext:
+      privileged: true
+    ports:
+    - containerPort: 9808
+      name: healthz
+      protocol: TCP
+    livenessProbe:
+      failureThreshold: 5
+      httpGet:
+        path: /healthz
+        port: healthz
+      initialDelaySeconds: 10
+      timeoutSeconds: 3
+      periodSeconds: 2
+      failureThreshold: 1
+    volumeMounts:
+    - mountPath: /csi
+      name: socket-dir
+    - mountPath: /var/lib/kubelet/pods
+      mountPropagation: Bidirectional
+      name: mountpoint-dir
+    args:
     - --v=5
     - --endpoint=$(CSI_ENDPOINT)
     - --nodeid=$(KUBE_NODE_NAME)
@@ -156,36 +180,15 @@ spec:
         fieldRef:
           apiVersion: v1
           fieldPath: spec.nodeName
-    image: quay.io/k8scsi/hostpathplugin:v0.2.0
-    imagePullPolicy: Always
-    name: hostpath-driver
-    securityContext:
-      privileged: true
-    volumeMounts:
-    - mountPath: /csi
-      name: socket-dir
-    - mountPath: /var/lib/kubelet/pods
-      mountPropagation: Bidirectional
-      name: mountpoint-dir
   - name: liveness-probe
-    image: quay.io/k8scsi/livenessprobe:v0.2.0
     imagePullPolicy: Always
-    command: ["/bin/sh"]
-    args: ["-c", "while true; do sleep 10;done"]    
-    livenessProbe:
-      exec:
-        command:
-        - ./livenessprobe
-        - --v=6
-        - --csi-address=/csi/csi.sock
-        - --connection-timeout=3s
-      initialDelaySeconds: 10
-      timeoutSeconds: 3
-      periodSeconds: 2
-      failureThreshold: 1 
     volumeMounts:
     - mountPath: /csi
       name: socket-dir
+    image: quay.io/k8scsi/livenessprobe:v0.2.0
+    args:
+    - --csi-address=/csi/csi.sock
+    - --connection-timeout=3s
   volumes:
   - hostPath:
       path: /var/lib/kubelet/plugins/csi-hostpath

pkg/connection/connection.go

@@ -67,7 +67,7 @@ func NewConnection(
 }
 
 func connect(address string, timeout time.Duration) (*grpc.ClientConn, error) {
-	glog.V(2).Infof("Connecting to %s", address)
+	glog.Infof("Connecting to %s", address)
 	dialOptions := []grpc.DialOption{
 		grpc.WithInsecure(),
 		grpc.WithBackoffMaxDelay(time.Second),
@@ -87,14 +87,14 @@ func connect(address string, timeout time.Duration) (*grpc.ClientConn, error) {
 	defer cancel()
 	for {
 		if !conn.WaitForStateChange(ctx, conn.GetState()) {
-			glog.V(4).Infof("Connection timed out")
+			glog.Infof("Connection timed out")
 			return conn, nil // return nil, subsequent GetPluginInfo will show the real connection error
 		}
 		if conn.GetState() == connectivity.Ready {
-			glog.V(3).Infof("Connected")
+			glog.Infof("Connected")
 			return conn, nil
 		}
-		glog.V(4).Infof("Still trying, connection is %s", conn.GetState())
+		glog.Infof("Still trying, connection is %s", conn.GetState())
 	}
 }
 
@@ -147,10 +147,10 @@ func (c *csiConnection) Close() error {
 }
 
 func logGRPC(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
-	glog.V(5).Infof("GRPC call: %s", method)
-	glog.V(5).Infof("GRPC request: %+v", req)
+	glog.Infof("GRPC call: %s", method)
+	glog.Infof("GRPC request: %+v", req)
 	err := invoker(ctx, method, req, reply, cc, opts...)
-	glog.V(5).Infof("GRPC response: %+v", reply)
-	glog.V(5).Infof("GRPC error: %v", err)
+	glog.Infof("GRPC response: %+v", reply)
+	glog.Infof("GRPC error: %v", err)
 	return err
 }