Add certwatcher to webhook server

Signed-off-by: Grant Griffiths <[email protected]>

Author: ggriffiths

go.mod

@@ -4,6 +4,7 @@ go 1.15
 
 require (
 	github.com/container-storage-interface/spec v1.2.0
+	github.com/fsnotify/fsnotify v1.4.9
 	github.com/golang/mock v1.4.3
 	github.com/golang/protobuf v1.4.2
 	github.com/google/gofuzz v1.1.0

pkg/validation-webhook/certwatcher.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package webhook
+
+import (
+	"context"
+	"crypto/tls"
+	"sync"
+
+	"github.com/fsnotify/fsnotify"
+	"k8s.io/klog"
+)
+
+// This file originated from github.com/kubernetes-sigs/controller-runtime/pkg/webhook/internal/certwatcher.
+// We cannot import this package as it's an internal one. In addition, we cannot yet easily integrate
+// with controller-runtime/pkg/webhook directly, as it would require extensive rework:
+// https://github.com/kubernetes-csi/external-snapshotter/issues/422
+
+// CertWatcher watches certificate and key files for changes.  When either file
+// changes, it reads and parses both and calls an optional callback with the new
+// certificate.
+type CertWatcher struct {
+	sync.Mutex
+
+	currentCert *tls.Certificate
+	watcher     *fsnotify.Watcher
+
+	certPath string
+	keyPath  string
+}
+
+// NewCertWatcher returns a new CertWatcher watching the given certificate and key.
+func NewCertWatcher(certPath, keyPath string) (*CertWatcher, error) {
+	var err error
+
+	cw := &CertWatcher{
+		certPath: certPath,
+		keyPath:  keyPath,
+	}
+
+	// Initial read of certificate and key.
+	if err := cw.ReadCertificate(); err != nil {
+		return nil, err
+	}
+
+	cw.watcher, err = fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	return cw, nil
+}
+
+// GetCertificate fetches the currently loaded certificate, which may be nil.
+func (cw *CertWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	cw.Lock()
+	defer cw.Unlock()
+	return cw.currentCert, nil
+}
+
+// Start starts the watch on the certificate and key files.
+func (cw *CertWatcher) Start(ctx context.Context) error {
+	files := []string{cw.certPath, cw.keyPath}
+
+	for _, f := range files {
+		if err := cw.watcher.Add(f); err != nil {
+			return err
+		}
+	}
+
+	go cw.Watch()
+
+	// Block until the context is done.
+	<-ctx.Done()
+
+	return cw.watcher.Close()
+}
+
+// Watch reads events from the watcher's channel and reacts to changes.
+func (cw *CertWatcher) Watch() {
+	for {
+		select {
+		case event, ok := <-cw.watcher.Events:
+			// Channel is closed.
+			if !ok {
+				return
+			}
+
+			cw.handleEvent(event)
+
+		case err, ok := <-cw.watcher.Errors:
+			// Channel is closed.
+			if !ok {
+				return
+			}
+
+			klog.Error(err, "certificate watch error")
+		}
+	}
+}
+
+// ReadCertificate reads the certificate and key files from disk, parses them,
+// and updates the current certificate on the watcher.  If a callback is set, it
+// is invoked with the new certificate.
+func (cw *CertWatcher) ReadCertificate() error {
+	cert, err := tls.LoadX509KeyPair(cw.certPath, cw.keyPath)
+	if err != nil {
+		return err
+	}
+
+	cw.Lock()
+	cw.currentCert = &cert
+	cw.Unlock()
+
+	klog.Info("Updated current TLS certificate")
+
+	return nil
+}
+
+func (cw *CertWatcher) handleEvent(event fsnotify.Event) {
+	// Only care about events which may modify the contents of the file.
+	if !(isWrite(event) || isRemove(event) || isCreate(event)) {
+		return
+	}
+
+	klog.V(1).Info("certificate event", "event", event)
+
+	// If the file was removed, re-add the watch.
+	if isRemove(event) {
+		if err := cw.watcher.Add(event.Name); err != nil {
+			klog.Error(err, "error re-watching file")
+		}
+	}
+
+	if err := cw.ReadCertificate(); err != nil {
+		klog.Error(err, "error re-reading certificate")
+	}
+}
+
+func isWrite(event fsnotify.Event) bool {
+	return event.Op&fsnotify.Write == fsnotify.Write
+}
+
+func isCreate(event fsnotify.Event) bool {
+	return event.Op&fsnotify.Create == fsnotify.Create
+}
+
+func isRemove(event fsnotify.Event) bool {
+	return event.Op&fsnotify.Remove == fsnotify.Remove
+}

pkg/validation-webhook/webhook.go

@@ -17,6 +17,8 @@ limitations under the License.
 package webhook
 
 import (
+	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -175,21 +177,45 @@ func serveSnapshotRequest(w http.ResponseWriter, r *http.Request) {
 	serve(w, r, newDelegateToV1AdmitHandler(admitSnapshot))
 }
 
-func main(cmd *cobra.Command, args []string) {
+func startServer(ctx context.Context, tlsConfig *tls.Config, cw *CertWatcher) error {
+	go func() {
+		klog.Info("Starting certificate watcher")
+		if err := cw.Start(ctx); err != nil {
+			klog.Errorf("certificate watcher error: %v", err)
+		}
+	}()
+
 	fmt.Println("Starting webhook server")
-	config := Config{
-		CertFile: certFile,
-		KeyFile:  keyFile,
+	mux := http.NewServeMux()
+	mux.HandleFunc("/volumesnapshot", serveSnapshotRequest)
+	mux.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("ok")) })
+	srv := &http.Server{
+		Handler:   mux,
+		TLSConfig: tlsConfig,
 	}
 
-	http.HandleFunc("/volumesnapshot", serveSnapshotRequest)
-	http.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("ok")) })
-	server := &http.Server{
-		Addr:      fmt.Sprintf(":%d", port),
-		TLSConfig: configTLS(config),
+	// listener is always closed by srv.Serve
+	listener, err := tls.Listen("tcp", fmt.Sprintf(":%d", port), tlsConfig)
+	if err != nil {
+		return err
 	}
-	err := server.ListenAndServeTLS("", "")
+
+	return srv.Serve(listener)
+}
+
+func main(cmd *cobra.Command, args []string) {
+	// Create new cert watcher
+	ctx, cancel := context.WithCancel(cmd.Context())
+	defer cancel() // stops certwatcher
+	cw, err := NewCertWatcher(certFile, keyFile)
 	if err != nil {
-		panic(err)
+		klog.Fatalf("failed to initialize new cert watcher: %v", err)
+	}
+	tlsConfig := &tls.Config{
+		GetCertificate: cw.GetCertificate,
+	}
+
+	if err := startServer(ctx, tlsConfig, cw); err != nil {
+		klog.Fatalf("server stopped: %v", err)
 	}
 }