Add mutual tls auth support to the router. The verification is via client

side certificates and its use can be mandated to be either required or optional.
In addition, an env variable ROUTER_MUTUAL_TLS_AUTH_CN provides more fine-grain
control on access based on certificate common names.

Author: ramr

contrib/completions/bash/oc

@@ -219,6 +219,9 @@ frontend fe_sni
     {{- if isTrue (env "ROUTER_STRICT_SNI") }} strict-sni {{ end }}
     {{- ""}} crt {{firstMatch ".+" .DefaultCertificate "/var/lib/haproxy/conf/default_pub_keys.pem"}}
     {{- ""}} crt-list /var/lib/haproxy/conf/cert_config.map accept-proxy
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH") }} verify {{.}} {{ end }}
   mode http
 
   # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/)
@@ -228,6 +231,27 @@ frontend fe_sni
   # before matching, or any requests containing uppercase characters will never match.
   http-request set-header Host %[req.hdr(Host),lower]
 
+  {{- if ne (env "ROUTER_MUTUAL_TLS_AUTH" "none") "none" }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CN") }}
+  # If a mutual TLS auth CN is set, we deny requests if the common name doesn't
+  # match. A custom template can change this behavior (e.g. set custom headers).
+  acl cert_cn_matches ssl_c_s_dn(CN) -m sub {{.}}
+  http-request deny unless cert_cn_matches
+    {{- end }}
+
+  # Add X-SSL* headers to pass client certificate information to the backend.
+  http-request set-header X-SSL                  %[ssl_fc]
+  http-request set-header X-SSL-Client-Verify    %[ssl_c_verify]
+  http-request set-header X-SSL-Client-Serial    %{+Q}[ssl_c_serial,hex]
+  http-request set-header X-SSL-Client-Version   %{+Q}[ssl_c_version]
+  http-request set-header X-SSL-Client-SHA1      %{+Q}[ssl_c_sha1,hex]
+  http-request set-header X-SSL-Client-DN        %{+Q}[ssl_c_s_dn]
+  http-request set-header X-SSL-Client-CN        %{+Q}[ssl_c_s_dn(cn)]
+  http-request set-header X-SSL-Issuer           %{+Q}[ssl_c_i_dn]
+  http-request set-header X-SSL-Client-NotBefore %{+Q}[ssl_c_notbefore]
+  http-request set-header X-SSL-Client-NotAfter  %{+Q}[ssl_c_notafter]
+  {{- end }}
+
   # map to backend
   # Search from most specific to general path (host case).
   # Note: If no match, haproxy uses the default_backend, no other
@@ -254,6 +278,9 @@ backend be_no_sni
 frontend fe_no_sni
   # terminate ssl on edge
   bind 127.0.0.1:{{env "ROUTER_SERVICE_NO_SNI_PORT" "10443"}} ssl no-sslv3 crt {{firstMatch ".+" .DefaultCertificate "/var/lib/haproxy/conf/default_pub_keys.pem"}} accept-proxy
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH") }} verify {{.}} {{ end }}
   mode http
 
   # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/)
@@ -263,6 +290,26 @@ frontend fe_no_sni
   # before matching, or any requests containing uppercase characters will never match.
   http-request set-header Host %[req.hdr(Host),lower]
 
+  {{- if ne (env "ROUTER_MUTUAL_TLS_AUTH" "none") "none" }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CN") }}
+  # If a mutual TLS auth CN is set, we deny requests if the common name doesn't
+  # match. A custom template can change this behavior (e.g. set custom headers).
+  acl cert_cn_matches ssl_c_s_dn(CN) -m sub {{.}}
+  http-request deny unless cert_cn_matches
+    {{- end }}
+
+  # Add X-SSL* headers to pass client certificate information to the backend.
+  http-request set-header X-SSL                  %[ssl_fc]
+  http-request set-header X-SSL-Client-Verify    %[ssl_c_verify]
+  http-request set-header X-SSL-Client-Serial    %{+Q}[ssl_c_serial,hex]
+  http-request set-header X-SSL-Client-Version   %{+Q}[ssl_c_version]
+  http-request set-header X-SSL-Client-SHA1      %{+Q}[ssl_c_sha1,hex]
+  http-request set-header X-SSL-Client-DN        %{+Q}[ssl_c_s_dn]
+  http-request set-header X-SSL-Client-CN        %{+Q}[ssl_c_s_dn(cn)]
+  http-request set-header X-SSL-Issuer           %{+Q}[ssl_c_i_dn]
+  http-request set-header X-SSL-Client-NotBefore %{+Q}[ssl_c_notbefore]
+  http-request set-header X-SSL-Client-NotAfter  %{+Q}[ssl_c_notafter]
+  {{- end }}
 
   # map to backend
   # Search from most specific to general path (host case).

contrib/completions/zsh/oc

@@ -18,6 +18,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	kapi "k8s.io/kubernetes/pkg/apis/core"
@@ -84,6 +85,11 @@ var (
 	privkeyName       = "router.pem"
 	privkeyPath       = secretsPath + "/" + privkeyName
 
+	defaultMutualTLSAuth = "none"
+	clientCertConfigDir  = "/etc/pki/tls/client-certs"
+	clientCertConfigCA   = "ca.pem"
+	clientCertConfigCRL  = "crl.pem"
+
 	defaultCertificatePath = path.Join(defaultCertificateDir, "tls.crt")
 )
 
@@ -229,6 +235,19 @@ type RouterConfig struct {
 	StrictSNI bool
 
 	Local bool
+
+	// MutualTLSAuth controls access to the router using a mutually agreed
+	// upon TLS authentication mechanism (ala client certificates).
+	// One of: required | optional | none  - the default is none.
+	MutualTLSAuth string
+
+	// MutualTLSAuthCA contains the CA certificates that will be used
+	// to verify a client's certificate.
+	MutualTLSAuthCA string
+
+	// MutualTLSAuthCRL contains the certificate revocation list used to
+	// verify a client's certificate.
+	MutualTLSAuthCRL string
 }
 
 const (
@@ -257,6 +276,8 @@ func NewCmdRouter(f *clientcmd.Factory, parentName, name string, out, errout io.
 		StatsPort:     defaultStatsPort,
 		HostNetwork:   true,
 		HostPorts:     true,
+
+		MutualTLSAuth: defaultMutualTLSAuth,
 	}
 
 	cmd := &cobra.Command{
@@ -309,15 +330,24 @@ func NewCmdRouter(f *clientcmd.Factory, parentName, name string, out, errout io.
 	cmd.Flags().BoolVar(&cfg.StrictSNI, "strict-sni", cfg.StrictSNI, "Use strict-sni bind processing (do not use default cert). Not supported for F5.")
 	cmd.Flags().BoolVar(&cfg.Local, "local", cfg.Local, "If true, do not contact the apiserver")
 
+	cmd.Flags().StringVar(&cfg.MutualTLSAuth, "mutual-tls-auth", cfg.MutualTLSAuth, "Controls access to the router using mutually agreed upon TLS configuration (ala client certificates). You can choose one of 'required', 'optional', or 'none'. The default is none.")
+	cmd.Flags().StringVar(&cfg.MutualTLSAuthCA, "mutual-tls-auth-ca", cfg.MutualTLSAuthCA, "Optional path to a file containing one or more CA certificates used for mutual TLS authentication. The CA certificate[s] are used by the router to verify a client's certificate.")
+	cmd.Flags().StringVar(&cfg.MutualTLSAuthCRL, "mutual-tls-auth-crl", cfg.MutualTLSAuthCRL, "Optional path to a file containing the certificate revocation list used for mutual TLS authentication. The certificate revocation list is used by the router to verify a client's certificate.")
+
 	cfg.Action.BindForOutput(cmd.Flags())
 	cmd.Flags().String("output-version", "", "The preferred API versions of the output objects")
 
 	return cmd
 }
 
+// generateMutualTLSSecretName generates a mutual TLS auth secret name.
+func generateMutualTLSSecretName(prefix string) string {
+	return fmt.Sprintf("%s-mutual-tls-auth", prefix)
+}
+
 // generateSecretsConfig generates any Secret and Volume objects, such
 // as SSH private keys, that are necessary for the router container.
-func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []byte, certName string) ([]*kapi.Secret, []kapi.Volume, []kapi.VolumeMount, error) {
+func generateSecretsConfig(cfg *RouterConfig, namespace string, certName string, defaultCert, mtlsAuthCA, mtlsAuthCRL []byte) ([]*kapi.Secret, []kapi.Volume, []kapi.VolumeMount, error) {
 	var secrets []*kapi.Secret
 	var volumes []kapi.Volume
 	var mounts []kapi.VolumeMount
@@ -424,6 +454,42 @@ func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []by
 	}
 	mounts = append(mounts, mount)
 
+	mtlsSecretData := map[string][]byte{}
+	if len(mtlsAuthCA) > 0 {
+		mtlsSecretData[clientCertConfigCA] = mtlsAuthCA
+	}
+	if len(mtlsAuthCRL) > 0 {
+		mtlsSecretData[clientCertConfigCRL] = mtlsAuthCRL
+	}
+
+	if len(mtlsSecretData) > 0 {
+		secretName := generateMutualTLSSecretName(cfg.Name)
+		secret := &kapi.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: secretName,
+			},
+			Data: mtlsSecretData,
+		}
+		secrets = append(secrets, secret)
+
+		volume := kapi.Volume{
+			Name: "mutual-tls-config",
+			VolumeSource: kapi.VolumeSource{
+				Secret: &kapi.SecretVolumeSource{
+					SecretName: secretName,
+				},
+			},
+		}
+		volumes = append(volumes, volume)
+
+		mount := kapi.VolumeMount{
+			Name:      volume.Name,
+			ReadOnly:  true,
+			MountPath: clientCertConfigDir,
+		}
+		mounts = append(mounts, mount)
+	}
+
 	return secrets, volumes, mounts, nil
 }
 
@@ -576,6 +642,14 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
 		if err != nil {
 			return fmt.Errorf("error getting client: %v", err)
 		}
+
+		if len(cfg.MutualTLSAuthCA) > 0 || len(cfg.MutualTLSAuthCRL) > 0 {
+			secretName := generateMutualTLSSecretName(cfg.Name)
+			if _, err := kClient.Core().Secrets(namespace).Get(secretName, metav1.GetOptions{}); err == nil {
+				return fmt.Errorf("router could not be created: mutual tls secret %q already exists", secretName)
+			}
+		}
+
 		service, err := kClient.Core().Services(namespace).Get(name, metav1.GetOptions{})
 		if err != nil {
 			if !generate {
@@ -628,6 +702,20 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
 		return fmt.Errorf("router could not be created; error reading default certificate file: %v", err)
 	}
 
+	mtlsAuthOptions := []string{"required", "optional", "none"}
+	allowedMutualTLSAuthOptions := sets.NewString(mtlsAuthOptions...)
+	if !allowedMutualTLSAuthOptions.Has(cfg.MutualTLSAuth) {
+		return fmt.Errorf("invalid mutual tls auth option %v, expected one of %v", cfg.MutualTLSAuth, mtlsAuthOptions)
+	}
+	mtlsAuthCA, err := fileutil.LoadData(cfg.MutualTLSAuthCA)
+	if err != nil {
+		return fmt.Errorf("reading ca certificates for mutual tls auth: %v", err)
+	}
+	mtlsAuthCRL, err := fileutil.LoadData(cfg.MutualTLSAuthCRL)
+	if err != nil {
+		return fmt.Errorf("reading certificate revocation list for mutual tls auth: %v", err)
+	}
+
 	if len(cfg.StatsPassword) == 0 {
 		cfg.StatsPassword = generateStatsPassword()
 		if !cfg.Action.ShouldPrint() {
@@ -685,6 +773,17 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
 		env["ROUTER_METRICS_TLS_CERT_FILE"] = "/etc/pki/tls/metrics/tls.crt"
 		env["ROUTER_METRICS_TLS_KEY_FILE"] = "/etc/pki/tls/metrics/tls.key"
 	}
+	mtlsAuth := strings.TrimSpace(cfg.MutualTLSAuth)
+	if len(mtlsAuth) > 0 && mtlsAuth != defaultMutualTLSAuth {
+		env["ROUTER_MUTUAL_TLS_AUTH"] = cfg.MutualTLSAuth
+		if len(mtlsAuthCA) > 0 {
+			env["ROUTER_MUTUAL_TLS_AUTH_CA"] = path.Join(clientCertConfigDir, clientCertConfigCA)
+		}
+		if len(mtlsAuthCRL) > 0 {
+			env["ROUTER_MUTUAL_TLS_AUTH_CRL"] = path.Join(clientCertConfigDir, clientCertConfigCRL)
+		}
+	}
+
 	env.Add(secretEnv)
 	if len(defaultCert) > 0 {
 		if cfg.SecretsAsEnv {
@@ -695,7 +794,7 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
 	}
 	env.Add(app.Environment{"DEFAULT_CERTIFICATE_DIR": defaultCertificateDir})
 	var certName = fmt.Sprintf("%s-certs", cfg.Name)
-	secrets, volumes, mounts, err := generateSecretsConfig(cfg, namespace, defaultCert, certName)
+	secrets, volumes, mounts, err := generateSecretsConfig(cfg, namespace, certName, defaultCert, mtlsAuthCA, mtlsAuthCRL)
 	if err != nil {
 		return fmt.Errorf("router could not be created: %v", err)
 	}