Add mutual tls auth support to the router. The verification is via client

side certificates and its use can be mandated to be either required or optional.
In addition, an env variable ROUTER_MUTUAL_TLS_AUTH_CN provides more fine-grain
control on access (via regular expressions) based on certificate common names.

Author: ramr

contrib/completions/bash/oc

@@ -225,6 +225,9 @@ frontend fe_sni
     {{- if isTrue (env "ROUTER_STRICT_SNI") }} strict-sni {{ end }}
     {{- ""}} crt {{firstMatch ".+" .DefaultCertificate "/var/lib/haproxy/conf/default_pub_keys.pem"}}
     {{- ""}} crt-list /var/lib/haproxy/conf/cert_config.map accept-proxy
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH") }} verify {{.}} {{ end }}
     {{- if isTrue (env "ROUTER_ENABLE_HTTP2") }} alpn h2,http/1.1{{ end }}
   mode http
 
@@ -235,6 +238,37 @@ frontend fe_sni
   # before matching, or any requests containing uppercase characters will never match.
   http-request set-header Host %[req.hdr(Host),lower]
 
+  {{ if ne (env "ROUTER_MUTUAL_TLS_AUTH" "none") "none" }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_FILTER") }}
+  # If a mutual TLS auth subject filter environment variable is set, we deny
+  # requests if the DN field in the client certificate doesn't match that value.
+  # Please note that this match is a regular expression match.
+  # Example: For DN set to: /CN=header.test/ST=CA/C=US/O=Security/OU=OpenShift3,
+  #          A. ROUTER_MUTUAL_TLS_AUTH_FILTER="header.test"   OR
+  #             ROUTER_MUTUAL_TLS_AUTH_FILTER="head"          OR
+  #             ROUTER_MUTUAL_TLS_AUTH_FILTER="^/CN=header.test/ST=CA/C=US/O=Security/OU=OpenShift3$" /* exact match example */
+  #             the filter would match the DN field (substring or exact match)
+  #             and the request will be passed on to the backend.
+  #          B. ROUTER_MUTUAL_TLS_AUTH_FILTER="legacy-web-client", the request
+  #             will be rejected.
+  acl cert_cn_matches ssl_c_s_dn -m reg {{.}}
+  http-request deny unless cert_cn_matches
+    {{- end }}
+
+  # Add X-SSL* headers to pass client certificate information to the backend.
+  http-request set-header X-SSL                  %[ssl_fc]
+  http-request set-header X-SSL-Client-Verify    %[ssl_c_verify]
+  http-request set-header X-SSL-Client-Serial    %{+Q}[ssl_c_serial,hex]
+  http-request set-header X-SSL-Client-Version   %{+Q}[ssl_c_version]
+  http-request set-header X-SSL-Client-SHA1      %{+Q}[ssl_c_sha1,hex]
+  http-request set-header X-SSL-Client-DN        %{+Q}[ssl_c_s_dn]
+  http-request set-header X-SSL-Client-CN        %{+Q}[ssl_c_s_dn(cn)]
+  http-request set-header X-SSL-Issuer           %{+Q}[ssl_c_i_dn]
+  http-request set-header X-SSL-Client-NotBefore %{+Q}[ssl_c_notbefore]
+  http-request set-header X-SSL-Client-NotAfter  %{+Q}[ssl_c_notafter]
+  http-request set-header X-SSL-Client-DER       %{+Q}[ssl_c_der,base64]
+  {{- end }}
+
   # map to backend
   # Search from most specific to general path (host case).
   # Note: If no match, haproxy uses the default_backend, no other
@@ -261,6 +295,9 @@ backend be_no_sni
 frontend fe_no_sni
   # terminate ssl on edge
   bind 127.0.0.1:{{env "ROUTER_SERVICE_NO_SNI_PORT" "10443"}} ssl no-sslv3 crt {{firstMatch ".+" .DefaultCertificate "/var/lib/haproxy/conf/default_pub_keys.pem"}} accept-proxy
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH") }} verify {{.}} {{ end }}
   mode http
 
   # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/)
@@ -270,6 +307,29 @@ frontend fe_no_sni
   # before matching, or any requests containing uppercase characters will never match.
   http-request set-header Host %[req.hdr(Host),lower]
 
+  {{ if ne (env "ROUTER_MUTUAL_TLS_AUTH" "none") "none" }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_FILTER") }}
+  # If a mutual TLS auth subject filter environment variable is set, we deny
+  # requests if the DN field in the client certificate doesn't match that value.
+  # Please note that this match is a regular expression match.
+  # See the config section 'frontend fe_sni' for examples.
+  acl cert_cn_matches ssl_c_s_dn -m reg {{.}}
+  http-request deny unless cert_cn_matches
+    {{- end }}
+
+  # Add X-SSL* headers to pass client certificate information to the backend.
+  http-request set-header X-SSL                  %[ssl_fc]
+  http-request set-header X-SSL-Client-Verify    %[ssl_c_verify]
+  http-request set-header X-SSL-Client-Serial    %{+Q}[ssl_c_serial,hex]
+  http-request set-header X-SSL-Client-Version   %{+Q}[ssl_c_version]
+  http-request set-header X-SSL-Client-SHA1      %{+Q}[ssl_c_sha1,hex]
+  http-request set-header X-SSL-Client-DN        %{+Q}[ssl_c_s_dn]
+  http-request set-header X-SSL-Client-CN        %{+Q}[ssl_c_s_dn(cn)]
+  http-request set-header X-SSL-Issuer           %{+Q}[ssl_c_i_dn]
+  http-request set-header X-SSL-Client-NotBefore %{+Q}[ssl_c_notbefore]
+  http-request set-header X-SSL-Client-NotAfter  %{+Q}[ssl_c_notafter]
+  http-request set-header X-SSL-Client-DER       %{+Q}[ssl_c_der,base64]
+  {{- end }}
 
   # map to backend
   # Search from most specific to general path (host case).

contrib/completions/zsh/oc

@@ -18,6 +18,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
@@ -84,6 +85,11 @@ var (
 	privkeyName       = "router.pem"
 	privkeyPath       = secretsPath + "/" + privkeyName
 
+	defaultMutualTLSAuth = "none"
+	clientCertConfigDir  = "/etc/pki/tls/client-certs"
+	clientCertConfigCA   = "ca.pem"
+	clientCertConfigCRL  = "crl.pem"
+
 	defaultCertificatePath = path.Join(defaultCertificateDir, "tls.crt")
 )
 
@@ -232,6 +238,23 @@ type RouterConfig struct {
 	Threads int32
 
 	Local bool
+
+	// MutualTLSAuth controls access to the router using a mutually agreed
+	// upon TLS authentication mechanism (example client certificates).
+	// One of: required | optional | none  - the default is none.
+	MutualTLSAuth string
+
+	// MutualTLSAuthCA contains the CA certificates that will be used
+	// to verify a client's certificate.
+	MutualTLSAuthCA string
+
+	// MutualTLSAuthCRL contains the certificate revocation list used to
+	// verify a client's certificate.
+	MutualTLSAuthCRL string
+
+	// MutualTLSAuthFilter contains the value to filter requests based on
+	// a client certificate subject field substring match.
+	MutualTLSAuthFilter string
 }
 
 const (
@@ -260,6 +283,8 @@ func NewCmdRouter(f kcmdutil.Factory, parentName, name string, out, errout io.Wr
 		StatsPort:     defaultStatsPort,
 		HostNetwork:   true,
 		HostPorts:     true,
+
+		MutualTLSAuth: defaultMutualTLSAuth,
 	}
 
 	cmd := &cobra.Command{
@@ -313,15 +338,25 @@ func NewCmdRouter(f kcmdutil.Factory, parentName, name string, out, errout io.Wr
 	cmd.Flags().BoolVar(&cfg.Local, "local", cfg.Local, "If true, do not contact the apiserver")
 	cmd.Flags().Int32Var(&cfg.Threads, "threads", cfg.Threads, "Specifies the number of threads for the haproxy router.")
 
+	cmd.Flags().StringVar(&cfg.MutualTLSAuth, "mutual-tls-auth", cfg.MutualTLSAuth, "Controls access to the router using mutually agreed upon TLS configuration (example client certificates). You can choose one of 'required', 'optional', or 'none'. The default is none.")
+	cmd.Flags().StringVar(&cfg.MutualTLSAuthCA, "mutual-tls-auth-ca", cfg.MutualTLSAuthCA, "Optional path to a file containing one or more CA certificates used for mutual TLS authentication. The CA certificate[s] are used by the router to verify a client's certificate.")
+	cmd.Flags().StringVar(&cfg.MutualTLSAuthCRL, "mutual-tls-auth-crl", cfg.MutualTLSAuthCRL, "Optional path to a file containing the certificate revocation list used for mutual TLS authentication. The certificate revocation list is used by the router to verify a client's certificate.")
+	cmd.Flags().StringVar(&cfg.MutualTLSAuthFilter, "mutual-tls-auth-filter", cfg.MutualTLSAuthFilter, "Optional regular expression to filter the client certificates. If the client certificate subject field does _not_ match this regular expression, requests will be rejected by the router.")
+
 	cfg.Action.BindForOutput(cmd.Flags())
 	cmd.Flags().String("output-version", "", "The preferred API versions of the output objects")
 
 	return cmd
 }
 
+// generateMutualTLSSecretName generates a mutual TLS auth secret name.
+func generateMutualTLSSecretName(prefix string) string {
+	return fmt.Sprintf("%s-mutual-tls-auth", prefix)
+}
+
 // generateSecretsConfig generates any Secret and Volume objects, such
 // as SSH private keys, that are necessary for the router container.
-func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []byte, certName string) ([]*kapi.Secret, []kapi.Volume, []kapi.VolumeMount, error) {
+func generateSecretsConfig(cfg *RouterConfig, namespace, certName string, defaultCert, mtlsAuthCA, mtlsAuthCRL []byte) ([]*kapi.Secret, []kapi.Volume, []kapi.VolumeMount, error) {
 	var secrets []*kapi.Secret
 	var volumes []kapi.Volume
 	var mounts []kapi.VolumeMount
@@ -428,6 +463,42 @@ func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []by
 	}
 	mounts = append(mounts, mount)
 
+	mtlsSecretData := map[string][]byte{}
+	if len(mtlsAuthCA) > 0 {
+		mtlsSecretData[clientCertConfigCA] = mtlsAuthCA
+	}
+	if len(mtlsAuthCRL) > 0 {
+		mtlsSecretData[clientCertConfigCRL] = mtlsAuthCRL
+	}
+
+	if len(mtlsSecretData) > 0 {
+		secretName := generateMutualTLSSecretName(cfg.Name)
+		secret := &kapi.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: secretName,
+			},
+			Data: mtlsSecretData,
+		}
+		secrets = append(secrets, secret)
+
+		volume := kapi.Volume{
+			Name: "mutual-tls-config",
+			VolumeSource: kapi.VolumeSource{
+				Secret: &kapi.SecretVolumeSource{
+					SecretName: secretName,
+				},
+			},
+		}
+		volumes = append(volumes, volume)
+
+		mount := kapi.VolumeMount{
+			Name:      volume.Name,
+			ReadOnly:  true,
+			MountPath: clientCertConfigDir,
+		}
+		mounts = append(mounts, mount)
+	}
+
 	return secrets, volumes, mounts, nil
 }
 
@@ -596,6 +667,14 @@ func RunCmdRouter(f kcmdutil.Factory, cmd *cobra.Command, out, errout io.Writer,
 		if err != nil {
 			return fmt.Errorf("error getting client: %v", err)
 		}
+
+		if len(cfg.MutualTLSAuthCA) > 0 || len(cfg.MutualTLSAuthCRL) > 0 {
+			secretName := generateMutualTLSSecretName(cfg.Name)
+			if _, err := kClient.Core().Secrets(namespace).Get(secretName, metav1.GetOptions{}); err == nil {
+				return fmt.Errorf("router could not be created: mutual tls secret %q already exists", secretName)
+			}
+		}
+
 		service, err := kClient.Core().Services(namespace).Get(name, metav1.GetOptions{})
 		if err != nil {
 			if !generate {
@@ -648,6 +727,20 @@ func RunCmdRouter(f kcmdutil.Factory, cmd *cobra.Command, out, errout io.Writer,
 		return fmt.Errorf("router could not be created; error reading default certificate file: %v", err)
 	}
 
+	mtlsAuthOptions := []string{"required", "optional", "none"}
+	allowedMutualTLSAuthOptions := sets.NewString(mtlsAuthOptions...)
+	if !allowedMutualTLSAuthOptions.Has(cfg.MutualTLSAuth) {
+		return fmt.Errorf("invalid mutual tls auth option %v, expected one of %v", cfg.MutualTLSAuth, mtlsAuthOptions)
+	}
+	mtlsAuthCA, err := fileutil.LoadData(cfg.MutualTLSAuthCA)
+	if err != nil {
+		return fmt.Errorf("reading ca certificates for mutual tls auth: %v", err)
+	}
+	mtlsAuthCRL, err := fileutil.LoadData(cfg.MutualTLSAuthCRL)
+	if err != nil {
+		return fmt.Errorf("reading certificate revocation list for mutual tls auth: %v", err)
+	}
+
 	if len(cfg.StatsPassword) == 0 {
 		cfg.StatsPassword = generateStatsPassword()
 		if !cfg.Action.ShouldPrint() {
@@ -707,6 +800,20 @@ func RunCmdRouter(f kcmdutil.Factory, cmd *cobra.Command, out, errout io.Writer,
 		env["ROUTER_METRICS_TLS_CERT_FILE"] = "/etc/pki/tls/metrics/tls.crt"
 		env["ROUTER_METRICS_TLS_KEY_FILE"] = "/etc/pki/tls/metrics/tls.key"
 	}
+	mtlsAuth := strings.TrimSpace(cfg.MutualTLSAuth)
+	if len(mtlsAuth) > 0 && mtlsAuth != defaultMutualTLSAuth {
+		env["ROUTER_MUTUAL_TLS_AUTH"] = cfg.MutualTLSAuth
+		if len(mtlsAuthCA) > 0 {
+			env["ROUTER_MUTUAL_TLS_AUTH_CA"] = path.Join(clientCertConfigDir, clientCertConfigCA)
+		}
+		if len(mtlsAuthCRL) > 0 {
+			env["ROUTER_MUTUAL_TLS_AUTH_CRL"] = path.Join(clientCertConfigDir, clientCertConfigCRL)
+		}
+		if len(cfg.MutualTLSAuthFilter) > 0 {
+			env["ROUTER_MUTUAL_TLS_AUTH_FILTER"] = strings.Replace(cfg.MutualTLSAuthFilter, " ", "\\ ", -1)
+		}
+	}
+
 	env.Add(secretEnv)
 	if len(defaultCert) > 0 {
 		if cfg.SecretsAsEnv {
@@ -717,7 +824,7 @@ func RunCmdRouter(f kcmdutil.Factory, cmd *cobra.Command, out, errout io.Writer,
 	}
 	env.Add(app.Environment{"DEFAULT_CERTIFICATE_DIR": defaultCertificateDir})
 	var certName = fmt.Sprintf("%s-certs", cfg.Name)
-	secrets, volumes, mounts, err := generateSecretsConfig(cfg, namespace, defaultCert, certName)
+	secrets, volumes, mounts, err := generateSecretsConfig(cfg, namespace, certName, defaultCert, mtlsAuthCA, mtlsAuthCRL)
 	if err != nil {
 		return fmt.Errorf("router could not be created: %v", err)
 	}