Allow user to use arbitraty userdata

pkg/apis/awsprovider/v1alpha1/awsmachineproviderconfig_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeadmv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
 )
@@ -77,6 +78,11 @@ type AWSMachineProviderSpec struct {
 	// KubeadmConfiguration holds the kubeadm configuration options
 	// +optional
 	KubeadmConfiguration KubeadmConfiguration `json:"kubeadmConfiguration,omitempty"`
+
+	// UserDataSecret contains a local reference to a secret that contains the
+	// UserData to apply to the instance
+	// + optional
+	UserDataSecret *corev1.LocalObjectReference `json:"userDataSecret,omitempty"`
 }
 
 // KubeadmConfiguration holds the various configurations that kubeadm uses

pkg/cloud/aws/actuators/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
         "//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library",

pkg/cloud/aws/actuators/machine/actuator.go

@@ -126,7 +126,12 @@ func (a *Actuator) isNodeJoin(scope *actuators.MachineScope, controlPlaneMachine
 func (a *Actuator) Create(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) error {
 	klog.Infof("Creating machine %v for cluster %v", machine.Name, cluster.Name)
 
-	scope, err := actuators.NewMachineScope(actuators.MachineScopeParams{Machine: machine, Cluster: cluster, Client: a.client})
+	coreClient, err := a.coreV1Client(cluster)
+	if err != nil {
+		return errors.Wrapf(err, "failed to retrieve corev1 client for cluster %q", cluster.Name)
+	}
+
+	scope, err := actuators.NewMachineScope(actuators.MachineScopeParams{Machine: machine, Cluster: cluster, Client: a.client, CoreClient: coreClient})
 	if err != nil {
 		return errors.Errorf("failed to create scope: %+v", err)
 	}
@@ -149,12 +154,7 @@ func (a *Actuator) Create(ctx context.Context, cluster *clusterv1.Cluster, machi
 
 	var bootstrapToken string
 	if isNodeJoin {
-		coreClient, err := a.coreV1Client(cluster)
-		if err != nil {
-			return errors.Wrapf(err, "failed to retrieve corev1 client for cluster %q", cluster.Name)
-		}
-
-		bootstrapToken, err = tokens.NewBootstrap(coreClient, defaultTokenTTL)
+		bootstrapToken, err = tokens.NewBootstrap(scope.CoreClient, defaultTokenTTL)
 		if err != nil {
 			return errors.Wrapf(err, "failed to create new bootstrap token")
 		}

pkg/cloud/aws/actuators/machine_scope.go

@@ -20,6 +20,7 @@ import (
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/klog"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsprovider/v1alpha1"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
@@ -30,9 +31,10 @@ import (
 // MachineScopeParams defines the input parameters used to create a new MachineScope.
 type MachineScopeParams struct {
 	AWSClients
-	Cluster *clusterv1.Cluster
-	Machine *clusterv1.Machine
-	Client  client.ClusterV1alpha1Interface
+	Cluster    *clusterv1.Cluster
+	Machine    *clusterv1.Machine
+	Client     client.ClusterV1alpha1Interface
+	CoreClient v1.CoreV1Interface
 }
 
 // NewMachineScope creates a new MachineScope from the supplied parameters.
@@ -58,12 +60,18 @@ func NewMachineScope(params MachineScopeParams) (*MachineScope, error) {
 		machineClient = params.Client.Machines(params.Machine.Namespace)
 	}
 
+	var coreClient v1.CoreV1Interface
+	if params.Client != nil {
+		coreClient = params.CoreClient
+	}
+
 	return &MachineScope{
 		Scope:         scope,
 		Machine:       params.Machine,
 		MachineClient: machineClient,
 		MachineConfig: machineConfig,
 		MachineStatus: machineStatus,
+		CoreClient:    coreClient,
 	}, nil
 }
 
@@ -75,6 +83,7 @@ type MachineScope struct {
 	MachineClient client.MachineInterface
 	MachineConfig *v1alpha1.AWSMachineProviderSpec
 	MachineStatus *v1alpha1.AWSMachineProviderStatus
+	CoreClient    v1.CoreV1Interface
 }
 
 // Name returns the machine name.

pkg/cloud/aws/services/ec2/instances.go

@@ -158,15 +158,73 @@ func (s *Service) createInstance(machine *actuators.MachineScope, bootstrapToken
 	}
 
 	// apply values based on the role of the machine
+	userData, err := s.getUserData(machine, bootstrapToken, caCertHash, kubeConfig)
+	if err != nil {
+		return nil, awserrors.NewFailedDependency(
+			errors.Errorf("failed to get userData for machine %q", machine.Name()),
+		)
+	}
+	input.UserData = &userData
+
+	securityGroupID, err := s.getSecurityGroupID(machine)
+	if err != nil {
+		return nil, awserrors.NewFailedDependency(
+			errors.Errorf("failed to get securityGroup ID for machine %q", machine.Name()),
+		)
+	}
+	input.SecurityGroupIDs = append(input.SecurityGroupIDs, securityGroupID)
+
+	// Pick SSH key, if any.
+	if machine.MachineConfig.KeyName != "" {
+		input.KeyName = aws.String(machine.MachineConfig.KeyName)
+	} else {
+		input.KeyName = aws.String(defaultSSHKeyName)
+	}
+
+	out, err := s.runInstance(machine.Role(), input)
+	if err != nil {
+		return nil, err
+	}
+
+	record.Eventf(machine.Machine, "CreatedInstance", "Created new %s instance with id %q", machine.Role(), out.ID)
+	return out, nil
+}
+
+func (s *Service) getSecurityGroupID(machine *actuators.MachineScope) (string, error) {
 	switch machine.Role() {
 	case "controlplane":
 		if s.scope.SecurityGroups()[v1alpha1.SecurityGroupControlPlane] == nil {
-			return nil, awserrors.NewFailedDependency(
+			return "", awserrors.NewFailedDependency(
 				errors.New("failed to run controlplane, security group not available"),
 			)
 		}
+		return s.scope.SecurityGroups()[v1alpha1.SecurityGroupControlPlane].ID, nil
+	case "node":
+		return s.scope.SecurityGroups()[v1alpha1.SecurityGroupNode].ID, nil
+	default:
+		return "", errors.Errorf("Unknown node role %q", machine.Role())
+	}
+	return "", nil
+}
+
+func (s *Service) getUserData(machine *actuators.MachineScope, bootstrapToken, caCertHash string, kubeConfig string) (string, error) {
+	var userData string
 
-		var userData string
+	userData, err := userdata.GetUserDataFromSecret(machine)
+	if err != nil {
+		return "", errors.Errorf("failed getting userData from secret %v", err)
+	}
+	if userData != "" {
+		return userData, nil
+	}
+
+	switch machine.Role() {
+	case "controlplane":
+		if s.scope.SecurityGroups()[v1alpha1.SecurityGroupControlPlane] == nil {
+			return "", awserrors.NewFailedDependency(
+				errors.New("failed to run controlplane, security group not available"),
+			)
+		}
 
 		if bootstrapToken != "" {
 			klog.V(2).Infof("Allowing machine %q to join control plane for cluster %q", machine.Name(), s.scope.Name())
@@ -175,7 +233,7 @@ func (s *Service) createInstance(machine *actuators.MachineScope, bootstrapToken
 			kubeadm.SetControlPlaneJoinConfigurationOverrides(&machine.MachineConfig.KubeadmConfiguration.Join)
 			joinConfigurationYAML, err := kubeadm.ConfigurationToYAML(&machine.MachineConfig.KubeadmConfiguration.Join)
 			if err != nil {
-				return nil, err
+				return "", err
 			}
 
 			userData, err = userdata.JoinControlPlane(&userdata.ContolPlaneJoinInput{
@@ -190,26 +248,26 @@ func (s *Service) createInstance(machine *actuators.MachineScope, bootstrapToken
 				JoinConfiguration: joinConfigurationYAML,
 			})
 			if err != nil {
-				return input, err
+				return userData, err
 			}
 		} else {
 			klog.V(2).Infof("Machine %q is the first controlplane machine for cluster %q", machine.Name(), s.scope.Name())
 			if !s.scope.ClusterConfig.CAKeyPair.HasCertAndKey() {
-				return nil, awserrors.NewFailedDependency(
+				return "", awserrors.NewFailedDependency(
 					errors.New("failed to run controlplane, missing CAPrivateKey"),
 				)
 			}
 
 			kubeadm.SetClusterConfigurationOverrides(machine, &s.scope.ClusterConfig.ClusterConfiguration)
 			clusterConfigYAML, err := kubeadm.ConfigurationToYAML(&s.scope.ClusterConfig.ClusterConfiguration)
 			if err != nil {
-				return nil, err
+				return "", err
 			}
 
 			kubeadm.SetInitConfigurationOverrides(&machine.MachineConfig.KubeadmConfiguration.Init)
 			initConfigYAML, err := kubeadm.ConfigurationToYAML(&machine.MachineConfig.KubeadmConfiguration.Init)
 			if err != nil {
-				return nil, err
+				return "", err
 			}
 
 			userData, err = userdata.NewControlPlane(&userdata.ControlPlaneInput{
@@ -226,49 +284,30 @@ func (s *Service) createInstance(machine *actuators.MachineScope, bootstrapToken
 			})
 
 			if err != nil {
-				return input, err
+				return "", err
 			}
 		}
-
-		input.UserData = aws.String(userData)
-		input.SecurityGroupIDs = append(input.SecurityGroupIDs, s.scope.SecurityGroups()[v1alpha1.SecurityGroupControlPlane].ID)
 	case "node":
-		input.SecurityGroupIDs = append(input.SecurityGroupIDs, s.scope.SecurityGroups()[v1alpha1.SecurityGroupNode].ID)
-
 		kubeadm.SetJoinNodeConfigurationOverrides(caCertHash, bootstrapToken, machine, &machine.MachineConfig.KubeadmConfiguration.Join)
 		joinConfigurationYAML, err := kubeadm.ConfigurationToYAML(&machine.MachineConfig.KubeadmConfiguration.Join)
 		if err != nil {
-			return nil, err
+			return "", err
 		}
 
 		userData, err := userdata.NewNode(&userdata.NodeInput{
 			JoinConfiguration: joinConfigurationYAML,
 		})
 
 		if err != nil {
-			return input, err
+			return "", err
 		}
-
-		input.UserData = aws.String(userData)
+		userData = userData
 
 	default:
-		return nil, errors.Errorf("Unknown node role %q", machine.Role())
-	}
-
-	// Pick SSH key, if any.
-	if machine.MachineConfig.KeyName != "" {
-		input.KeyName = aws.String(machine.MachineConfig.KeyName)
-	} else {
-		input.KeyName = aws.String(defaultSSHKeyName)
+		return "", errors.Errorf("Unknown node role %q", machine.Role())
 	}
 
-	out, err := s.runInstance(machine.Role(), input)
-	if err != nil {
-		return nil, err
-	}
-
-	record.Eventf(machine.Machine, "CreatedInstance", "Created new %s instance with id %q", machine.Role(), out.ID)
-	return out, nil
+	return userData, nil
 }
 
 // TerminateInstance terminates an EC2 instance.

pkg/cloud/aws/services/userdata/BUILD.bazel

@@ -10,7 +10,11 @@ go_library(
     ],
     importpath = "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services/userdata",
     visibility = ["//visibility:public"],
-    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+    deps = [
+        "//pkg/cloud/aws/actuators:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+    ],
 )
 
 go_test(

pkg/cloud/aws/services/userdata/userdata.go

@@ -18,9 +18,12 @@ package userdata
 
 import (
 	"bytes"
+	"encoding/base64"
 	"text/template"
 
 	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/actuators"
 )
 
 const (
@@ -90,3 +93,21 @@ func funcMap(funcs map[string]interface{}) template.FuncMap {
 
 	return funcMap
 }
+
+func GetUserDataFromSecret(machine *actuators.MachineScope) (string, error) {
+	if machine.MachineConfig.UserDataSecret != nil {
+		userData := []byte{}
+		userDataSecret, err := machine.CoreClient.Secrets(machine.Namespace()).Get(machine.MachineConfig.UserDataSecret.Name, metav1.GetOptions{})
+		if err != nil {
+			return "", errors.Errorf("failed to get userData secret %q", machine.MachineConfig.UserDataSecret.Name)
+		}
+		if data, exists := userDataSecret.Data["userData"]; exists {
+			userData = data
+		} else {
+			return "", errors.Errorf("secret %q does not have field %q", machine.MachineConfig.UserDataSecret.Name, "userData")
+		}
+		encodedUserData := base64.StdEncoding.EncodeToString(userData)
+		return encodedUserData, nil
+	}
+	return "", nil
+}