Skip to content

Allow user to use arbitraty userdata #651

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package v1alpha1

import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeadmv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
)
Expand Down Expand Up @@ -77,6 +78,11 @@ type AWSMachineProviderSpec struct {
// KubeadmConfiguration holds the kubeadm configuration options
// +optional
KubeadmConfiguration KubeadmConfiguration `json:"kubeadmConfiguration,omitempty"`

// UserDataSecret contains a local reference to a secret that contains the
// UserData to apply to the instance
// + optional
UserDataSecret *corev1.LocalObjectReference `json:"userDataSecret,omitempty"`
}

// KubeadmConfiguration holds the various configurations that kubeadm uses
Expand Down
6 changes: 6 additions & 0 deletions pkg/apis/awsprovider/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pkg/cloud/aws/actuators/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ go_library(
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library",
"//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library",
Expand Down
14 changes: 7 additions & 7 deletions pkg/cloud/aws/actuators/machine/actuator.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,12 @@ func (a *Actuator) isNodeJoin(scope *actuators.MachineScope, controlPlaneMachine
func (a *Actuator) Create(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) error {
klog.Infof("Creating machine %v for cluster %v", machine.Name, cluster.Name)

scope, err := actuators.NewMachineScope(actuators.MachineScopeParams{Machine: machine, Cluster: cluster, Client: a.client})
coreClient, err := a.coreV1Client(cluster)
if err != nil {
return errors.Wrapf(err, "failed to retrieve corev1 client for cluster %q", cluster.Name)
}

scope, err := actuators.NewMachineScope(actuators.MachineScopeParams{Machine: machine, Cluster: cluster, Client: a.client, CoreClient: coreClient})
if err != nil {
return errors.Errorf("failed to create scope: %+v", err)
}
Expand All @@ -149,12 +154,7 @@ func (a *Actuator) Create(ctx context.Context, cluster *clusterv1.Cluster, machi

var bootstrapToken string
if isNodeJoin {
coreClient, err := a.coreV1Client(cluster)
if err != nil {
return errors.Wrapf(err, "failed to retrieve corev1 client for cluster %q", cluster.Name)
}

bootstrapToken, err = tokens.NewBootstrap(coreClient, defaultTokenTTL)
bootstrapToken, err = tokens.NewBootstrap(scope.CoreClient, defaultTokenTTL)
if err != nil {
return errors.Wrapf(err, "failed to create new bootstrap token")
}
Expand Down
15 changes: 12 additions & 3 deletions pkg/cloud/aws/actuators/machine_scope.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/klog"
"sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsprovider/v1alpha1"
clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
Expand All @@ -30,9 +31,10 @@ import (
// MachineScopeParams defines the input parameters used to create a new MachineScope.
type MachineScopeParams struct {
AWSClients
Cluster *clusterv1.Cluster
Machine *clusterv1.Machine
Client client.ClusterV1alpha1Interface
Cluster *clusterv1.Cluster
Machine *clusterv1.Machine
Client client.ClusterV1alpha1Interface
CoreClient v1.CoreV1Interface
}

// NewMachineScope creates a new MachineScope from the supplied parameters.
Expand All @@ -58,12 +60,18 @@ func NewMachineScope(params MachineScopeParams) (*MachineScope, error) {
machineClient = params.Client.Machines(params.Machine.Namespace)
}

var coreClient v1.CoreV1Interface
if params.Client != nil {
coreClient = params.CoreClient
}

return &MachineScope{
Scope: scope,
Machine: params.Machine,
MachineClient: machineClient,
MachineConfig: machineConfig,
MachineStatus: machineStatus,
CoreClient: coreClient,
}, nil
}

Expand All @@ -75,6 +83,7 @@ type MachineScope struct {
MachineClient client.MachineInterface
MachineConfig *v1alpha1.AWSMachineProviderSpec
MachineStatus *v1alpha1.AWSMachineProviderStatus
CoreClient v1.CoreV1Interface
}

// Name returns the machine name.
Expand Down
103 changes: 71 additions & 32 deletions pkg/cloud/aws/services/ec2/instances.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,15 +158,73 @@ func (s *Service) createInstance(machine *actuators.MachineScope, bootstrapToken
}

// apply values based on the role of the machine
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this comment still valid?

userData, err := s.getUserData(machine, bootstrapToken, caCertHash, kubeConfig)
if err != nil {
return nil, awserrors.NewFailedDependency(
errors.Errorf("failed to get userData for machine %q", machine.Name()),
)
}
input.UserData = &userData

securityGroupID, err := s.getSecurityGroupID(machine)
if err != nil {
return nil, awserrors.NewFailedDependency(
errors.Errorf("failed to get securityGroup ID for machine %q", machine.Name()),
)
}
input.SecurityGroupIDs = append(input.SecurityGroupIDs, securityGroupID)

// Pick SSH key, if any.
if machine.MachineConfig.KeyName != "" {
input.KeyName = aws.String(machine.MachineConfig.KeyName)
} else {
input.KeyName = aws.String(defaultSSHKeyName)
}

out, err := s.runInstance(machine.Role(), input)
if err != nil {
return nil, err
}

record.Eventf(machine.Machine, "CreatedInstance", "Created new %s instance with id %q", machine.Role(), out.ID)
return out, nil
}

func (s *Service) getSecurityGroupID(machine *actuators.MachineScope) (string, error) {
switch machine.Role() {
case "controlplane":
if s.scope.SecurityGroups()[v1alpha1.SecurityGroupControlPlane] == nil {
return nil, awserrors.NewFailedDependency(
return "", awserrors.NewFailedDependency(
errors.New("failed to run controlplane, security group not available"),
)
}
return s.scope.SecurityGroups()[v1alpha1.SecurityGroupControlPlane].ID, nil
case "node":
return s.scope.SecurityGroups()[v1alpha1.SecurityGroupNode].ID, nil
default:
return "", errors.Errorf("Unknown node role %q", machine.Role())
}
return "", nil
}

func (s *Service) getUserData(machine *actuators.MachineScope, bootstrapToken, caCertHash string, kubeConfig string) (string, error) {
var userData string

var userData string
userData, err := userdata.GetUserDataFromSecret(machine)
if err != nil {
return "", errors.Errorf("failed getting userData from secret %v", err)
}
if userData != "" {
return userData, nil
}

switch machine.Role() {
case "controlplane":
if s.scope.SecurityGroups()[v1alpha1.SecurityGroupControlPlane] == nil {
return "", awserrors.NewFailedDependency(
errors.New("failed to run controlplane, security group not available"),
)
}

if bootstrapToken != "" {
klog.V(2).Infof("Allowing machine %q to join control plane for cluster %q", machine.Name(), s.scope.Name())
Expand All @@ -175,7 +233,7 @@ func (s *Service) createInstance(machine *actuators.MachineScope, bootstrapToken
kubeadm.SetControlPlaneJoinConfigurationOverrides(&machine.MachineConfig.KubeadmConfiguration.Join)
joinConfigurationYAML, err := kubeadm.ConfigurationToYAML(&machine.MachineConfig.KubeadmConfiguration.Join)
if err != nil {
return nil, err
return "", err
}

userData, err = userdata.JoinControlPlane(&userdata.ContolPlaneJoinInput{
Expand All @@ -190,26 +248,26 @@ func (s *Service) createInstance(machine *actuators.MachineScope, bootstrapToken
JoinConfiguration: joinConfigurationYAML,
})
if err != nil {
return input, err
return userData, err
}
} else {
klog.V(2).Infof("Machine %q is the first controlplane machine for cluster %q", machine.Name(), s.scope.Name())
if !s.scope.ClusterConfig.CAKeyPair.HasCertAndKey() {
return nil, awserrors.NewFailedDependency(
return "", awserrors.NewFailedDependency(
errors.New("failed to run controlplane, missing CAPrivateKey"),
)
}

kubeadm.SetClusterConfigurationOverrides(machine, &s.scope.ClusterConfig.ClusterConfiguration)
clusterConfigYAML, err := kubeadm.ConfigurationToYAML(&s.scope.ClusterConfig.ClusterConfiguration)
if err != nil {
return nil, err
return "", err
}

kubeadm.SetInitConfigurationOverrides(&machine.MachineConfig.KubeadmConfiguration.Init)
initConfigYAML, err := kubeadm.ConfigurationToYAML(&machine.MachineConfig.KubeadmConfiguration.Init)
if err != nil {
return nil, err
return "", err
}

userData, err = userdata.NewControlPlane(&userdata.ControlPlaneInput{
Expand All @@ -226,49 +284,30 @@ func (s *Service) createInstance(machine *actuators.MachineScope, bootstrapToken
})

if err != nil {
return input, err
return "", err
}
}

input.UserData = aws.String(userData)
input.SecurityGroupIDs = append(input.SecurityGroupIDs, s.scope.SecurityGroups()[v1alpha1.SecurityGroupControlPlane].ID)
case "node":
input.SecurityGroupIDs = append(input.SecurityGroupIDs, s.scope.SecurityGroups()[v1alpha1.SecurityGroupNode].ID)

kubeadm.SetJoinNodeConfigurationOverrides(caCertHash, bootstrapToken, machine, &machine.MachineConfig.KubeadmConfiguration.Join)
joinConfigurationYAML, err := kubeadm.ConfigurationToYAML(&machine.MachineConfig.KubeadmConfiguration.Join)
if err != nil {
return nil, err
return "", err
}

userData, err := userdata.NewNode(&userdata.NodeInput{
JoinConfiguration: joinConfigurationYAML,
})

if err != nil {
return input, err
return "", err
}

input.UserData = aws.String(userData)
userData = userData

default:
return nil, errors.Errorf("Unknown node role %q", machine.Role())
}

// Pick SSH key, if any.
if machine.MachineConfig.KeyName != "" {
input.KeyName = aws.String(machine.MachineConfig.KeyName)
} else {
input.KeyName = aws.String(defaultSSHKeyName)
return "", errors.Errorf("Unknown node role %q", machine.Role())
}

out, err := s.runInstance(machine.Role(), input)
if err != nil {
return nil, err
}

record.Eventf(machine.Machine, "CreatedInstance", "Created new %s instance with id %q", machine.Role(), out.ID)
return out, nil
return userData, nil
}

// TerminateInstance terminates an EC2 instance.
Expand Down
6 changes: 5 additions & 1 deletion pkg/cloud/aws/services/userdata/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ go_library(
],
importpath = "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services/userdata",
visibility = ["//visibility:public"],
deps = ["//vendor/github.com/pkg/errors:go_default_library"],
deps = [
"//pkg/cloud/aws/actuators:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
],
)

go_test(
Expand Down
21 changes: 21 additions & 0 deletions pkg/cloud/aws/services/userdata/userdata.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,12 @@ package userdata

import (
"bytes"
"encoding/base64"
"text/template"

"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/actuators"
)

const (
Expand Down Expand Up @@ -90,3 +93,21 @@ func funcMap(funcs map[string]interface{}) template.FuncMap {

return funcMap
}

func GetUserDataFromSecret(machine *actuators.MachineScope) (string, error) {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to create a unit test case for this function?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A couple of suggestions and questions:

  • flip around the nil check and return early if the UserDataSecret is nil
  • the userData var is unnecessary, directly encode and return at line 105 and remove the else statement
  • why is it an error if the secret doesn't contain the userData key? My expectation would be to log an error and return nothing
  • please add godoc and a test to this function

if machine.MachineConfig.UserDataSecret != nil {
userData := []byte{}
userDataSecret, err := machine.CoreClient.Secrets(machine.Namespace()).Get(machine.MachineConfig.UserDataSecret.Name, metav1.GetOptions{})
Copy link
Contributor

@ashish-amarnath ashish-amarnath Mar 13, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How will this secret be created?
If the secret doesn't exist, the machine reconciliation will not make progress until the secret is created. Correct?

Copy link
Member Author

@enxebre enxebre Mar 13, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if err != nil {
return "", errors.Errorf("failed to get userData secret %q", machine.MachineConfig.UserDataSecret.Name)
}
if data, exists := userDataSecret.Data["userData"]; exists {
userData = data
} else {
return "", errors.Errorf("secret %q does not have field %q", machine.MachineConfig.UserDataSecret.Name, "userData")
}
encodedUserData := base64.StdEncoding.EncodeToString(userData)
return encodedUserData, nil
}
return "", nil
}