Add AWSMachines to back the EC2 instances in AWSMachinePools and AWSManagedMachinePools

Co-authored-by: Cameron McAvoy <[email protected]>

Author: AndiDog

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml

@@ -1144,6 +1144,10 @@ spec:
                   can be added as events to the Machine object and/or logged in the
                   controller's output.
                 type: string
+              infrastructureMachineKind:
+                description: InfrastructureMachineKind is the kind of the infrastructure
+                  resources behind MachinePool Machines.
+                type: string
               instances:
                 description: Instances contains the status for each instance in the
                   pool

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml

@@ -1072,6 +1072,10 @@ spec:
                   can be added as events to the MachinePool object and/or logged in the
                   controller's output.
                 type: string
+              infrastructureMachineKind:
+                description: InfrastructureMachineKind is the kind of the infrastructure
+                  resources behind MachinePool Machines.
+                type: string
               launchTemplateID:
                 description: The ID of the launch template
                 type: string

config/rbac/role.yaml

@@ -72,7 +72,6 @@ rules:
   - clusters
   - clusters/status
   - machinedeployments
-  - machines
   - machines/status
   verbs:
   - get
@@ -88,6 +87,15 @@ rules:
   - list
   - patch
   - watch
+- apiGroups:
+  - cluster.x-k8s.io
+  resources:
+  - machines
+  verbs:
+  - delete
+  - get
+  - list
+  - watch
 - apiGroups:
   - controlplane.cluster.x-k8s.io
   resources:
@@ -150,7 +158,6 @@ rules:
   - awsclusters
   - awsfargateprofiles
   - awsmachinepools
-  - awsmachines
   - awsmanagedclusters
   - awsmanagedmachinepools
   - rosaclusters
@@ -186,6 +193,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - infrastructure.cluster.x-k8s.io
+  resources:
+  - awsmachines
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - infrastructure.cluster.x-k8s.io
   resources:

controllers/awsmachine_controller.go

@@ -143,10 +143,11 @@ func (r *AWSMachineReconciler) getObjectStoreService(scope scope.S3Scope) servic
 	return s3.NewService(scope)
 }
 
-// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachines,verbs=get;list;watch;update;patch;delete
-// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachines/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=*,verbs=get;list;watch
-// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines;machines/status,verbs=get;list;watch
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachines,verbs=create;get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachines/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;list;watch;delete
+// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines/status,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch
@@ -459,6 +460,7 @@ func (r *AWSMachineReconciler) findInstance(machineScope *scope.MachineScope, ec
 	return instance, nil
 }
 
+//nolint:gocyclo
 func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, ec2Scope scope.EC2Scope, elbScope scope.ELBScope, objectStoreScope scope.S3Scope) (ctrl.Result, error) {
 	machineScope.Trace("Reconciling AWSMachine")
 
@@ -482,7 +484,7 @@ func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *
 	}
 
 	// Make sure bootstrap data is available and populated.
-	if machineScope.Machine.Spec.Bootstrap.DataSecretName == nil {
+	if !machineScope.IsMachinePoolMachine() && machineScope.Machine.Spec.Bootstrap.DataSecretName == nil {
 		machineScope.Info("Bootstrap data secret reference is not yet available")
 		conditions.MarkFalse(machineScope.AWSMachine, infrav1.InstanceReadyCondition, infrav1.WaitingForBootstrapDataReason, clusterv1.ConditionSeverityInfo, "")
 		return ctrl.Result{}, nil
@@ -497,6 +499,12 @@ func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *
 		conditions.MarkUnknown(machineScope.AWSMachine, infrav1.InstanceReadyCondition, infrav1.InstanceNotFoundReason, err.Error())
 		return ctrl.Result{}, err
 	}
+	if instance == nil && machineScope.IsMachinePoolMachine() {
+		err = errors.New("no instance found for machine pool")
+		machineScope.Error(err, "unable to find instance")
+		conditions.MarkUnknown(machineScope.AWSMachine, infrav1.InstanceReadyCondition, infrav1.InstanceNotFoundReason, err.Error())
+		return ctrl.Result{}, err
+	}
 
 	// If the AWSMachine doesn't have our finalizer, add it.
 	if controllerutil.AddFinalizer(machineScope.AWSMachine, infrav1.MachineFinalizer) {
@@ -586,9 +594,18 @@ func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *
 		conditions.MarkTrue(machineScope.AWSMachine, infrav1.InstanceReadyCondition)
 	case infrav1.InstanceStateShuttingDown, infrav1.InstanceStateTerminated:
 		machineScope.SetNotReady()
-		machineScope.Info("Unexpected EC2 instance termination", "state", instance.State, "instance-id", *machineScope.GetInstanceID())
-		r.Recorder.Eventf(machineScope.AWSMachine, corev1.EventTypeWarning, "InstanceUnexpectedTermination", "Unexpected EC2 instance termination")
-		conditions.MarkFalse(machineScope.AWSMachine, infrav1.InstanceReadyCondition, infrav1.InstanceTerminatedReason, clusterv1.ConditionSeverityError, "")
+
+		if machineScope.IsMachinePoolMachine() {
+			// In an auto-scaling group, instance termination is perfectly normal on scale-down
+			// and therefore should not be reported as error.
+			machineScope.Info("EC2 instance of machine pool was terminated", "state", instance.State, "instance-id", *machineScope.GetInstanceID())
+			r.Recorder.Eventf(machineScope.AWSMachine, corev1.EventTypeNormal, infrav1.InstanceTerminatedReason, "EC2 instance termination")
+			conditions.MarkFalse(machineScope.AWSMachine, infrav1.InstanceReadyCondition, infrav1.InstanceTerminatedReason, clusterv1.ConditionSeverityInfo, "")
+		} else {
+			machineScope.Info("Unexpected EC2 instance termination", "state", instance.State, "instance-id", *machineScope.GetInstanceID())
+			r.Recorder.Eventf(machineScope.AWSMachine, corev1.EventTypeWarning, "InstanceUnexpectedTermination", "Unexpected EC2 instance termination")
+			conditions.MarkFalse(machineScope.AWSMachine, infrav1.InstanceReadyCondition, infrav1.InstanceTerminatedReason, clusterv1.ConditionSeverityError, "")
+		}
 	default:
 		machineScope.SetNotReady()
 		machineScope.Info("EC2 instance state is undefined", "state", instance.State, "instance-id", *machineScope.GetInstanceID())
@@ -599,14 +616,18 @@ func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *
 	}
 
 	// reconcile the deletion of the bootstrap data secret now that we have updated instance state
-	if deleteSecretErr := r.deleteBootstrapData(machineScope, clusterScope, objectStoreScope); deleteSecretErr != nil {
-		r.Log.Error(deleteSecretErr, "unable to delete secrets")
-		return ctrl.Result{}, deleteSecretErr
-	}
+	if !machineScope.IsMachinePoolMachine() {
+		if deleteSecretErr := r.deleteBootstrapData(machineScope, clusterScope, objectStoreScope); deleteSecretErr != nil {
+			r.Log.Error(deleteSecretErr, "unable to delete secrets")
+			return ctrl.Result{}, deleteSecretErr
+		}
 
-	if instance.State == infrav1.InstanceStateTerminated {
-		machineScope.SetFailureReason(capierrors.UpdateMachineError)
-		machineScope.SetFailureMessage(errors.Errorf("EC2 instance state %q is unexpected", instance.State))
+		// For machine pool machines, it is expected that the ASG terminates instances at any time,
+		// so no error is logged for those.
+		if instance.State == infrav1.InstanceStateTerminated {
+			machineScope.SetFailureReason(capierrors.UpdateMachineError)
+			machineScope.SetFailureMessage(errors.Errorf("EC2 instance state %q is unexpected", instance.State))
+		}
 	}
 
 	// tasks that can take place during all known instance states
@@ -876,9 +897,13 @@ func getIgnitionVersion(scope *scope.MachineScope) string {
 }
 
 func (r *AWSMachineReconciler) deleteBootstrapData(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, objectStoreScope scope.S3Scope) error {
-	_, userDataFormat, err := machineScope.GetRawBootstrapDataWithFormat()
-	if client.IgnoreNotFound(err) != nil {
-		return errors.Wrap(err, "failed to get raw userdata")
+	var userDataFormat string
+	var err error
+	if machineScope.Machine.Spec.Bootstrap.DataSecretName != nil {
+		_, userDataFormat, err = machineScope.GetRawBootstrapDataWithFormat()
+		if client.IgnoreNotFound(err) != nil {
+			return errors.Wrap(err, "failed to get raw userdata")
+		}
 	}
 
 	if machineScope.UseSecretsManager(userDataFormat) {

exp/api/v1beta1/conversion.go

@@ -52,6 +52,7 @@ func (src *AWSMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 	if restored.Spec.AvailabilityZoneSubnetType != nil {
 		dst.Spec.AvailabilityZoneSubnetType = restored.Spec.AvailabilityZoneSubnetType
 	}
+	dst.Status.InfrastructureMachineKind = restored.Status.InfrastructureMachineKind
 
 	if restored.Spec.AWSLaunchTemplate.PrivateDNSName != nil {
 		dst.Spec.AWSLaunchTemplate.PrivateDNSName = restored.Spec.AWSLaunchTemplate.PrivateDNSName
@@ -83,7 +84,6 @@ func (src *AWSMachinePoolList) ConvertTo(dstRaw conversion.Hub) error {
 // ConvertFrom converts the v1beta2 AWSMachinePoolList receiver to v1beta1 AWSMachinePoolList.
 func (r *AWSMachinePoolList) ConvertFrom(srcRaw conversion.Hub) error {
 	src := srcRaw.(*infrav1exp.AWSMachinePoolList)
-
 	return Convert_v1beta2_AWSMachinePoolList_To_v1beta1_AWSMachinePoolList(src, r, nil)
 }
 
@@ -114,6 +114,8 @@ func (src *AWSManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 		dst.Spec.AvailabilityZoneSubnetType = restored.Spec.AvailabilityZoneSubnetType
 	}
 
+	dst.Status.InfrastructureMachineKind = restored.Status.InfrastructureMachineKind
+
 	return nil
 }
 
@@ -133,6 +135,14 @@ func Convert_v1beta2_AWSManagedMachinePoolSpec_To_v1beta1_AWSManagedMachinePoolS
 	return autoConvert_v1beta2_AWSManagedMachinePoolSpec_To_v1beta1_AWSManagedMachinePoolSpec(in, out, s)
 }
 
+func Convert_v1beta2_AWSMachinePoolStatus_To_v1beta1_AWSMachinePoolStatus(in *infrav1exp.AWSMachinePoolStatus, out *AWSMachinePoolStatus, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_AWSMachinePoolStatus_To_v1beta1_AWSMachinePoolStatus(in, out, s)
+}
+
+func Convert_v1beta2_AWSManagedMachinePoolStatus_To_v1beta1_AWSManagedMachinePoolStatus(in *infrav1exp.AWSManagedMachinePoolStatus, out *AWSManagedMachinePoolStatus, s apiconversion.Scope) error {
+	return autoConvert_v1beta2_AWSManagedMachinePoolStatus_To_v1beta1_AWSManagedMachinePoolStatus(in, out, s)
+}
+
 // ConvertTo converts the v1beta1 AWSManagedMachinePoolList receiver to a v1beta2 AWSManagedMachinePoolList.
 func (src *AWSManagedMachinePoolList) ConvertTo(dstRaw conversion.Hub) error {
 	dst := dstRaw.(*infrav1exp.AWSManagedMachinePoolList)

exp/api/v1beta1/zz_generated.conversion.go

@@ -210,6 +210,10 @@ type AWSMachinePoolStatus struct {
 	// +optional
 	LaunchTemplateVersion *string `json:"launchTemplateVersion,omitempty"`
 
+	// InfrastructureMachineKind is the kind of the infrastructure resources behind MachinePool Machines.
+	// +optional
+	InfrastructureMachineKind string `json:"infrastructureMachineKind,omitempty"`
+
 	// FailureReason will be set in the event that there is a terminal problem
 	// reconciling the Machine and will contain a succinct value suitable
 	// for machine interpretation.

exp/api/v1beta2/awsmachinepool_types.go

@@ -199,6 +199,10 @@ type AWSManagedMachinePoolStatus struct {
 	// +optional
 	LaunchTemplateVersion *string `json:"launchTemplateVersion,omitempty"`
 
+	// InfrastructureMachineKind is the kind of the infrastructure resources behind MachinePool Machines.
+	// +optional
+	InfrastructureMachineKind string `json:"infrastructureMachineKind,omitempty"`
+
 	// FailureReason will be set in the event that there is a terminal problem
 	// reconciling the MachinePool and will contain a succinct value suitable
 	// for machine interpretation.

exp/api/v1beta2/awsmanagedmachinepool_types.go

@@ -54,6 +54,11 @@ const (
 	InstanceRefreshNotReadyReason = "InstanceRefreshNotReady"
 	// InstanceRefreshFailedReason used to report when there instance refresh is not initiated.
 	InstanceRefreshFailedReason = "InstanceRefreshFailed"
+
+	// AWSMachineCreationFailed reports if creating AWSMachines to represent ASG (machine pool) machines failed.
+	AWSMachineCreationFailed = "AWSMachineCreationFailed"
+	// AWSMachineDeletionFailed reports if deleting AWSMachines failed.
+	AWSMachineDeletionFailed = "AWSMachineDeletionFailed"
 )
 
 const (

exp/api/v1beta2/conditions_consts.go

@@ -22,6 +22,11 @@ import (
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
 )
 
+const (
+	// KindMachinePool is a MachinePool resource Kind
+	KindMachinePool string = "MachinePool"
+)
+
 // EBS can be used to automatically set up EBS volumes when an instance is launched.
 type EBS struct {
 	// Encrypted is whether the volume should be encrypted or not.