add AzureEnvironment to AzureManagedControlPlaneSpec

Author: luthermonson

api/v1beta1/azurecluster_default.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"k8s.io/utils/pointer"
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
 )
 
 const (
@@ -41,8 +42,6 @@ const (
 	DefaultInternalLBIPAddress = "10.0.0.100"
 	// DefaultOutboundRuleIdleTimeoutInMinutes is the default for IdleTimeoutInMinutes for the load balancer.
 	DefaultOutboundRuleIdleTimeoutInMinutes = 4
-	// DefaultAzureCloud is the public cloud that will be used by most users.
-	DefaultAzureCloud = "AzurePublicCloud"
 )
 
 func (c *AzureCluster) setDefaults() {
@@ -69,7 +68,7 @@ func (c *AzureCluster) setResourceGroupDefault() {
 
 func (c *AzureCluster) setAzureEnvironmentDefault() {
 	if c.Spec.AzureEnvironment == "" {
-		c.Spec.AzureEnvironment = DefaultAzureCloud
+		c.Spec.AzureEnvironment = azure.PublicCloudName
 	}
 }
 

api/v1beta1/azurecluster_default_test.go

@@ -21,6 +21,8 @@ import (
 	"reflect"
 	"testing"
 
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/pointer"
 )
@@ -1037,7 +1039,7 @@ func TestAzureEnviromentDefault(t *testing.T) {
 				},
 				Spec: AzureClusterSpec{
 					AzureClusterClassSpec: AzureClusterClassSpec{
-						AzureEnvironment: DefaultAzureCloud,
+						AzureEnvironment: azure.PublicCloudName,
 					},
 				},
 			},
@@ -1049,7 +1051,7 @@ func TestAzureEnviromentDefault(t *testing.T) {
 				},
 				Spec: AzureClusterSpec{
 					AzureClusterClassSpec: AzureClusterClassSpec{
-						AzureEnvironment: DefaultAzureCloud,
+						AzureEnvironment: azure.PublicCloudName,
 					},
 				},
 			},
@@ -1059,7 +1061,7 @@ func TestAzureEnviromentDefault(t *testing.T) {
 				},
 				Spec: AzureClusterSpec{
 					AzureClusterClassSpec: AzureClusterClassSpec{
-						AzureEnvironment: DefaultAzureCloud,
+						AzureEnvironment: azure.PublicCloudName,
 					},
 				},
 			},

api/v1beta1/azuremanagedcontrolplane_default.go

@@ -22,6 +22,7 @@ import (
 
 	"golang.org/x/crypto/ssh"
 	"k8s.io/utils/pointer"
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
 	utilSSH "sigs.k8s.io/cluster-api-provider-azure/util/ssh"
 )
 
@@ -144,3 +145,9 @@ func (m *AzureManagedControlPlane) setDefaultAutoScalerProfile() {
 		m.Spec.AutoScalerProfile.SkipNodesWithSystemPods = (*SkipNodesWithSystemPods)(pointer.String(string(SkipNodesWithSystemPodsTrue)))
 	}
 }
+
+func (m *AzureManagedControlPlane) setAzureEnvironmentDefault() {
+	if m.Spec.AzureEnvironment == "" {
+		m.Spec.AzureEnvironment = azure.PublicCloudName
+	}
+}

api/v1beta1/azuremanagedcontrolplane_types.go

@@ -138,6 +138,15 @@ type AzureManagedControlPlaneSpec struct {
 	// AutoscalerProfile is the parameters to be applied to the cluster-autoscaler when enabled
 	// +optional
 	AutoScalerProfile *AutoScalerProfile `json:"autoscalerProfile,omitempty"`
+
+	// AzureEnvironment is the name of the AzureCloud to be used.
+	// The default value that would be used by most users is "AzurePublicCloud", other values are:
+	// - ChinaCloud: "AzureChinaCloud"
+	// - GermanCloud: "AzureGermanCloud"
+	// - PublicCloud: "AzurePublicCloud"
+	// - USGovernmentCloud: "AzureUSGovernmentCloud"
+	// +optional
+	AzureEnvironment string `json:"azureEnvironment,omitempty"`
 }
 
 // AADProfile - AAD integration managed by AKS.

api/v1beta1/azuremanagedcontrolplane_webhook.go

@@ -193,6 +193,18 @@ func (mw *azureManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, o
 		allErrs = append(allErrs, err)
 	}
 
+	if !reflect.DeepEqual(m.Spec.AzureEnvironment, old.Spec.AzureEnvironment) {
+		old.setAzureEnvironmentDefault()
+
+		// if it's still not equal, return error.
+		if !reflect.DeepEqual(m.Spec.AzureEnvironment, old.Spec.AzureEnvironment) {
+			allErrs = append(allErrs,
+				field.Invalid(field.NewPath("spec", "AzureEnvironment"),
+					m.Spec.AzureEnvironment, "field is immutable"),
+			)
+		}
+	}
+
 	if old.Spec.AADProfile != nil {
 		if m.Spec.AADProfile == nil {
 			allErrs = append(allErrs,

api/v1beta1/types_class.go

@@ -16,7 +16,10 @@ limitations under the License.
 
 package v1beta1
 
-import corev1 "k8s.io/api/core/v1"
+import (
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
+)
 
 // AzureClusterClassSpec defines the AzureCluster properties that may be shared across several Azure clusters.
 type AzureClusterClassSpec struct {
@@ -134,7 +137,7 @@ type FrontendIPClass struct {
 // setDefaults sets default values for AzureClusterClassSpec.
 func (acc *AzureClusterClassSpec) setDefaults() {
 	if acc.AzureEnvironment == "" {
-		acc.AzureEnvironment = DefaultAzureCloud
+		acc.AzureEnvironment = azure.PublicCloudName
 	}
 }
 

azure/scope/managedcontrolplane.go

@@ -73,7 +73,7 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane
 	}
 
 	if params.ControlPlane.Spec.IdentityRef == nil {
-		if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, ""); err != nil {
+		if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.AzureEnvironment); err != nil {
 			return nil, errors.Wrap(err, "failed to create Azure session")
 		}
 	} else {
@@ -82,7 +82,7 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane
 			return nil, errors.Wrap(err, "failed to init credentials provider")
 		}
 
-		if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, "", credentialsProvider); err != nil {
+		if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.AzureEnvironment, credentialsProvider); err != nil {
 			return nil, errors.Wrap(err, "failed to configure azure settings and credentials for Identity")
 		}
 	}

config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml

@@ -217,6 +217,13 @@ spec:
                     - "false"
                     type: string
                 type: object
+              azureEnvironment:
+                description: 'AzureEnvironment is the name of the AzureCloud to be
+                  used. The default value that would be used by most users is "AzurePublicCloud",
+                  other values are: - ChinaCloud: "AzureChinaCloud" - GermanCloud:
+                  "AzureGermanCloud" - PublicCloud: "AzurePublicCloud" - USGovernmentCloud:
+                  "AzureUSGovernmentCloud"'
+                type: string
               controlPlaneEndpoint:
                 description: ControlPlaneEndpoint represents the endpoint used to
                   communicate with the control plane.

controllers/azuremanagedmachinepool_reconciler.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute"
+	azureautorest "github.com/Azure/go-autorest/autorest/azure"
 	"github.com/pkg/errors"
 	azprovider "sigs.k8s.io/cloud-provider-azure/pkg/provider"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
@@ -74,22 +75,32 @@ func (a *AgentPoolVMSSNotFoundError) Is(target error) bool {
 
 // newAzureManagedMachinePoolService populates all the services based on input scope.
 func newAzureManagedMachinePoolService(scope *scope.ManagedMachinePoolScope) (*azureManagedMachinePoolService, error) {
-	var authorizer azure.Authorizer = scope
-	if scope.Location() != "" {
-		regionalAuthorizer, err := azure.WithRegionalBaseURI(scope, scope.Location())
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create a regional authorizer")
-		}
-		authorizer = regionalAuthorizer
+	scaleSetAuthorizer, err := scaleSetAuthorizer(scope)
+	if err != nil {
+		return nil, err
 	}
 
 	return &azureManagedMachinePoolService{
 		scope:         scope,
 		agentPoolsSvc: agentpools.New(scope),
-		scaleSetsSvc:  scalesets.NewClient(authorizer),
+		scaleSetsSvc:  scalesets.NewClient(scaleSetAuthorizer),
 	}, nil
 }
 
+// scaleSetAuthorizer takes a scope and determines if a regional authorizer is needed for scale sets
+// see https://github.com/kubernetes-sigs/cluster-api-provider-azure/pull/1850 for context on region based authorizer.
+func scaleSetAuthorizer(scope *scope.ManagedMachinePoolScope) (azure.Authorizer, error) {
+	if scope.Location() == "" {
+		return scope, nil // no location so use default
+	}
+
+	if scope.ControlPlane.Spec.AzureEnvironment == azureautorest.USGovernmentCloud.Name {
+		return scope, nil // no region support in usgovcloud
+	}
+
+	return azure.WithRegionalBaseURI(scope, scope.Location())
+}
+
 // Reconcile reconciles all the services in a predetermined order.
 func (s *azureManagedMachinePoolService) Reconcile(ctx context.Context) error {
 	ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.azureManagedMachinePoolService.Reconcile")