Skip to content

Commit 29a6e9a

Browse files
chrischdichrischdi
authored
🌱 test: improve autoscale tests for to/from zero and running autoscaler in bootstrap cluster (#11082)
* test: allow deploying autoscaler to management cluster * test: make machine pools optional in autoscaler test * test: implement optional scale from/to zero tests for autoscale * test: allow modification of apigroup for infrastructure * test: wait for rollouts to finish in autoscaler tests * test: drop cleaning up autoscaler for machine pools * review fix * add comment about AutoScaleFromZero * remove autoscale from zero test for unsupported MachinePools * review fixes
1 parent 88c757f commit 29a6e9a

File tree

5 files changed

+193
-83
lines changed

5 files changed

+193
-83
lines changed

‎test/e2e/autoscaler.go

+136-70
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,17 @@ type AutoscalerSpecInput struct {
6262
InfrastructureMachineTemplateKind string
6363
InfrastructureMachinePoolTemplateKind string
6464
InfrastructureMachinePoolKind string
65+
InfrastructureAPIGroup string
6566
AutoscalerVersion string
6667

68+
// InstallOnManagementCluster steers if the autoscaler should get installed to the management or workload cluster.
69+
// Depending on the CI environments, there may be no connectivity from the workload to the management cluster.
70+
InstallOnManagementCluster bool
71+
72+
// ScaleToAndFromZero enables tests to scale to and from zero.
73+
// Note: This is only implemented for MachineDeployments.
74+
ScaleToAndFromZero bool
75+
6776
// Allows to inject a function to be run after test namespace is created.
6877
// If not specified, this is a no-op.
6978
PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)
@@ -115,6 +124,8 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
115124
infrastructureProvider = *input.InfrastructureProvider
116125
}
117126

127+
hasMachinePool := input.InfrastructureMachinePoolTemplateKind != ""
128+
118129
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
119130
ClusterProxy: input.BootstrapClusterProxy,
120131
ConfigCluster: clusterctl.ConfigClusterInput{
@@ -137,6 +148,7 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
137148
}, clusterResources)
138149

139150
Expect(clusterResources.Cluster.Spec.Topology).NotTo(BeNil(), "Autoscaler test expected a Classy Cluster")
151+
140152
// Ensure the MachineDeploymentTopology has the autoscaler annotations.
141153
mdTopology := clusterResources.Cluster.Spec.Topology.Workers.MachineDeployments[0]
142154
Expect(mdTopology.Metadata.Annotations).NotTo(BeNil(), "MachineDeployment is expected to have autoscaler annotations")
@@ -145,21 +157,27 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
145157
mdNodeGroupMaxSize, ok := mdTopology.Metadata.Annotations[clusterv1.AutoscalerMaxSizeAnnotation]
146158
Expect(ok).To(BeTrue(), "MachineDeploymentTopology %s does not have the %q autoscaler annotation", mdTopology.Name, clusterv1.AutoscalerMaxSizeAnnotation)
147159

148-
// Ensure the MachinePoolTopology does NOT have the autoscaler annotations so we can test MachineDeployments first.
149-
mpTopology := clusterResources.Cluster.Spec.Topology.Workers.MachinePools[0]
150-
if mpTopology.Metadata.Annotations != nil {
151-
_, ok = mpTopology.Metadata.Annotations[clusterv1.AutoscalerMinSizeAnnotation]
152-
Expect(ok).To(BeFalse(), "MachinePoolTopology %s does have the %q autoscaler annotation", mpTopology.Name, clusterv1.AutoscalerMinSizeAnnotation)
153-
_, ok = mpTopology.Metadata.Annotations[clusterv1.AutoscalerMaxSizeAnnotation]
154-
Expect(ok).To(BeFalse(), "MachinePoolTopology %s does have the %q autoscaler annotation", mpTopology.Name, clusterv1.AutoscalerMaxSizeAnnotation)
160+
if hasMachinePool {
161+
// Ensure the MachinePoolTopology does NOT have the autoscaler annotations so we can test MachineDeployments first.
162+
mpTopology := clusterResources.Cluster.Spec.Topology.Workers.MachinePools[0]
163+
if mpTopology.Metadata.Annotations != nil {
164+
_, ok = mpTopology.Metadata.Annotations[clusterv1.AutoscalerMinSizeAnnotation]
165+
Expect(ok).To(BeFalse(), "MachinePoolTopology %s does have the %q autoscaler annotation", mpTopology.Name, clusterv1.AutoscalerMinSizeAnnotation)
166+
_, ok = mpTopology.Metadata.Annotations[clusterv1.AutoscalerMaxSizeAnnotation]
167+
Expect(ok).To(BeFalse(), "MachinePoolTopology %s does have the %q autoscaler annotation", mpTopology.Name, clusterv1.AutoscalerMaxSizeAnnotation)
168+
}
155169
}
156170

157171
// Get a ClusterProxy so we can interact with the workload cluster
158172
workloadClusterProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, clusterResources.Cluster.Namespace, clusterResources.Cluster.Name)
159173
mdOriginalReplicas := *clusterResources.MachineDeployments[0].Spec.Replicas
160174
Expect(strconv.Itoa(int(mdOriginalReplicas))).To(Equal(mdNodeGroupMinSize), "MachineDeployment should have replicas as defined in %s", clusterv1.AutoscalerMinSizeAnnotation)
161-
mpOriginalReplicas := *clusterResources.MachinePools[0].Spec.Replicas
162-
Expect(int(mpOriginalReplicas)).To(Equal(1), "MachinePool should default to 1 replica via the MachinePool webhook")
175+
176+
var mpOriginalReplicas int32
177+
if hasMachinePool {
178+
mpOriginalReplicas = *clusterResources.MachinePools[0].Spec.Replicas
179+
Expect(int(mpOriginalReplicas)).To(Equal(1), "MachinePool should default to 1 replica via the MachinePool webhook")
180+
}
163181

164182
By("Installing the autoscaler on the workload cluster")
165183
autoscalerWorkloadYAMLPath := input.E2EConfig.GetVariable(AutoscalerWorkloadYAMLPath)
@@ -168,11 +186,13 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
168186
InfrastructureMachineTemplateKind: input.InfrastructureMachineTemplateKind,
169187
InfrastructureMachinePoolTemplateKind: input.InfrastructureMachinePoolTemplateKind,
170188
InfrastructureMachinePoolKind: input.InfrastructureMachinePoolKind,
189+
InfrastructureAPIGroup: input.InfrastructureAPIGroup,
171190
WorkloadYamlPath: autoscalerWorkloadYAMLPath,
172191
ManagementClusterProxy: input.BootstrapClusterProxy,
173192
WorkloadClusterProxy: workloadClusterProxy,
174193
Cluster: clusterResources.Cluster,
175194
AutoscalerVersion: input.AutoscalerVersion,
195+
AutoscalerOnManagementCluster: input.InstallOnManagementCluster,
176196
}, input.E2EConfig.GetIntervals(specName, "wait-controllers")...)
177197

178198
By("Creating workload that forces the system to scale up")
@@ -226,6 +246,50 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
226246
WaitForMachineDeployment: input.E2EConfig.GetIntervals(specName, "wait-controllers"),
227247
})
228248

249+
if input.ScaleToAndFromZero {
250+
By("Enabling autoscaler for the MachineDeployment to zero")
251+
// Enable autoscaler on the MachineDeployment.
252+
framework.EnableAutoscalerForMachineDeploymentTopologyAndWait(ctx, framework.EnableAutoscalerForMachineDeploymentTopologyAndWaitInput{
253+
ClusterProxy: input.BootstrapClusterProxy,
254+
Cluster: clusterResources.Cluster,
255+
NodeGroupMinSize: "0",
256+
NodeGroupMaxSize: mdNodeGroupMaxSize,
257+
WaitForAnnotationsToBeAdded: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
258+
})
259+
260+
By("Scaling the MachineDeployment scale up deployment to zero")
261+
framework.ScaleScaleUpDeploymentAndWait(ctx, framework.ScaleScaleUpDeploymentAndWaitInput{
262+
ClusterProxy: workloadClusterProxy,
263+
// We need to sum up the expected number of MachineDeployment replicas and the current
264+
// number of MachinePool replicas because otherwise the pods get scheduled on the MachinePool nodes.
265+
Replicas: mpOriginalReplicas + 0,
266+
}, input.E2EConfig.GetIntervals(specName, "wait-autoscaler")...)
267+
268+
By("Checking the MachineDeployment finished scaling down to zero")
269+
framework.AssertMachineDeploymentReplicas(ctx, framework.AssertMachineDeploymentReplicasInput{
270+
Getter: input.BootstrapClusterProxy.GetClient(),
271+
MachineDeployment: clusterResources.MachineDeployments[0],
272+
Replicas: 0,
273+
WaitForMachineDeployment: input.E2EConfig.GetIntervals(specName, "wait-controllers"),
274+
})
275+
276+
By("Scaling the MachineDeployment scale up deployment to 1")
277+
framework.ScaleScaleUpDeploymentAndWait(ctx, framework.ScaleScaleUpDeploymentAndWaitInput{
278+
ClusterProxy: workloadClusterProxy,
279+
// We need to sum up the expected number of MachineDeployment replicas and the current
280+
// number of MachinePool replicas because otherwise the pods get scheduled on the MachinePool nodes.
281+
Replicas: mpOriginalReplicas + 1,
282+
}, input.E2EConfig.GetIntervals(specName, "wait-autoscaler")...)
283+
284+
By("Checking the MachineDeployment finished scaling up")
285+
framework.AssertMachineDeploymentReplicas(ctx, framework.AssertMachineDeploymentReplicasInput{
286+
Getter: input.BootstrapClusterProxy.GetClient(),
287+
MachineDeployment: clusterResources.MachineDeployments[0],
288+
Replicas: 1,
289+
WaitForMachineDeployment: input.E2EConfig.GetIntervals(specName, "wait-controllers"),
290+
})
291+
}
292+
229293
By("Disabling the autoscaler for MachineDeployments to test MachinePools")
230294
framework.DisableAutoscalerForMachineDeploymentTopologyAndWait(ctx, framework.DisableAutoscalerForMachineDeploymentTopologyAndWaitInput{
231295
ClusterProxy: input.BootstrapClusterProxy,
@@ -239,67 +303,69 @@ func AutoscalerSpec(ctx context.Context, inputGetter func() AutoscalerSpecInput)
239303
WaitForDelete: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
240304
})
241305

242-
By("Enabling autoscaler for the MachinePool")
243-
// Enable autoscaler on the MachinePool.
244-
framework.EnableAutoscalerForMachinePoolTopologyAndWait(ctx, framework.EnableAutoscalerForMachinePoolTopologyAndWaitInput{
245-
ClusterProxy: input.BootstrapClusterProxy,
246-
Cluster: clusterResources.Cluster,
247-
NodeGroupMinSize: mpNodeGroupMinSize,
248-
NodeGroupMaxSize: mpNodeGroupMaxSize,
249-
WaitForAnnotationsToBeAdded: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
250-
})
251-
252-
By("Creating workload that forces the system to scale up")
253-
framework.AddScaleUpDeploymentAndWait(ctx, framework.AddScaleUpDeploymentAndWaitInput{
254-
ClusterProxy: workloadClusterProxy,
255-
}, input.E2EConfig.GetIntervals(specName, "wait-autoscaler")...)
256-
257-
By("Checking the MachinePool is scaled up")
258-
mpScaledUpReplicas := mpOriginalReplicas + 1
259-
framework.AssertMachinePoolReplicas(ctx, framework.AssertMachinePoolReplicasInput{
260-
Getter: input.BootstrapClusterProxy.GetClient(),
261-
MachinePool: clusterResources.MachinePools[0],
262-
Replicas: mpScaledUpReplicas,
263-
WaitForMachinePool: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
264-
})
265-
266-
By("Disabling the autoscaler")
267-
framework.DisableAutoscalerForMachinePoolTopologyAndWait(ctx, framework.DisableAutoscalerForMachinePoolTopologyAndWaitInput{
268-
ClusterProxy: input.BootstrapClusterProxy,
269-
Cluster: clusterResources.Cluster,
270-
WaitForAnnotationsToBeDropped: input.E2EConfig.GetIntervals(specName, "wait-controllers"),
271-
})
272-
273-
By("Checking we can manually scale up the MachinePool")
274-
// Scale up the MachinePool. Since autoscaler is disabled we should be able to do this.
275-
mpExcessReplicas := mpScaledUpReplicas + 1
276-
framework.ScaleMachinePoolTopologyAndWait(ctx, framework.ScaleMachinePoolTopologyAndWaitInput{
277-
ClusterProxy: input.BootstrapClusterProxy,
278-
Cluster: clusterResources.Cluster,
279-
Replicas: mpExcessReplicas,
280-
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
281-
Getter: input.BootstrapClusterProxy.GetClient(),
282-
})
283-
284-
By("Checking enabling autoscaler will scale down the MachinePool to correct size")
285-
// Enable autoscaler on the MachinePool.
286-
framework.EnableAutoscalerForMachinePoolTopologyAndWait(ctx, framework.EnableAutoscalerForMachinePoolTopologyAndWaitInput{
287-
ClusterProxy: input.BootstrapClusterProxy,
288-
Cluster: clusterResources.Cluster,
289-
NodeGroupMinSize: mpNodeGroupMinSize,
290-
NodeGroupMaxSize: mpNodeGroupMaxSize,
291-
WaitForAnnotationsToBeAdded: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
292-
})
293-
294-
By("Checking the MachinePool is scaled down")
295-
// Since we scaled up the MachinePool manually and the workload has not changed auto scaler
296-
// should detect that there are unneeded nodes and scale down the MachinePool.
297-
framework.AssertMachinePoolReplicas(ctx, framework.AssertMachinePoolReplicasInput{
298-
Getter: input.BootstrapClusterProxy.GetClient(),
299-
MachinePool: clusterResources.MachinePools[0],
300-
Replicas: mpScaledUpReplicas,
301-
WaitForMachinePool: input.E2EConfig.GetIntervals(specName, "wait-controllers"),
302-
})
306+
if hasMachinePool {
307+
By("Enabling autoscaler for the MachinePool")
308+
// Enable autoscaler on the MachinePool.
309+
framework.EnableAutoscalerForMachinePoolTopologyAndWait(ctx, framework.EnableAutoscalerForMachinePoolTopologyAndWaitInput{
310+
ClusterProxy: input.BootstrapClusterProxy,
311+
Cluster: clusterResources.Cluster,
312+
NodeGroupMinSize: mpNodeGroupMinSize,
313+
NodeGroupMaxSize: mpNodeGroupMaxSize,
314+
WaitForAnnotationsToBeAdded: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
315+
})
316+
317+
By("Creating workload that forces the system to scale up")
318+
framework.AddScaleUpDeploymentAndWait(ctx, framework.AddScaleUpDeploymentAndWaitInput{
319+
ClusterProxy: workloadClusterProxy,
320+
}, input.E2EConfig.GetIntervals(specName, "wait-autoscaler")...)
321+
322+
By("Checking the MachinePool is scaled up")
323+
mpScaledUpReplicas := mpOriginalReplicas + 1
324+
framework.AssertMachinePoolReplicas(ctx, framework.AssertMachinePoolReplicasInput{
325+
Getter: input.BootstrapClusterProxy.GetClient(),
326+
MachinePool: clusterResources.MachinePools[0],
327+
Replicas: mpScaledUpReplicas,
328+
WaitForMachinePool: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
329+
})
330+
331+
By("Disabling the autoscaler")
332+
framework.DisableAutoscalerForMachinePoolTopologyAndWait(ctx, framework.DisableAutoscalerForMachinePoolTopologyAndWaitInput{
333+
ClusterProxy: input.BootstrapClusterProxy,
334+
Cluster: clusterResources.Cluster,
335+
WaitForAnnotationsToBeDropped: input.E2EConfig.GetIntervals(specName, "wait-controllers"),
336+
})
337+
338+
By("Checking we can manually scale up the MachinePool")
339+
// Scale up the MachinePool. Since autoscaler is disabled we should be able to do this.
340+
mpExcessReplicas := mpScaledUpReplicas + 1
341+
framework.ScaleMachinePoolTopologyAndWait(ctx, framework.ScaleMachinePoolTopologyAndWaitInput{
342+
ClusterProxy: input.BootstrapClusterProxy,
343+
Cluster: clusterResources.Cluster,
344+
Replicas: mpExcessReplicas,
345+
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
346+
Getter: input.BootstrapClusterProxy.GetClient(),
347+
})
348+
349+
By("Checking enabling autoscaler will scale down the MachinePool to correct size")
350+
// Enable autoscaler on the MachinePool.
351+
framework.EnableAutoscalerForMachinePoolTopologyAndWait(ctx, framework.EnableAutoscalerForMachinePoolTopologyAndWaitInput{
352+
ClusterProxy: input.BootstrapClusterProxy,
353+
Cluster: clusterResources.Cluster,
354+
NodeGroupMinSize: mpNodeGroupMinSize,
355+
NodeGroupMaxSize: mpNodeGroupMaxSize,
356+
WaitForAnnotationsToBeAdded: input.E2EConfig.GetIntervals(specName, "wait-autoscaler"),
357+
})
358+
359+
By("Checking the MachinePool is scaled down")
360+
// Since we scaled up the MachinePool manually and the workload has not changed auto scaler
361+
// should detect that there are unneeded nodes and scale down the MachinePool.
362+
framework.AssertMachinePoolReplicas(ctx, framework.AssertMachinePoolReplicasInput{
363+
Getter: input.BootstrapClusterProxy.GetClient(),
364+
MachinePool: clusterResources.MachinePools[0],
365+
Replicas: mpScaledUpReplicas,
366+
WaitForMachinePool: input.E2EConfig.GetIntervals(specName, "wait-controllers"),
367+
})
368+
}
303369

304370
By("PASSED!")
305371
})

0 commit comments

Comments
 (0)