Skip to content

Commit 8798c7f

Browse files
miyadavmiyadav
committed
tests for disks,shieldedVMs,Confidential VMs
1 parent af0b63d commit 8798c7f

File tree

2 files changed

+301
-1
lines changed

2 files changed

+301
-1
lines changed

‎pkg/capi/gcp.go

+295
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,295 @@
1+
package capi
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
configv1 "github.com/openshift/api/config/v1"
10+
mapiv1 "github.com/openshift/api/machine/v1beta1"
11+
framework "github.com/openshift/cluster-api-actuator-pkg/pkg/framework"
12+
corev1 "k8s.io/api/core/v1"
13+
apierrors "k8s.io/apimachinery/pkg/api/errors"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
gcpv1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1"
16+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
17+
"sigs.k8s.io/controller-runtime/pkg/client"
18+
"sigs.k8s.io/controller-runtime/pkg/envtest/komega"
19+
yaml "sigs.k8s.io/yaml"
20+
)
21+
22+
const (
23+
infraAPIVersion = "infrastructure.cluster.x-k8s.io/v1beta1"
24+
gcpMachineTemplateName = "gcp-machine-template"
25+
)
26+
27+
var (
28+
ctx = context.Background()
29+
clusterName string
30+
cl client.Client
31+
)
32+
33+
var _ = Describe("Cluster API GCP MachineSet", Ordered, func() {
34+
var gcpMachineTemplate *gcpv1.GCPMachineTemplate
35+
var machineSet *clusterv1.MachineSet
36+
var mapiMachineSpec *mapiv1.GCPMachineProviderSpec
37+
var ctx context.Context
38+
var platform configv1.PlatformType
39+
var clusterName string
40+
var err error
41+
42+
BeforeAll(func() {
43+
cl, err = framework.LoadClient()
44+
Expect(err).NotTo(HaveOccurred(), "Failed to create Kubernetes client for test")
45+
komega.SetClient(cl)
46+
ctx = framework.GetContext()
47+
platform, err = framework.GetPlatform(ctx, cl)
48+
Expect(err).ToNot(HaveOccurred(), "Failed to get platform")
49+
if platform != configv1.GCPPlatformType {
50+
Skip("Skipping GCP E2E tests")
51+
}
52+
oc, _ := framework.NewCLI()
53+
framework.SkipIfNotTechPreviewNoUpgrade(oc, cl)
54+
55+
infra, err := framework.GetInfrastructure(ctx, cl)
56+
Expect(err).NotTo(HaveOccurred(), "Failed to get cluster infrastructure object")
57+
Expect(infra.Status.InfrastructureName).ShouldNot(BeEmpty(), "infrastructure name was empty on Infrastructure.Status.")
58+
clusterName = infra.Status.InfrastructureName
59+
60+
framework.CreateCoreCluster(ctx, cl, clusterName, "GCPCluster")
61+
mapiMachineSpec = getGCPMAPIProviderSpec(cl)
62+
})
63+
AfterEach(func() {
64+
if platform != configv1.GCPPlatformType {
65+
// Because AfterEach always runs, even when tests are skipped, we have to
66+
// explicitly skip it here for other platforms.
67+
Skip("Skipping GCP E2E tests")
68+
}
69+
framework.DeleteCAPIMachineSets(ctx, cl, machineSet)
70+
framework.WaitForCAPIMachineSetsDeleted(ctx, cl, machineSet)
71+
framework.DeleteObjects(ctx, cl, gcpMachineTemplate)
72+
})
73+
74+
diskTypes := map[string]gcpv1.DiskType{
75+
"pd-standard": gcpv1.PdStandardDiskType,
76+
"pd-ssd": gcpv1.PdSsdDiskType,
77+
// "local-ssd": gcpv1.LocalSsdDiskType,//boot disk cannot be local disk hence commenting it
78+
//"pd-balanced": gcpv1.PdBalancedDiskType,//This need a change to vendor file - vendor/sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1/gcpmachine_types.go
79+
}
80+
81+
for diskTypeName := range diskTypes {
82+
// OCP-77825 - instances support disks pd-standard and pd-ssd
83+
// author: [email protected]
84+
It(fmt.Sprintf("should be able to run a machine with disk type: %s", diskTypeName), func() {
85+
mapiProviderSpec := getGCPMAPIProviderSpec(cl)
86+
Expect(mapiProviderSpec).ToNot(BeNil())
87+
mapiProviderSpec.OnHostMaintenance = "Migrate"
88+
mapiProviderSpec.ConfidentialCompute = "Disabled"
89+
mapiProviderSpec.Disks[0].Type = diskTypeName
90+
gcpMachineTemplate = createGCPMachineTemplate(cl, mapiProviderSpec)
91+
machineSet, _ = framework.CreateCAPIMachineSet(ctx, cl, framework.NewCAPIMachineSetParams(
92+
"gcp-machineset",
93+
clusterName,
94+
mapiMachineSpec.Zone,
95+
1,
96+
corev1.ObjectReference{
97+
Kind: "GCPMachineTemplate",
98+
APIVersion: infraAPIVersion,
99+
Name: gcpMachineTemplateName,
100+
},
101+
))
102+
Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset")
103+
framework.WaitForCAPIMachinesRunning(framework.GetContext(), cl, machineSet.Name)
104+
})
105+
}
106+
107+
// OCP-74795 - add support for shielded VMs - It takes defaults if configs are not supported; eg-vtpm alone set to Enabled will result in IntergrityMonitoring also as Enabled
108+
// doesn't matter what we pass - all enabled/secureboot only disabled(default)/secureboot only enabled are valid changes which we can apply
109+
// author: [email protected]
110+
DescribeTable("should configure Shielded VM options correctly",
111+
func(enableSecureBoot mapiv1.SecureBootPolicy, enableVtpm mapiv1.VirtualizedTrustedPlatformModulePolicy, enableIntegrityMonitoring mapiv1.IntegrityMonitoringPolicy) {
112+
mapiProviderSpec := getGCPMAPIProviderSpec(cl)
113+
Expect(mapiProviderSpec).ToNot(BeNil())
114+
mapiProviderSpec.OnHostMaintenance = "Migrate"
115+
mapiProviderSpec.ConfidentialCompute = "Disabled"
116+
// Setting Shielded VM configuration
117+
mapiProviderSpec.ShieldedInstanceConfig = mapiv1.GCPShieldedInstanceConfig{
118+
SecureBoot: enableSecureBoot,
119+
VirtualizedTrustedPlatformModule: enableVtpm,
120+
IntegrityMonitoring: enableIntegrityMonitoring,
121+
}
122+
gcpMachineTemplate = createGCPMachineTemplate(cl, mapiProviderSpec)
123+
machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, framework.NewCAPIMachineSetParams(
124+
"gcp-machineset-shieldedvm",
125+
clusterName,
126+
mapiMachineSpec.Zone,
127+
1,
128+
corev1.ObjectReference{
129+
Kind: "GCPMachineTemplate",
130+
APIVersion: infraAPIVersion,
131+
Name: gcpMachineTemplateName,
132+
},
133+
))
134+
Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset with Shielded VM config")
135+
136+
framework.WaitForCAPIMachinesRunning(framework.GetContext(), cl, machineSet.Name)
137+
138+
By("Verifying the Shielded VM configuration on the created GCP MachineTemplate")
139+
createdTemplate := &gcpv1.GCPMachineTemplate{}
140+
Expect(cl.Get(ctx, client.ObjectKey{
141+
Namespace: framework.ClusterAPINamespace,
142+
Name: gcpMachineTemplateName,
143+
}, createdTemplate)).To(Succeed())
144+
Expect(createdTemplate.Spec.Template.Spec.ShieldedInstanceConfig).ToNot(BeNil())
145+
Expect(fmt.Sprintf("%v", createdTemplate.Spec.Template.Spec.ShieldedInstanceConfig.SecureBoot)).To(Equal(fmt.Sprintf("%v", enableSecureBoot)))
146+
Expect(fmt.Sprintf("%v", createdTemplate.Spec.Template.Spec.ShieldedInstanceConfig.VirtualizedTrustedPlatformModule)).To(Equal(fmt.Sprintf("%v", enableVtpm)))
147+
Expect(fmt.Sprintf("%v", createdTemplate.Spec.Template.Spec.ShieldedInstanceConfig.IntegrityMonitoring)).To(Equal(fmt.Sprintf("%v", enableIntegrityMonitoring)))
148+
},
149+
Entry("all Shielded VM options enabled", mapiv1.SecureBootPolicyEnabled, mapiv1.VirtualizedTrustedPlatformModulePolicyEnabled, mapiv1.IntegrityMonitoringPolicyEnabled),
150+
Entry("only SecureBoot enabled", mapiv1.SecureBootPolicyEnabled, mapiv1.VirtualizedTrustedPlatformModulePolicyDisabled, mapiv1.IntegrityMonitoringPolicyDisabled),
151+
/*Below configs doesn't make difference due to defaulting conditions of shielded VMs
152+
Entry("only Vtpm enabled", mapiv1.SecureBootPolicyDisabled, mapiv1.VirtualizedTrustedPlatformModulePolicyEnabled, mapiv1.IntegrityMonitoringPolicyDisabled),
153+
Entry("only IntegrityMonitoring enabled", mapiv1.SecureBootPolicyDisabled, mapiv1.VirtualizedTrustedPlatformModulePolicyDisabled, mapiv1.IntegrityMonitoringPolicyEnabled),
154+
Entry("SecureBoot and Vtpm enabled", mapiv1.SecureBootPolicyEnabled, mapiv1.VirtualizedTrustedPlatformModulePolicyEnabled, mapiv1.IntegrityMonitoringPolicyDisabled),
155+
Entry("SecureBoot and IntegrityMonitoring enabled", mapiv1.SecureBootPolicyEnabled, mapiv1.VirtualizedTrustedPlatformModulePolicyDisabled, mapiv1.IntegrityMonitoringPolicyEnabled),
156+
Entry("all Shielded VM options disabled", mapiv1.SecureBootPolicyDisabled, mapiv1.VirtualizedTrustedPlatformModulePolicyDisabled, mapiv1.IntegrityMonitoringPolicyDisabled),
157+
*/
158+
)
159+
// OCP-74703 - Create confidential compute VMs on gcp
160+
// author: [email protected]
161+
DescribeTable("should configure Confidential VM correctly",
162+
func(confidentialCompute mapiv1.ConfidentialComputePolicy) {
163+
mapiProviderSpec := getGCPMAPIProviderSpec(cl)
164+
Expect(mapiProviderSpec).ToNot(BeNil())
165+
166+
// Set the Confidential Compute configuration
167+
mapiProviderSpec.ConfidentialCompute = confidentialCompute
168+
if confidentialCompute == mapiv1.ConfidentialComputePolicyEnabled {
169+
mapiProviderSpec.OnHostMaintenance = mapiv1.GCPHostMaintenanceType(gcpv1.HostMaintenancePolicyTerminate)
170+
mapiProviderSpec.MachineType = "n2d-standard-4"
171+
}
172+
173+
By("Creating a GCP MachineTemplate for Confidential VM")
174+
gcpMachineTemplate = createGCPMachineTemplate(cl, mapiProviderSpec)
175+
176+
By("Creating a MachineSet for Confidential VM")
177+
machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, framework.NewCAPIMachineSetParams(
178+
"gcp-machineset-confidential",
179+
clusterName,
180+
mapiProviderSpec.Zone,
181+
1,
182+
corev1.ObjectReference{
183+
Kind: "GCPMachineTemplate",
184+
APIVersion: infraAPIVersion,
185+
Name: gcpMachineTemplateName,
186+
},
187+
))
188+
Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI MachineSet with Confidential VM configuration")
189+
190+
framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name)
191+
192+
By("Verifying the Confidential VM configuration on the created GCP MachineTemplate")
193+
createdTemplate := &gcpv1.GCPMachineTemplate{}
194+
Expect(cl.Get(framework.GetContext(), client.ObjectKey{
195+
Namespace: framework.ClusterAPINamespace,
196+
Name: gcpMachineTemplateName,
197+
}, createdTemplate)).To(Succeed())
198+
Expect(fmt.Sprintf("%v", createdTemplate.Spec.Template.Spec.ConfidentialCompute)).To(Equal(fmt.Sprintf("%v", confidentialCompute)))
199+
},
200+
Entry("Confidential Compute enabled", mapiv1.ConfidentialComputePolicyEnabled),
201+
Entry("Confidential Compute disabled", mapiv1.ConfidentialComputePolicyDisabled),
202+
)
203+
})
204+
205+
func getGCPMAPIProviderSpec(cl client.Client) *mapiv1.GCPMachineProviderSpec {
206+
machineSetList := &mapiv1.MachineSetList{}
207+
Expect(cl.List(ctx, machineSetList, client.InNamespace(framework.MachineAPINamespace))).To(Succeed())
208+
209+
Expect(machineSetList.Items).ToNot(HaveLen(0))
210+
machineSet := machineSetList.Items[0]
211+
Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil())
212+
213+
providerSpec := &mapiv1.GCPMachineProviderSpec{}
214+
Expect(yaml.Unmarshal(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec)).To(Succeed())
215+
216+
return providerSpec
217+
}
218+
219+
func createGCPMachineTemplate(cl client.Client, mapiProviderSpec *mapiv1.GCPMachineProviderSpec) *gcpv1.GCPMachineTemplate {
220+
By("Creating GCP machine template")
221+
222+
Expect(mapiProviderSpec).ToNot(BeNil())
223+
Expect(mapiProviderSpec.Disks).ToNot(BeNil())
224+
Expect(len(mapiProviderSpec.Disks)).To(BeNumerically(">", 0))
225+
Expect(mapiProviderSpec.Disks[0].Type).ToNot(BeEmpty())
226+
Expect(mapiProviderSpec.MachineType).ToNot(BeEmpty())
227+
Expect(mapiProviderSpec.NetworkInterfaces).ToNot(BeNil())
228+
Expect(len(mapiProviderSpec.NetworkInterfaces)).To(BeNumerically(">", 0))
229+
Expect(mapiProviderSpec.NetworkInterfaces[0].Subnetwork).ToNot(BeEmpty())
230+
Expect(mapiProviderSpec.ServiceAccounts).ToNot(BeNil())
231+
Expect(mapiProviderSpec.ServiceAccounts[0].Email).ToNot(BeEmpty())
232+
Expect(mapiProviderSpec.ServiceAccounts[0].Scopes).ToNot(BeNil())
233+
Expect(len(mapiProviderSpec.ServiceAccounts)).To(BeNumerically(">", 0))
234+
Expect(mapiProviderSpec.Tags).ToNot(BeNil())
235+
Expect(len(mapiProviderSpec.Tags)).To(BeNumerically(">", 0))
236+
237+
var rootDeviceType gcpv1.DiskType
238+
239+
switch mapiProviderSpec.Disks[0].Type {
240+
case "pd-standard":
241+
rootDeviceType = gcpv1.PdStandardDiskType
242+
case "pd-ssd":
243+
rootDeviceType = gcpv1.PdSsdDiskType
244+
}
245+
/*case "local-ssd":
246+
rootDeviceType = gcpv1.LocalSsdDiskType*/
247+
/*case "pd-balanced":
248+
rootDeviceType = gcpv1.PdBalancedDiskType*/
249+
//Above cases are not include Pdbalanced needs vendor file update, local-ssd not supported for boot disks
250+
251+
ipForwardingDisabled := gcpv1.IPForwardingDisabled
252+
253+
gcpMachineSpec := gcpv1.GCPMachineSpec{
254+
RootDeviceType: &rootDeviceType,
255+
RootDeviceSize: mapiProviderSpec.Disks[0].SizeGB,
256+
InstanceType: mapiProviderSpec.MachineType,
257+
OnHostMaintenance: (*gcpv1.HostMaintenancePolicy)(&mapiProviderSpec.OnHostMaintenance),
258+
Image: &mapiProviderSpec.Disks[0].Image,
259+
Subnet: &mapiProviderSpec.NetworkInterfaces[0].Subnetwork,
260+
ServiceAccount: &gcpv1.ServiceAccount{
261+
Email: mapiProviderSpec.ServiceAccounts[0].Email,
262+
Scopes: mapiProviderSpec.ServiceAccounts[0].Scopes,
263+
},
264+
// Shielded VM Configuration
265+
ShieldedInstanceConfig: &gcpv1.GCPShieldedInstanceConfig{
266+
SecureBoot: gcpv1.SecureBootPolicy(mapiProviderSpec.ShieldedInstanceConfig.SecureBoot),
267+
VirtualizedTrustedPlatformModule: gcpv1.VirtualizedTrustedPlatformModulePolicy(mapiProviderSpec.ShieldedInstanceConfig.VirtualizedTrustedPlatformModule),
268+
IntegrityMonitoring: gcpv1.IntegrityMonitoringPolicy(mapiProviderSpec.ShieldedInstanceConfig.IntegrityMonitoring),
269+
},
270+
271+
// Confidential VM Configuration
272+
ConfidentialCompute: (*gcpv1.ConfidentialComputePolicy)(&mapiProviderSpec.ConfidentialCompute),
273+
AdditionalNetworkTags: mapiProviderSpec.Tags,
274+
AdditionalLabels: gcpv1.Labels{fmt.Sprintf("kubernetes-io-cluster-%s", clusterName): "owned"},
275+
IPForwarding: &ipForwardingDisabled,
276+
}
277+
278+
gcpMachineTemplate := &gcpv1.GCPMachineTemplate{
279+
ObjectMeta: metav1.ObjectMeta{
280+
Name: gcpMachineTemplateName,
281+
Namespace: framework.ClusterAPINamespace,
282+
},
283+
Spec: gcpv1.GCPMachineTemplateSpec{
284+
Template: gcpv1.GCPMachineTemplateResource{
285+
Spec: gcpMachineSpec,
286+
},
287+
},
288+
}
289+
290+
if err := cl.Create(ctx, gcpMachineTemplate); err != nil && !apierrors.IsAlreadyExists(err) {
291+
Expect(err).ToNot(HaveOccurred())
292+
}
293+
294+
return gcpMachineTemplate
295+
}

‎pkg/e2e_test.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/openshift/cluster-api-actuator-pkg/pkg/framework"
1616
caov1alpha1 "github.com/openshift/cluster-autoscaler-operator/pkg/apis"
1717
azurev1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
18+
gcpv1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1"
1819
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1920

2021
_ "github.com/openshift/cluster-api-actuator-pkg/pkg/autoscaler"
@@ -48,6 +49,10 @@ func init() {
4849
if err := azurev1.AddToScheme(scheme.Scheme); err != nil {
4950
klog.Fatal(err)
5051
}
52+
53+
if err := gcpv1.AddToScheme(scheme.Scheme); err != nil {
54+
klog.Fatal(err)
55+
}
5156
}
5257

5358
func TestE2E(t *testing.T) {
@@ -66,7 +71,7 @@ var _ = BeforeSuite(func() {
6671

6772
// Extend timeouts for slower providers
6873
switch platform {
69-
case osconfigv1.AzurePlatformType, osconfigv1.VSpherePlatformType, osconfigv1.OpenStackPlatformType, osconfigv1.PowerVSPlatformType, osconfigv1.NutanixPlatformType:
74+
case osconfigv1.AzurePlatformType, osconfigv1.GCPPlatformType, osconfigv1.VSpherePlatformType, osconfigv1.OpenStackPlatformType, osconfigv1.PowerVSPlatformType, osconfigv1.NutanixPlatformType:
7075
framework.WaitShort = 2 * time.Minute // Normally 1m
7176
framework.WaitMedium = 6 * time.Minute // Normally 3m
7277
framework.WaitLong = 30 * time.Minute // Normally 15m

0 commit comments

Comments
 (0)