Skip to content

Add support for running tests on confidential VMs that use NVMe #1636

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 2, 2024
Merged

Add support for running tests on confidential VMs that use NVMe #1636

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 28 additions & 22 deletions test/e2e/tests/setup_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,19 @@ import (
)

var (
project = flag.String("project", "", "Project to run tests in")
serviceAccount = flag.String("service-account", "", "Service account to bring up instance with")
architecture = flag.String("arch", "amd64", "Architecture pd csi driver build on")
zones = flag.String("zones", "us-east4-a,us-east4-c", "Zones to run tests in. If there are multiple zones, separate each by comma")
machineType = flag.String("machine-type", "n2-standard-2", "Type of machine to provision instance on")
imageURL = flag.String("image-url", "projects/debian-cloud/global/images/family/debian-11", "OS image url to get image from")
runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys")
deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run")
cloudtopHost = flag.Bool("cloudtop-host", false, "The local host is cloudtop, a kind of googler machine with special requirements to access GCP")
extraDriverFlags = flag.String("extra-driver-flags", "", "Extra flags to pass to the driver")
project = flag.String("project", "", "Project to run tests in")
serviceAccount = flag.String("service-account", "", "Service account to bring up instance with")
vmNamePrefix = flag.String("vm-name-prefix", "gce-pd-csi-e2e", "VM name prefix")
architecture = flag.String("arch", "amd64", "Architecture pd csi driver build on")
minCpuPlatform = flag.String("min-cpu-platform", "AMD Milan", "Minimum CPU architecture")
zones = flag.String("zones", "us-east4-a,us-east4-c", "Zones to run tests in. If there are multiple zones, separate each by comma")
machineType = flag.String("machine-type", "n2d-standard-2", "Type of machine to provision instance on")
imageURL = flag.String("image-url", "projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2310-amd64", "OS image url to get image from")
runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys")
deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run")
cloudtopHost = flag.Bool("cloudtop-host", false, "The local host is cloudtop, a kind of googler machine with special requirements to access GCP")
extraDriverFlags = flag.String("extra-driver-flags", "", "Extra flags to pass to the driver")
enableConfidentialCompute = flag.Bool("enable-confidential-compute", false, "Create VMs with confidential compute mode. This uses NVMe devices")

testContexts = []*remote.TestContext{}
computeService *compute.Service
Expand Down Expand Up @@ -130,21 +133,24 @@ func getDriverConfig() testutils.DriverConfig {
}
}

func getRemoteInstanceConfig() *remote.InstanceConfig {
return &remote.InstanceConfig{
Project: *project,
Architecture: *architecture,
MachineType: *machineType,
ServiceAccount: *serviceAccount,
ImageURL: *imageURL,
CloudtopHost: *cloudtopHost}
}

func NewTestContext(zone string) *remote.TestContext {
nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone)
nodeID := fmt.Sprintf("%s-%s", *vmNamePrefix, zone)
klog.Infof("Setting up node %s", nodeID)

i, err := remote.SetupInstance(getRemoteInstanceConfig(), zone, nodeID, computeService)
instanceConfig := remote.InstanceConfig{
Project: *project,
Architecture: *architecture,
MinCpuPlatform: *minCpuPlatform,
Zone: zone,
Name: nodeID,
MachineType: *machineType,
ServiceAccount: *serviceAccount,
ImageURL: *imageURL,
CloudtopHost: *cloudtopHost,
EnableConfidentialCompute: *enableConfidentialCompute,
ComputeService: computeService,
}
i, err := remote.SetupInstance(instanceConfig)
if err != nil {
klog.Fatalf("Failed to setup instance %v: %v", nodeID, err)
}
Expand Down
20 changes: 6 additions & 14 deletions test/e2e/tests/single_zone_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
Expect(volumeLimit).To(Equal(defaultVolumeLimit))
})

It("Should create->attach->stage->mount volume and check if it is writable, then unmount->unstage->detach->delete and check disk is deleted", func() {
It("[NVMe] Should create->attach->stage->mount volume and check if it is writable, then unmount->unstage->detach->delete and check disk is deleted", func() {
testContext := getRandomTestContext()

p, z, _ := testContext.Instance.GetIdentity()
Expand Down Expand Up @@ -173,7 +173,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
}()
})

It("Should automatically add a symlink between /dev/* and /dev/by-id if disk is not found", func() {
It("[NVMe] Should automatically add a symlink between /dev/* and /dev/by-id if disk is not found", func() {
testContext := getRandomTestContext()

p, z, _ := testContext.Instance.GetIdentity()
Expand Down Expand Up @@ -331,7 +331,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
Entry("on pd-ssd", ssdDiskType),
)

DescribeTable("Should complete publish/unpublish lifecycle with underspecified volume ID and missing volume",
DescribeTable("[NVMe] Should complete publish/unpublish lifecycle with underspecified volume ID and missing volume",
func(diskType string) {
testContext := getRandomTestContext()

Expand Down Expand Up @@ -1317,15 +1317,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
_, err := getRandomTestContext().Client.ListVolumes()
Expect(err).To(BeNil(), "no error expected when passed valid compute url")

zone := "us-central1-c"
nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone)
i, err := remote.SetupInstance(getRemoteInstanceConfig(), zone, nodeID, computeService)

if err != nil {
klog.Fatalf("Failed to setup instance %v: %v", nodeID, err)
}

klog.Infof("Creating new driver and client for node %s\n", i.GetName())
i := getRandomTestContext().Instance

// Create new driver and client with valid, empty endpoint
klog.Infof("Setup driver with empty compute endpoint %s\n", i.GetName())
Expand All @@ -1349,7 +1341,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
Expect(err).To(BeNil(), "no error expected when passed valid compute url")
})

It("Should update readahead if read_ahead_kb passed on mount", func() {
It("[NVMe] Should update readahead if read_ahead_kb passed on mount", func() {
testContext := getRandomTestContext()

p, z, _ := testContext.Instance.GetIdentity()
Expand Down Expand Up @@ -1410,7 +1402,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
Expect(err).To(BeNil(), "Failed to symlink devicePath")
devFsPathPieces := strings.Split(devFsPath, "/")
devName = devFsPathPieces[len(devFsPathPieces)-1]

break
}
}
Expect(validated).To(BeTrue(), "could not find device in %v that links to volume %s", devicePaths, volName)
Expand Down
4 changes: 3 additions & 1 deletion test/e2e/utils/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,11 +293,13 @@ func ValidateLogicalLinkIsDisk(instance *remote.InstanceInfo, link, diskName str

devFsPath, err := instance.SSH("find", link, "-printf", "'%l'")
if err != nil {
return false, fmt.Errorf("failed to find symbolic link for %s. Output: %v, errror: %v", link, devFsPath, err.Error())
// Skip over if there is no matching symlink.
return false, nil
}
if len(devFsPath) == 0 {
return false, nil
}

if sdx := sdRegex.FindString(devFsPath); len(sdx) != 0 {
fullDevPath := path.Join("/dev/", string(sdx))
scsiIDOut, err := instance.SSH("/lib/udev_containerized/scsi_id", "--page=0x83", "--whitelisted", fmt.Sprintf("--device=%v", fullDevPath))
Expand Down
Loading