Skip to content

Commit 3b8add3

Browse files
davidz627davidz627
committed
Add Cloud KMS encrypted disk lifecycle test
1 parent f3894db commit 3b8add3

File tree

3 files changed

+217
-19
lines changed

3 files changed

+217
-19
lines changed

‎test/e2e/tests/multi_zone_e2e_test.go

+46-17
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"fmt"
1819
"path/filepath"
1920
"strings"
2021

@@ -143,68 +144,96 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
143144

144145
})
145146

146-
func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) {
147+
func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) error {
147148
var err error
148149

149150
glog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly)
150151
// Attach Disk
151152
err = client.ControllerPublishVolume(volID, instance.GetNodeID())
152-
Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volID, instance.GetNodeID())
153+
if err != nil {
154+
return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err)
155+
}
153156

154157
defer func() {
155-
156158
// Detach Disk
157159
err = client.ControllerUnpublishVolume(volID, instance.GetNodeID())
158-
Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error")
160+
if err != nil {
161+
glog.Errorf("Failed to detach disk: %v", err)
162+
}
163+
159164
}()
160165

161166
// Stage Disk
162167
stageDir := filepath.Join("/tmp/", volName, "stage")
163-
client.NodeStageVolume(volID, stageDir)
164-
Expect(err).To(BeNil(), "NodeStageVolume failed with error")
168+
err = client.NodeStageVolume(volID, stageDir)
169+
if err != nil {
170+
return fmt.Errorf("NodeStageVolume failed with error: %v", err)
171+
}
165172

166173
defer func() {
167174
// Unstage Disk
168175
err = client.NodeUnstageVolume(volID, stageDir)
169-
Expect(err).To(BeNil(), "NodeUnstageVolume failed with error")
170-
err = testutils.RmAll(instance, filepath.Join("/tmp/", volName))
171-
Expect(err).To(BeNil(), "Failed to remove temp directory")
176+
if err != nil {
177+
glog.Errorf("Failed to unstage volume: %v", err)
178+
}
179+
fp := filepath.Join("/tmp/", volName)
180+
err = testutils.RmAll(instance, fp)
181+
if err != nil {
182+
glog.Errorf("Failed to rm file path %s: %v", fp, err)
183+
}
172184
}()
173185

174186
// Mount Disk
175187
publishDir := filepath.Join("/tmp/", volName, "mount")
176188
err = client.NodePublishVolume(volID, stageDir, publishDir)
177-
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
189+
if err != nil {
190+
return fmt.Errorf("NodePublishVolume failed with error: %v", err)
191+
}
178192
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
179-
Expect(err).To(BeNil(), "Chmod failed with error")
193+
if err != nil {
194+
return fmt.Errorf("Chmod failed with error: %v", err)
195+
}
180196
testFileContents := "test"
181197
if !readOnly {
182198
// Write a file
183199
testFile := filepath.Join(publishDir, "testfile")
184200
err = testutils.WriteFile(instance, testFile, testFileContents)
185-
Expect(err).To(BeNil(), "Failed to write file")
201+
if err != nil {
202+
return fmt.Errorf("Failed to write file: %v", err)
203+
}
186204
}
187205

188206
// Unmount Disk
189207
err = client.NodeUnpublishVolume(volID, publishDir)
190-
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
208+
if err != nil {
209+
return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
210+
}
191211

192212
// Mount disk somewhere else
193213
secondPublishDir := filepath.Join("/tmp/", volName, "secondmount")
194214
err = client.NodePublishVolume(volID, stageDir, secondPublishDir)
195-
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
215+
if err != nil {
216+
return fmt.Errorf("NodePublishVolume failed with error: %v", err)
217+
}
196218
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
197-
Expect(err).To(BeNil(), "Chmod failed with error")
219+
if err != nil {
220+
return fmt.Errorf("Chmod failed with error: %v", err)
221+
}
198222

199223
// Read File
200224
secondTestFile := filepath.Join(secondPublishDir, "testfile")
201225
readContents, err := testutils.ReadFile(instance, secondTestFile)
202-
Expect(err).To(BeNil(), "ReadFile failed with error")
226+
if err != nil {
227+
return fmt.Errorf("ReadFile failed with error: %v", err)
228+
}
203229
Expect(strings.TrimSpace(string(readContents))).To(Equal(testFileContents))
204230

205231
// Unmount Disk
206232
err = client.NodeUnpublishVolume(volID, secondPublishDir)
207-
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
233+
if err != nil {
234+
return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
235+
}
208236

209237
glog.Infof("Completed testAttachWriteReadDetach with volume %v node %v\n", volID, instance.GetNodeID())
238+
return nil
210239
}

‎test/e2e/tests/setup_e2e_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,14 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"context"
1819
"flag"
1920
"fmt"
2021
"math/rand"
2122
"testing"
2223
"time"
2324

25+
cloudkms "cloud.google.com/go/kms/apiv1"
2426
"github.com/golang/glog"
2527
. "github.com/onsi/ginkgo"
2628
. "github.com/onsi/gomega"
@@ -39,6 +41,7 @@ var (
3941
testContexts = []*remote.TestContext{}
4042
computeService *compute.Service
4143
betaComputeService *computebeta.Service
44+
kmsClient *cloudkms.KeyManagementClient
4245
)
4346

4447
func TestE2E(t *testing.T) {
@@ -62,6 +65,10 @@ var _ = BeforeSuite(func() {
6265
betaComputeService, err = remote.GetBetaComputeClient()
6366
Expect(err).To(BeNil())
6467

68+
// Create the KMS client.
69+
kmsClient, err = cloudkms.NewKeyManagementClient(context.Background())
70+
Expect(err).To(BeNil())
71+
6572
if *runInProw {
6673
*project, *serviceAccount = testutils.SetupProwConfig("gce-project")
6774
}

‎test/e2e/tests/single_zone_e2e_test.go

+164-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"context"
19+
"fmt"
1820
"strings"
1921
"time"
2022

@@ -26,6 +28,10 @@ import (
2628
csi "github.com/container-storage-interface/spec/lib/go/csi"
2729
. "github.com/onsi/ginkgo"
2830
. "github.com/onsi/gomega"
31+
32+
"google.golang.org/api/iterator"
33+
kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1"
34+
fieldmask "google.golang.org/genproto/protobuf/field_mask"
2935
)
3036

3137
const (
@@ -78,7 +84,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
7884
}()
7985

8086
// Attach Disk
81-
testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
87+
err = testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
88+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
8289

8390
})
8491

@@ -152,7 +159,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
152159
}()
153160

154161
// Attach Disk
155-
testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
162+
err = testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
163+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
156164

157165
})
158166

@@ -295,6 +303,160 @@ var _ = Describe("GCE PD CSI Driver", func() {
295303
}()
296304
})
297305

306+
It("Should create CMEK key, go through volume lifecycle, validate behavior on key revoke and restore", func() {
307+
ctx := context.Background()
308+
Expect(testContexts).ToNot(BeEmpty())
309+
testContext := getRandomTestContext()
310+
311+
controllerInstance := testContext.Instance
312+
controllerClient := testContext.Client
313+
314+
p, z, _ := controllerInstance.GetIdentity()
315+
locationID := "global"
316+
317+
// The resource name of the key rings.
318+
parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID)
319+
keyRingId := "gce-pd-csi-test-ring"
320+
321+
// Create KeyRing
322+
ringReq := &kmspb.CreateKeyRingRequest{
323+
Parent: parentName,
324+
KeyRingId: keyRingId,
325+
}
326+
keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq)
327+
if !gce.IsGCEError(err, "alreadyExists") {
328+
getKeyRingReq := &kmspb.GetKeyRingRequest{
329+
Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId),
330+
}
331+
keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq)
332+
333+
}
334+
Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId)
335+
336+
// Create CryptoKey in KeyRing
337+
keyId := "test-key-" + string(uuid.NewUUID())
338+
keyReq := &kmspb.CreateCryptoKeyRequest{
339+
Parent: keyRing.Name,
340+
CryptoKeyId: keyId,
341+
CryptoKey: &kmspb.CryptoKey{
342+
Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT,
343+
VersionTemplate: &kmspb.CryptoKeyVersionTemplate{
344+
Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION,
345+
},
346+
},
347+
}
348+
key, err := kmsClient.CreateCryptoKey(ctx, keyReq)
349+
Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name)
350+
351+
keyVersions := []string{}
352+
keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{
353+
Parent: key.Name,
354+
}
355+
356+
it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq)
357+
358+
for {
359+
keyVersion, err := it.Next()
360+
if err == iterator.Done {
361+
break
362+
}
363+
Expect(err).To(BeNil(), "Failed to list crypto key versions")
364+
365+
keyVersions = append(keyVersions, keyVersion.Name)
366+
}
367+
368+
// Defer deletion of all key versions
369+
// https://cloud.google.com/kms/docs/destroy-restore
370+
defer func() {
371+
372+
for _, keyVersion := range keyVersions {
373+
destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{
374+
Name: keyVersion,
375+
}
376+
_, err = kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq)
377+
Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion)
378+
}
379+
380+
}()
381+
382+
// Go through volume lifecycle using CMEK-ed PD
383+
// Create Disk
384+
volName := testNamePrefix + string(uuid.NewUUID())
385+
volID, err := controllerClient.CreateVolume(volName, map[string]string{
386+
common.ParameterKeyDiskEncryptionKmsKey: key.Name,
387+
}, defaultSizeGb,
388+
&csi.TopologyRequirement{
389+
Requisite: []*csi.Topology{
390+
{
391+
Segments: map[string]string{common.TopologyKeyZone: z},
392+
},
393+
},
394+
})
395+
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
396+
397+
// Validate Disk Created
398+
cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
399+
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
400+
Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
401+
Expect(cloudDisk.Status).To(Equal(readyState))
402+
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
403+
Expect(cloudDisk.Name).To(Equal(volName))
404+
405+
defer func() {
406+
// Delete Disk
407+
err = controllerClient.DeleteVolume(volID)
408+
Expect(err).To(BeNil(), "DeleteVolume failed")
409+
410+
// Validate Disk Deleted
411+
_, err = computeService.Disks.Get(p, z, volName).Do()
412+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
413+
}()
414+
415+
// Test disk works
416+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
417+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key")
418+
419+
// Revoke CMEK key
420+
// https://cloud.google.com/kms/docs/enable-disable
421+
422+
for _, keyVersion := range keyVersions {
423+
disableReq := &kmspb.UpdateCryptoKeyVersionRequest{
424+
CryptoKeyVersion: &kmspb.CryptoKeyVersion{
425+
Name: keyVersion,
426+
State: kmspb.CryptoKeyVersion_DISABLED,
427+
},
428+
UpdateMask: &fieldmask.FieldMask{
429+
Paths: []string{"state"},
430+
},
431+
}
432+
_, err = kmsClient.UpdateCryptoKeyVersion(ctx, disableReq)
433+
Expect(err).To(BeNil(), "Failed to disable crypto key")
434+
}
435+
436+
// Make sure attach of PD fails
437+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
438+
Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded")
439+
440+
// Restore CMEK key
441+
for _, keyVersion := range keyVersions {
442+
enableReq := &kmspb.UpdateCryptoKeyVersionRequest{
443+
CryptoKeyVersion: &kmspb.CryptoKeyVersion{
444+
Name: keyVersion,
445+
State: kmspb.CryptoKeyVersion_ENABLED,
446+
},
447+
UpdateMask: &fieldmask.FieldMask{
448+
Paths: []string{"state"},
449+
},
450+
}
451+
_, err = kmsClient.UpdateCryptoKeyVersion(ctx, enableReq)
452+
Expect(err).To(BeNil(), "Failed to enable crypto key")
453+
}
454+
455+
// Make sure attach of PD succeeds
456+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
457+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key")
458+
})
459+
298460
It("Should create and delete snapshot for RePD in two zones ", func() {
299461
Expect(testContexts).ToNot(BeEmpty())
300462
testContext := getRandomTestContext()

0 commit comments

Comments
 (0)