Skip to content

Commit 6567a89

Browse files
jsafranebertinatto
jsafrane
authored and
bertinatto
committed
UPSTREAM: <carry>: Add volume group snapshot test driver
Upstream enables volume group snapshots by editing yaml files in a shell script [1]. We can't use this script in openshift-tests. Create a brand new, OCP specific test driver based on csi-driver-hostpath, only with the --feature-gate=VolumeGroupSnapshot on external-snapshotter command line. We will need to carry this patch until the feature graduates to GA. I've chosen to create brand new files in this carry patch, so it can't conflict with the existing ones. 1: https://github.com/kubernetes/kubernetes/blob/91d6fd3455c4a071408df20c7f48df221f2b6d30/test/e2e/testing-manifests/storage-csi/external-snapshotter/volume-group-snapshots/run_group_snapshot_e2e.sh
1 parent 5576b91 commit 6567a89

File tree

4 files changed

+336
-0
lines changed

4 files changed

+336
-0
lines changed

‎openshift-hack/e2e/annotate/rules.go

+5
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,11 @@ var (
6666
// host. Enabling the test would result in the bastion being created for every parallel test execution.
6767
// Given that we have existing oc and WMCO tests that cover this functionality, we can safely disable it.
6868
`\[Feature:NodeLogQuery\]`,
69+
70+
// volumegroupsnapshot in csi-hostpath tests requires changes in the test yaml files,
71+
// which are done by a script upstream. In OCP, we added a separate driver csi-hostpath-groupsnapshot,
72+
// that will not be skipped by any rule here.
73+
`\[Driver: csi-hostpath\].*\[Feature:volumegroupsnapshot\]`,
6974
},
7075
// tests that are known broken and need to be fixed upstream or in openshift
7176
// always add an issue here

‎openshift-hack/test-kubernetes-e2e.sh

+3
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ NETWORK_SKIPS="\[Skipped:Network/OVNKubernetes\]|\[Feature:Networking-IPv6\]|\[F
3030
# Support serial and parallel test suites
3131
TEST_SUITE="${TEST_SUITE:-parallel}"
3232
COMMON_SKIPS="\[Slow\]|\[Disruptive\]|\[Flaky\]|\[Disabled:.+\]|\[Skipped:${PLATFORM}\]|${NETWORK_SKIPS}"
33+
# Skip tests for features that require a TechPreview cluster. TODO: Remove when the feature is enabled by default.
34+
COMMON_SKIPS="\[OCPFeatureGate:VolumeGroupSnapshot\]|${COMMON_SKIPS}"
35+
3336
case "${TEST_SUITE}" in
3437
serial)
3538
DEFAULT_TEST_ARGS="-focus=\[Serial\] -skip=${COMMON_SKIPS}"

‎test/e2e/storage/drivers/openshift_group_snapshot_driver.go

+283
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
package drivers
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
"github.com/onsi/ginkgo/v2"
9+
10+
appsv1 "k8s.io/api/apps/v1"
11+
v1 "k8s.io/api/core/v1"
12+
storagev1 "k8s.io/api/storage/v1"
13+
storagev1beta1 "k8s.io/api/storage/v1beta1"
14+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
15+
"k8s.io/apimachinery/pkg/util/sets"
16+
"k8s.io/kubernetes/test/e2e/framework"
17+
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
18+
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
19+
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
20+
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
21+
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
22+
"k8s.io/kubernetes/test/e2e/storage/utils"
23+
)
24+
25+
// Special test driver for volume group snapshots.
26+
//
27+
// Upstream uses a script to install csi-driver-hostpath with group snapshots enabled in its CSI sidecars.
28+
// We can't use that in OCP, so let's create a new test driver based on [Driver: csi-hospath],
29+
// only with the group snapshots enabled.
30+
31+
// The rest of the file is a copy of Kubernete's HostPath test driver from test/e2e/storage/drivers/csi.go
32+
// Differences:
33+
// - the tests driver name is: [Driver: csi-hospath-groupsnapshot].
34+
// - enabled group snapshots in the external-snapshotter sidecar.
35+
// - still use "csi-hostpath" as PatchCSIOptions.OldDriverName, because it's a name of a directory than needs to be replaced in the driver yaml files.
36+
37+
type groupSnapshotHostpathCSIDriver struct {
38+
driverInfo storageframework.DriverInfo
39+
manifests []string
40+
volumeAttributes []map[string]string
41+
}
42+
43+
func initGroupSnapshotHostpathCSIDriver(name string, capabilities map[storageframework.Capability]bool, volumeAttributes []map[string]string, manifests ...string) storageframework.TestDriver {
44+
return &groupSnapshotHostpathCSIDriver{
45+
driverInfo: storageframework.DriverInfo{
46+
Name: name,
47+
MaxFileSize: storageframework.FileSizeMedium,
48+
SupportedFsType: sets.NewString(
49+
"", // Default fsType
50+
),
51+
SupportedSizeRange: e2evolume.SizeRange{
52+
Min: "1Mi",
53+
},
54+
Capabilities: capabilities,
55+
StressTestOptions: &storageframework.StressTestOptions{
56+
NumPods: 10,
57+
NumRestarts: 10,
58+
},
59+
VolumeSnapshotStressTestOptions: &storageframework.VolumeSnapshotStressTestOptions{
60+
NumPods: 10,
61+
NumSnapshots: 10,
62+
},
63+
PerformanceTestOptions: &storageframework.PerformanceTestOptions{
64+
ProvisioningOptions: &storageframework.PerformanceTestProvisioningOptions{
65+
VolumeSize: "1Mi",
66+
Count: 300,
67+
// Volume provisioning metrics are compared to a high baseline.
68+
// Failure to pass would suggest a performance regression.
69+
ExpectedMetrics: &storageframework.Metrics{
70+
AvgLatency: 2 * time.Minute,
71+
Throughput: 0.5,
72+
},
73+
},
74+
},
75+
TestTags: []interface{}{"[OCPFeatureGate:VolumeGroupSnapshot]"},
76+
},
77+
manifests: manifests,
78+
volumeAttributes: volumeAttributes,
79+
}
80+
}
81+
82+
var _ storageframework.TestDriver = &groupSnapshotHostpathCSIDriver{}
83+
var _ storageframework.DynamicPVTestDriver = &groupSnapshotHostpathCSIDriver{}
84+
var _ storageframework.SnapshottableTestDriver = &groupSnapshotHostpathCSIDriver{}
85+
var _ storageframework.EphemeralTestDriver = &groupSnapshotHostpathCSIDriver{}
86+
87+
// InitgroupSnapshotHostpathCSIDriver returns groupSnapshotHostpathCSIDriver that implements TestDriver interface
88+
func InitGroupSnapshotHostpathCSIDriver() storageframework.TestDriver {
89+
capabilities := map[storageframework.Capability]bool{
90+
storageframework.CapPersistence: true,
91+
storageframework.CapSnapshotDataSource: true,
92+
storageframework.CapMultiPODs: true,
93+
storageframework.CapBlock: true,
94+
storageframework.CapPVCDataSource: true,
95+
storageframework.CapControllerExpansion: true,
96+
storageframework.CapOfflineExpansion: true,
97+
storageframework.CapOnlineExpansion: true,
98+
storageframework.CapSingleNodeVolume: true,
99+
storageframework.CapReadWriteOncePod: true,
100+
storageframework.CapMultiplePVsSameID: true,
101+
storageframework.CapFSResizeFromSourceNotSupported: true,
102+
storageframework.CapVolumeGroupSnapshot: true,
103+
104+
// This is needed for the
105+
// testsuites/volumelimits.go `should support volume limits`
106+
// test. --maxvolumespernode=10 gets
107+
// added when patching the deployment.
108+
storageframework.CapVolumeLimits: true,
109+
}
110+
// OCP specific code: a different driver name (csi-hostpath-groupsnapshot)
111+
return initGroupSnapshotHostpathCSIDriver("csi-hostpath-groupsnapshot",
112+
capabilities,
113+
// Volume attributes don't matter, but we have to provide at least one map.
114+
[]map[string]string{
115+
{"foo": "bar"},
116+
},
117+
"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
118+
"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
119+
"test/e2e/testing-manifests/storage-csi/external-snapshotter/csi-snapshotter/rbac-csi-snapshotter.yaml",
120+
"test/e2e/testing-manifests/storage-csi/external-health-monitor/external-health-monitor-controller/rbac.yaml",
121+
"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
122+
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml",
123+
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml",
124+
"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
125+
)
126+
}
127+
128+
func (h *groupSnapshotHostpathCSIDriver) GetDriverInfo() *storageframework.DriverInfo {
129+
return &h.driverInfo
130+
}
131+
132+
func (h *groupSnapshotHostpathCSIDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
133+
if pattern.VolType == storageframework.CSIInlineVolume && len(h.volumeAttributes) == 0 {
134+
e2eskipper.Skipf("%s has no volume attributes defined, doesn't support ephemeral inline volumes", h.driverInfo.Name)
135+
}
136+
}
137+
138+
func (h *groupSnapshotHostpathCSIDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
139+
provisioner := config.GetUniqueDriverName()
140+
parameters := map[string]string{}
141+
ns := config.Framework.Namespace.Name
142+
143+
return storageframework.GetStorageClass(provisioner, parameters, nil, ns)
144+
}
145+
146+
func (h *groupSnapshotHostpathCSIDriver) GetVolume(config *storageframework.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) {
147+
return h.volumeAttributes[volumeNumber%len(h.volumeAttributes)], false /* not shared */, false /* read-write */
148+
}
149+
150+
func (h *groupSnapshotHostpathCSIDriver) GetCSIDriverName(config *storageframework.PerTestConfig) string {
151+
return config.GetUniqueDriverName()
152+
}
153+
154+
func (h *groupSnapshotHostpathCSIDriver) GetSnapshotClass(ctx context.Context, config *storageframework.PerTestConfig, parameters map[string]string) *unstructured.Unstructured {
155+
snapshotter := config.GetUniqueDriverName()
156+
ns := config.Framework.Namespace.Name
157+
158+
return utils.GenerateSnapshotClassSpec(snapshotter, parameters, ns)
159+
}
160+
161+
func (h *groupSnapshotHostpathCSIDriver) GetVolumeAttributesClass(_ context.Context, config *storageframework.PerTestConfig) *storagev1beta1.VolumeAttributesClass {
162+
return storageframework.CopyVolumeAttributesClass(&storagev1beta1.VolumeAttributesClass{
163+
DriverName: config.GetUniqueDriverName(),
164+
Parameters: map[string]string{
165+
hostpathCSIDriverMutableParameterName: hostpathCSIDriverMutableParameterValue,
166+
},
167+
}, config.Framework.Namespace.Name, "e2e-vac-hostpath")
168+
}
169+
func (h *groupSnapshotHostpathCSIDriver) GetVolumeGroupSnapshotClass(ctx context.Context, config *storageframework.PerTestConfig, parameters map[string]string) *unstructured.Unstructured {
170+
snapshotter := config.GetUniqueDriverName()
171+
ns := config.Framework.Namespace.Name
172+
173+
return utils.GenerateVolumeGroupSnapshotClassSpec(snapshotter, parameters, ns)
174+
}
175+
176+
func (h *groupSnapshotHostpathCSIDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
177+
// Create secondary namespace which will be used for creating driver
178+
driverNamespace := utils.CreateDriverNamespace(ctx, f)
179+
driverns := driverNamespace.Name
180+
181+
ginkgo.By(fmt.Sprintf("deploying %s driver", h.driverInfo.Name))
182+
cancelLogging := utils.StartPodLogs(ctx, f, driverNamespace)
183+
cs := f.ClientSet
184+
185+
// The hostpath CSI driver only works when everything runs on the same node.
186+
node, err := e2enode.GetRandomReadySchedulableNode(ctx, cs)
187+
framework.ExpectNoError(err)
188+
config := &storageframework.PerTestConfig{
189+
Driver: h,
190+
Prefix: "hostpath",
191+
Framework: f,
192+
ClientNodeSelection: e2epod.NodeSelection{Name: node.Name},
193+
DriverNamespace: driverNamespace,
194+
}
195+
196+
patches := []utils.PatchCSIOptions{}
197+
198+
patches = append(patches, utils.PatchCSIOptions{
199+
OldDriverName: "csi-hostpath", // OCP: hardcode csi-hostpath here, it specifies directories in yaml files that need to be replaced with the unique driver name.
200+
NewDriverName: config.GetUniqueDriverName(),
201+
DriverContainerName: "hostpath",
202+
DriverContainerArguments: []string{"--drivername=" + config.GetUniqueDriverName(),
203+
// This is needed for the
204+
// testsuites/volumelimits.go `should support volume limits`
205+
// test.
206+
"--maxvolumespernode=10",
207+
// Enable volume lifecycle checks, to report failure if
208+
// the volume is not unpublished / unstaged correctly.
209+
"--check-volume-lifecycle=true",
210+
},
211+
ProvisionerContainerName: "csi-provisioner",
212+
SnapshotterContainerName: "csi-snapshotter",
213+
NodeName: node.Name,
214+
})
215+
216+
// VAC E2E HostPath patch
217+
// Enables ModifyVolume support in the hostpath CSI driver, and adds an enabled parameter name
218+
patches = append(patches, utils.PatchCSIOptions{
219+
DriverContainerName: "hostpath",
220+
DriverContainerArguments: []string{"--enable-controller-modify-volume=true", "--accepted-mutable-parameter-names=e2eVacTest"},
221+
})
222+
223+
// VAC E2E FeatureGate patches
224+
// TODO: These can be removed after the VolumeAttributesClass feature is default enabled
225+
patches = append(patches, utils.PatchCSIOptions{
226+
DriverContainerName: "csi-provisioner",
227+
DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"},
228+
})
229+
patches = append(patches, utils.PatchCSIOptions{
230+
DriverContainerName: "csi-resizer",
231+
DriverContainerArguments: []string{"--feature-gates=VolumeAttributesClass=true"},
232+
})
233+
234+
// OCP specific code: enable group snapshot
235+
patches = append(patches, utils.PatchCSIOptions{
236+
DriverContainerName: "csi-snapshotter",
237+
DriverContainerArguments: []string{"--feature-gates=CSIVolumeGroupSnapshot=true"},
238+
})
239+
240+
err = utils.CreateFromManifests(ctx, config.Framework, driverNamespace, func(item interface{}) error {
241+
for _, o := range patches {
242+
if err := utils.PatchCSIDeployment(config.Framework, o, item); err != nil {
243+
return err
244+
}
245+
}
246+
247+
// Remove csi-external-health-monitor-agent and
248+
// csi-external-health-monitor-controller
249+
// containers. The agent is obsolete.
250+
// The controller is not needed for any of the
251+
// tests and is causing too much overhead when
252+
// running in a large cluster (see
253+
// https://github.com/kubernetes/kubernetes/issues/102452#issuecomment-856991009).
254+
switch item := item.(type) {
255+
case *appsv1.StatefulSet:
256+
var containers []v1.Container
257+
for _, container := range item.Spec.Template.Spec.Containers {
258+
switch container.Name {
259+
case "csi-external-health-monitor-agent", "csi-external-health-monitor-controller":
260+
// Remove these containers.
261+
default:
262+
// Keep the others.
263+
containers = append(containers, container)
264+
}
265+
}
266+
item.Spec.Template.Spec.Containers = containers
267+
}
268+
return nil
269+
}, h.manifests...)
270+
271+
if err != nil {
272+
framework.Failf("deploying %s driver: %v", h.driverInfo.Name, err)
273+
}
274+
275+
cleanupFunc := generateDriverCleanupFunc(
276+
f,
277+
h.driverInfo.Name,
278+
driverns,
279+
cancelLogging)
280+
ginkgo.DeferCleanup(cleanupFunc)
281+
282+
return config
283+
}

‎test/e2e/storage/openshift_csi_volumes.go

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// This is a copy of csi_volumes.go with OpenShift specific test driver.
18+
// Used a copy of the file to avoid conflicts when editing the existing file.
19+
package storage
20+
21+
import (
22+
"k8s.io/kubernetes/test/e2e/framework"
23+
"k8s.io/kubernetes/test/e2e/storage/drivers"
24+
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
25+
"k8s.io/kubernetes/test/e2e/storage/testsuites"
26+
"k8s.io/kubernetes/test/e2e/storage/utils"
27+
)
28+
29+
// List of testDrivers to be executed in below loop
30+
var ocpCSITestDrivers = []func() storageframework.TestDriver{
31+
drivers.InitGroupSnapshotHostpathCSIDriver,
32+
}
33+
34+
// This executes testSuites for csi volumes.
35+
var _ = utils.SIGDescribe("OCP CSI Volumes", func() {
36+
for _, initDriver := range ocpCSITestDrivers {
37+
curDriver := initDriver()
38+
39+
args := storageframework.GetDriverNameWithFeatureTags(curDriver)
40+
args = append(args, func() {
41+
storageframework.DefineTestSuites(curDriver, testsuites.CSISuites)
42+
})
43+
framework.Context(args...)
44+
}
45+
})

0 commit comments

Comments
 (0)