Skip to content

Commit 37c9996

Browse files
jparrilljparrill
committed
OADP-5782: Add hypershift-oadp-plugin E2E tests
Signed-off-by: Juan Manuel Parrilla Madrid <[email protected]>
1 parent a0c5fe4 commit 37c9996

11 files changed

+1060
-24
lines changed

‎tests/e2e/e2e_suite_test.go

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,15 @@ import (
88
"testing"
99
"time"
1010

11-
volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
1211
"github.com/onsi/ginkgo/v2"
1312
"github.com/onsi/gomega"
14-
openshiftappsv1 "github.com/openshift/api/apps/v1"
15-
openshiftbuildv1 "github.com/openshift/api/build/v1"
16-
openshiftroutev1 "github.com/openshift/api/route/v1"
17-
openshiftsecurityv1 "github.com/openshift/api/security/v1"
18-
openshifttemplatev1 "github.com/openshift/api/template/v1"
19-
operatorsv1 "github.com/operator-framework/api/pkg/operators/v1"
20-
operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
21-
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
22-
corev1 "k8s.io/api/core/v1"
2313
"k8s.io/apimachinery/pkg/util/uuid"
2414
"k8s.io/client-go/dynamic"
2515
"k8s.io/client-go/kubernetes"
2616
"k8s.io/client-go/rest"
2717
"sigs.k8s.io/controller-runtime/pkg/client"
2818
"sigs.k8s.io/controller-runtime/pkg/client/config"
2919

30-
oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1"
3120
"github.com/openshift/oadp-operator/tests/e2e/lib"
3221
)
3322

@@ -136,21 +125,9 @@ func TestOADPE2E(t *testing.T) {
136125
kubernetesClientForSuiteRun, err = kubernetes.NewForConfig(kubeConfig)
137126
gomega.Expect(err).NotTo(gomega.HaveOccurred())
138127

139-
runTimeClientForSuiteRun, err = client.New(kubeConfig, client.Options{})
128+
runTimeClientForSuiteRun, err = client.New(kubeConfig, client.Options{Scheme: lib.Scheme})
140129
gomega.Expect(err).NotTo(gomega.HaveOccurred())
141130

142-
oadpv1alpha1.AddToScheme(runTimeClientForSuiteRun.Scheme())
143-
velerov1.AddToScheme(runTimeClientForSuiteRun.Scheme())
144-
openshiftappsv1.AddToScheme(runTimeClientForSuiteRun.Scheme())
145-
openshiftbuildv1.AddToScheme(runTimeClientForSuiteRun.Scheme())
146-
openshiftsecurityv1.AddToScheme(runTimeClientForSuiteRun.Scheme())
147-
openshifttemplatev1.AddToScheme(runTimeClientForSuiteRun.Scheme())
148-
openshiftroutev1.AddToScheme(runTimeClientForSuiteRun.Scheme())
149-
corev1.AddToScheme(runTimeClientForSuiteRun.Scheme())
150-
volumesnapshotv1.AddToScheme(runTimeClientForSuiteRun.Scheme())
151-
operatorsv1alpha1.AddToScheme(runTimeClientForSuiteRun.Scheme())
152-
operatorsv1.AddToScheme(runTimeClientForSuiteRun.Scheme())
153-
154131
dynamicClientForSuiteRun, err = dynamic.NewForConfig(kubeConfig)
155132
gomega.Expect(err).NotTo(gomega.HaveOccurred())
156133

‎tests/e2e/hcp_backup_restore_suite_test.go

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
package e2e_test
2+
3+
import (
4+
"context"
5+
"log"
6+
"time"
7+
8+
"github.com/onsi/ginkgo/v2"
9+
"github.com/onsi/gomega"
10+
hypershiftv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
11+
"github.com/openshift/oadp-operator/tests/e2e/lib"
12+
)
13+
14+
type HCPBackupRestoreCase struct {
15+
BackupRestoreCase
16+
HostedCluster *hypershiftv1.HostedCluster
17+
}
18+
19+
func runHCPBackupAndRestore(brCase HCPBackupRestoreCase, updateLastBRcase func(brCase HCPBackupRestoreCase), h *lib.HCHandler) {
20+
updateLastBRcase(brCase)
21+
22+
log.Printf("Preparing backup and restore")
23+
backupName, restoreName := prepareBackupAndRestore(brCase.BackupRestoreCase, func() {})
24+
25+
err := lib.AddHCPPluginToDPA(h, dpaCR.Namespace, dpaCR.Name, false)
26+
gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to add HCP plugin to DPA: %v", err)
27+
// TODO: move the wait for HC just after the DPA modification to allow reconciliation to go ahead without waiting for the HC to be created
28+
29+
//Wait for HCP plugin to be added
30+
gomega.Eventually(lib.IsHCPPluginAdded(h.Client, brCase.BackupRestoreCase.Namespace, brCase.BackupRestoreCase.Name), 3*time.Minute, 1*time.Second).Should(gomega.BeTrue())
31+
32+
log.Printf("Running pre-backup verification")
33+
if brCase.PreBackupVerify != nil {
34+
err := brCase.PreBackupVerify(runTimeClientForSuiteRun, brCase.Namespace)
35+
gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run HCP pre-backup verification: %v", err)
36+
}
37+
38+
// Backup HCP & HC
39+
log.Printf("Backing up HC")
40+
includedResources := lib.HCPIncludedResources
41+
excludedResources := lib.HCPExcludedResources
42+
includedNamespaces := lib.HCPIncludedNamespaces
43+
44+
nsRequiresResticDCWorkaround := runHCPBackup(brCase.BackupRestoreCase, backupName, h, includedNamespaces, includedResources, excludedResources)
45+
46+
// Delete everything in HCP namespace
47+
log.Printf("Deleting HCP & HC")
48+
err = lib.RemoveHCP(h, brCase.HostedCluster)
49+
gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to remove HCP: %v", err)
50+
51+
// Wait for HC to be deleted
52+
log.Printf("Waiting for HC to be deleted")
53+
gomega.Eventually(lib.IsHCDeleted(h, brCase.HostedCluster), 10*time.Minute, 1*time.Second).Should(gomega.BeTrue())
54+
55+
// Restore HC
56+
log.Printf("Restoring HC")
57+
runRestore(brCase.BackupRestoreCase, backupName, restoreName, nsRequiresResticDCWorkaround)
58+
59+
// Wait for HCP to be restored
60+
log.Printf("Validating HC")
61+
err = lib.ValidateHCP()(h.Client, lib.ClustersNamespace)
62+
gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run HCP post-restore verification: %v", err)
63+
}
64+
65+
var _ = ginkgo.Describe("HCP Backup and Restore tests", ginkgo.Ordered, func() {
66+
var (
67+
lastInstallTime time.Time
68+
lastBRCase HCPBackupRestoreCase
69+
h *lib.HCHandler
70+
ctx context.Context
71+
err error
72+
hc *hypershiftv1.HostedCluster
73+
)
74+
75+
updateLastBRcase := func(brCase HCPBackupRestoreCase) {
76+
lastBRCase = brCase
77+
}
78+
79+
// Before All
80+
var _ = ginkgo.BeforeAll(func() {
81+
reqOperators := []lib.RequiredOperator{
82+
{
83+
Name: lib.MCEName,
84+
Namespace: lib.MCENamespace,
85+
OperatorGroup: lib.MCEOperatorGroup,
86+
},
87+
}
88+
ctx = context.Background()
89+
90+
// Install MCE and Hypershift operators
91+
h, err = lib.InstallRequiredOperators(ctx, runTimeClientForSuiteRun, reqOperators)
92+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
93+
gomega.Expect(h).ToNot(gomega.BeNil())
94+
95+
// Deploy the MCE manifest
96+
err = h.DeployMCEManifest()
97+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
98+
99+
// Deploy the MCE and wait for it to be ready
100+
err = h.ValidateMCE()
101+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
102+
103+
// Deploy MCE manifest
104+
err = h.DeployMCEManifest()
105+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
106+
107+
// Validate the Hypershift operator
108+
err = h.ValidateHO()
109+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
110+
111+
// Create the HostedCluster for the test
112+
hc, err = h.DeployHCManifest()
113+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
114+
lastBRCase.HostedCluster = hc
115+
})
116+
117+
// After All
118+
var _ = ginkgo.AfterAll(func() {
119+
err := lib.RemoveHCP(h, hc)
120+
gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to remove HCP: %v", err)
121+
gomega.Eventually(lib.IsHCDeleted(h, hc), 10*time.Minute, 1*time.Second).Should(gomega.BeTrue(), "HCP was not deleted")
122+
lib.RemoveHCPPluginFromDPA(h, dpaCR.Namespace, dpaCR.Name)
123+
})
124+
125+
// After Each
126+
var _ = ginkgo.AfterEach(func(ctx ginkgo.SpecContext) {
127+
tearDownBackupAndRestore(lastBRCase.BackupRestoreCase, lastInstallTime, ctx.SpecReport())
128+
})
129+
130+
ginkgo.DescribeTable("Basic HCP backup and restore test",
131+
func(brCase BackupRestoreCase, expectedErr error) {
132+
if ginkgo.CurrentSpecReport().NumAttempts > 1 && !knownFlake {
133+
ginkgo.Fail("No known FLAKE found in a previous run, marking test as failed.")
134+
}
135+
runHCPBackupAndRestore(HCPBackupRestoreCase{
136+
BackupRestoreCase: brCase,
137+
}, updateLastBRcase, h)
138+
},
139+
140+
// Test Cases
141+
ginkgo.Entry("None HostedCluster backup and restore", BackupRestoreCase{
142+
Namespace: lib.HCPNamespace,
143+
Name: lib.HostedClusterName,
144+
BackupRestoreType: lib.CSIDataMover,
145+
PreBackupVerify: lib.ValidateHCP(),
146+
PostRestoreVerify: lib.ValidateHCP(),
147+
BackupTimeout: 10 * time.Minute,
148+
}, nil),
149+
)
150+
})
151+
152+
func runHCPBackup(brCase BackupRestoreCase, backupName string, h *lib.HCHandler, namespaces []string, includedResources, excludedResources []string) bool {
153+
nsRequiresResticDCWorkaround, err := lib.NamespaceRequiresResticDCWorkaround(h.Client, brCase.Namespace)
154+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
155+
156+
// create backup
157+
log.Printf("Creating backup %s for case %s", backupName, brCase.Name)
158+
err = lib.CreateCustomBackupForNamespaces(h.Client, namespace, backupName, namespaces, includedResources, excludedResources, brCase.BackupRestoreType == lib.RESTIC || brCase.BackupRestoreType == lib.KOPIA, brCase.BackupRestoreType == lib.CSIDataMover)
159+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
160+
161+
// wait for backup to not be running
162+
gomega.Eventually(lib.IsBackupDone(h.Client, namespace, backupName), brCase.BackupTimeout, time.Second*10).Should(gomega.BeTrue())
163+
// TODO only log on fail?
164+
describeBackup := lib.DescribeBackup(h.Client, namespace, backupName)
165+
ginkgo.GinkgoWriter.Println(describeBackup)
166+
167+
backupLogs := lib.BackupLogs(kubernetesClientForSuiteRun, h.Client, namespace, backupName)
168+
backupErrorLogs := lib.BackupErrorLogs(kubernetesClientForSuiteRun, h.Client, namespace, backupName)
169+
accumulatedTestLogs = append(accumulatedTestLogs, describeBackup, backupLogs)
170+
171+
if !brCase.SkipVerifyLogs {
172+
gomega.Expect(backupErrorLogs).Should(gomega.Equal([]string{}))
173+
}
174+
175+
// check if backup succeeded
176+
succeeded, err := lib.IsBackupCompletedSuccessfully(kubernetesClientForSuiteRun, h.Client, namespace, backupName)
177+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
178+
gomega.Expect(succeeded).To(gomega.Equal(true))
179+
log.Printf("Backup for case %s succeeded", brCase.Name)
180+
181+
if brCase.BackupRestoreType == lib.CSI {
182+
// wait for volume snapshot to be Ready
183+
gomega.Eventually(lib.AreVolumeSnapshotsReady(h.Client, backupName), time.Minute*4, time.Second*10).Should(gomega.BeTrue())
184+
}
185+
186+
return nsRequiresResticDCWorkaround
187+
}

‎tests/e2e/lib/apps.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,7 @@ func IsDeploymentReady(ocClient client.Client, namespace, dName string) wait.Con
285285
if err != nil {
286286
return false, err
287287
}
288+
log.Printf("Deployment %s status: %v", dName, deployment.Status)
288289
if deployment.Status.AvailableReplicas != deployment.Status.Replicas || deployment.Status.Replicas == 0 {
289290
for _, condition := range deployment.Status.Conditions {
290291
if len(condition.Message) > 0 {

‎tests/e2e/lib/backup.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,23 @@ func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupNa
3333
return ocClient.Create(context.Background(), &backup)
3434
}
3535

36+
func CreateCustomBackupForNamespaces(ocClient client.Client, veleroNamespace, backupName string, namespaces []string, includedResources, excludedResources []string, defaultVolumesToFsBackup bool, snapshotMoveData bool) error {
37+
backup := velero.Backup{
38+
ObjectMeta: metav1.ObjectMeta{
39+
Name: backupName,
40+
Namespace: veleroNamespace,
41+
},
42+
Spec: velero.BackupSpec{
43+
IncludedNamespaces: namespaces,
44+
IncludedResources: includedResources,
45+
ExcludedResources: excludedResources,
46+
DefaultVolumesToFsBackup: &defaultVolumesToFsBackup,
47+
SnapshotMoveData: &snapshotMoveData,
48+
},
49+
}
50+
return ocClient.Create(context.Background(), &backup)
51+
}
52+
3653
func GetBackup(c client.Client, namespace string, name string) (*velero.Backup, error) {
3754
backup := velero.Backup{}
3855
err := c.Get(context.Background(), client.ObjectKey{

0 commit comments

Comments
 (0)