|
| 1 | +package e2e_test |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "fmt" |
| 6 | + "log" |
| 7 | + "time" |
| 8 | + |
| 9 | + "github.com/onsi/ginkgo/v2" |
| 10 | + "github.com/onsi/gomega" |
| 11 | + |
| 12 | + "github.com/openshift/oadp-operator/tests/e2e/lib" |
| 13 | + libhcp "github.com/openshift/oadp-operator/tests/e2e/lib/hcp" |
| 14 | +) |
| 15 | + |
| 16 | +type HCPBackupRestoreCase struct { |
| 17 | + BackupRestoreCase |
| 18 | + Template string |
| 19 | + Provider string |
| 20 | +} |
| 21 | + |
| 22 | +func runHCPBackupAndRestore(brCase HCPBackupRestoreCase, updateLastBRcase func(brCase HCPBackupRestoreCase), h *libhcp.HCHandler) { |
| 23 | + updateLastBRcase(brCase) |
| 24 | + |
| 25 | + log.Printf("Preparing backup and restore") |
| 26 | + backupName, restoreName := prepareBackupAndRestore(brCase.BackupRestoreCase, func() {}) |
| 27 | + |
| 28 | + err := h.AddHCPPluginToDPA(dpaCR.Namespace, dpaCR.Name, false) |
| 29 | + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to add HCP plugin to DPA: %v", err) |
| 30 | + // TODO: move the wait for HC just after the DPA modification to allow reconciliation to go ahead without waiting for the HC to be created |
| 31 | + |
| 32 | + //Wait for HCP plugin to be added |
| 33 | + gomega.Eventually(libhcp.IsHCPPluginAdded(h.Client, dpaCR.Namespace, dpaCR.Name), 3*time.Minute, 1*time.Second).Should(gomega.BeTrue()) |
| 34 | + |
| 35 | + // Create the HostedCluster for the test |
| 36 | + h.HCPNamespace = libhcp.GetHCPNamespace(brCase.BackupRestoreCase.Name, libhcp.ClustersNamespace) |
| 37 | + h.HostedCluster, err = h.DeployHCManifest(brCase.Template, brCase.Provider, brCase.BackupRestoreCase.Name) |
| 38 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 39 | + |
| 40 | + if brCase.PreBackupVerify != nil { |
| 41 | + err := brCase.PreBackupVerify(runTimeClientForSuiteRun, brCase.Namespace) |
| 42 | + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run HCP pre-backup verification: %v", err) |
| 43 | + } |
| 44 | + |
| 45 | + // Backup HCP & HC |
| 46 | + log.Printf("Backing up HC") |
| 47 | + includedResources := libhcp.HCPIncludedResources |
| 48 | + excludedResources := libhcp.HCPExcludedResources |
| 49 | + includedNamespaces := append(libhcp.HCPIncludedNamespaces, libhcp.GetHCPNamespace(h.HostedCluster.Name, libhcp.ClustersNamespace)) |
| 50 | + |
| 51 | + nsRequiresResticDCWorkaround := runHCPBackup(brCase.BackupRestoreCase, backupName, h, includedNamespaces, includedResources, excludedResources) |
| 52 | + |
| 53 | + // Delete everything in HCP namespace |
| 54 | + log.Printf("Deleting HCP & HC") |
| 55 | + err = h.RemoveHCP(libhcp.Wait10Min) |
| 56 | + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to remove HCP: %v", err) |
| 57 | + |
| 58 | + // Restore HC |
| 59 | + log.Printf("Restoring HC") |
| 60 | + runHCPRestore(brCase.BackupRestoreCase, backupName, restoreName, nsRequiresResticDCWorkaround) |
| 61 | + |
| 62 | + // Wait for HCP to be restored |
| 63 | + log.Printf("Validating HC") |
| 64 | + err = libhcp.ValidateHCP(libhcp.ValidateHCPTimeout, libhcp.Wait10Min, []string{}, h.HCPNamespace)(h.Client, libhcp.ClustersNamespace) |
| 65 | + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to run HCP post-restore verification: %v", err) |
| 66 | +} |
| 67 | + |
| 68 | +var _ = ginkgo.Describe("HCP Backup and Restore tests", ginkgo.Ordered, func() { |
| 69 | + var ( |
| 70 | + lastInstallTime time.Time |
| 71 | + lastBRCase HCPBackupRestoreCase |
| 72 | + h *libhcp.HCHandler |
| 73 | + err error |
| 74 | + ctx = context.Background() |
| 75 | + ) |
| 76 | + |
| 77 | + updateLastBRcase := func(brCase HCPBackupRestoreCase) { |
| 78 | + lastBRCase = brCase |
| 79 | + } |
| 80 | + |
| 81 | + // Before All |
| 82 | + var _ = ginkgo.BeforeAll(func() { |
| 83 | + reqOperators := []libhcp.RequiredOperator{ |
| 84 | + { |
| 85 | + Name: libhcp.MCEName, |
| 86 | + Namespace: libhcp.MCENamespace, |
| 87 | + OperatorGroup: libhcp.MCEOperatorGroup, |
| 88 | + }, |
| 89 | + } |
| 90 | + |
| 91 | + // Install MCE and Hypershift operators |
| 92 | + h, err = libhcp.InstallRequiredOperators(ctx, runTimeClientForSuiteRun, reqOperators) |
| 93 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 94 | + gomega.Expect(h).ToNot(gomega.BeNil()) |
| 95 | + gomega.Eventually(lib.IsDeploymentReady(h.Client, libhcp.MCENamespace, libhcp.MCEOperatorName), libhcp.Wait10Min, time.Second*5).Should(gomega.BeTrue()) |
| 96 | + |
| 97 | + // Deploy the MCE manifest |
| 98 | + err = h.DeployMCEManifest() |
| 99 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 100 | + |
| 101 | + // Deploy the MCE and wait for it to be ready |
| 102 | + gomega.Eventually(lib.IsDeploymentReady(h.Client, libhcp.MCENamespace, libhcp.MCEOperatorName), libhcp.Wait10Min, time.Second*5).Should(gomega.BeTrue()) |
| 103 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 104 | + |
| 105 | + // Validate the Hypershift operator |
| 106 | + gomega.Eventually(lib.IsDeploymentReady(h.Client, libhcp.HONamespace, libhcp.HypershiftOperatorName), libhcp.Wait10Min, time.Second*5).Should(gomega.BeTrue()) |
| 107 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 108 | + }) |
| 109 | + |
| 110 | + // After All |
| 111 | + var _ = ginkgo.AfterAll(func() { |
| 112 | + err := h.RemoveHCP(libhcp.Wait10Min) |
| 113 | + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "failed to remove HCP: %v", err) |
| 114 | + }) |
| 115 | + |
| 116 | + // After Each |
| 117 | + var _ = ginkgo.AfterEach(func(ctx ginkgo.SpecContext) { |
| 118 | + h.RemoveHCP(libhcp.Wait10Min) |
| 119 | + tearDownBackupAndRestore(lastBRCase.BackupRestoreCase, lastInstallTime, ctx.SpecReport()) |
| 120 | + }) |
| 121 | + |
| 122 | + ginkgo.DescribeTable("Basic HCP backup and restore test", |
| 123 | + func(brCase HCPBackupRestoreCase, expectedErr error) { |
| 124 | + if ginkgo.CurrentSpecReport().NumAttempts > 1 && !knownFlake { |
| 125 | + ginkgo.Fail("No known FLAKE found in a previous run, marking test as failed.") |
| 126 | + } |
| 127 | + runHCPBackupAndRestore(brCase, updateLastBRcase, h) |
| 128 | + }, |
| 129 | + |
| 130 | + // Test Cases |
| 131 | + ginkgo.Entry("None HostedCluster backup and restore", ginkgo.Label("hcp"), HCPBackupRestoreCase{ |
| 132 | + Template: libhcp.HCPNoneManifest, |
| 133 | + Provider: "None", |
| 134 | + BackupRestoreCase: BackupRestoreCase{ |
| 135 | + Namespace: libhcp.GetHCPNamespace(fmt.Sprintf("%s-none", libhcp.HostedClusterPrefix), libhcp.ClustersNamespace), |
| 136 | + Name: fmt.Sprintf("%s-none", libhcp.HostedClusterPrefix), |
| 137 | + BackupRestoreType: lib.CSIDataMover, |
| 138 | + PreBackupVerify: libhcp.ValidateHCP(libhcp.ValidateHCPTimeout, libhcp.Wait10Min, []string{}, libhcp.GetHCPNamespace(fmt.Sprintf("%s-none", libhcp.HostedClusterPrefix), libhcp.ClustersNamespace)), |
| 139 | + PostRestoreVerify: libhcp.ValidateHCP(libhcp.ValidateHCPTimeout, libhcp.Wait10Min, []string{}, libhcp.GetHCPNamespace(fmt.Sprintf("%s-none", libhcp.HostedClusterPrefix), libhcp.ClustersNamespace)), |
| 140 | + BackupTimeout: libhcp.HCPBackupTimeout, |
| 141 | + }, |
| 142 | + }, nil), |
| 143 | + |
| 144 | + ginkgo.Entry("Agent HostedCluster backup and restore", ginkgo.Label("hcp"), HCPBackupRestoreCase{ |
| 145 | + Template: libhcp.HCPAgentManifest, |
| 146 | + Provider: "Agent", |
| 147 | + BackupRestoreCase: BackupRestoreCase{ |
| 148 | + Namespace: libhcp.GetHCPNamespace(fmt.Sprintf("%s-agent", libhcp.HostedClusterPrefix), libhcp.ClustersNamespace), |
| 149 | + Name: fmt.Sprintf("%s-agent", libhcp.HostedClusterPrefix), |
| 150 | + BackupRestoreType: lib.CSIDataMover, |
| 151 | + PreBackupVerify: libhcp.ValidateHCP(libhcp.ValidateHCPTimeout, libhcp.Wait10Min, []string{}, libhcp.GetHCPNamespace(fmt.Sprintf("%s-agent", libhcp.HostedClusterPrefix), libhcp.ClustersNamespace)), |
| 152 | + PostRestoreVerify: libhcp.ValidateHCP(libhcp.ValidateHCPTimeout, libhcp.Wait10Min, []string{}, libhcp.GetHCPNamespace(fmt.Sprintf("%s-agent", libhcp.HostedClusterPrefix), libhcp.ClustersNamespace)), |
| 153 | + BackupTimeout: libhcp.HCPBackupTimeout, |
| 154 | + }, |
| 155 | + }, nil), |
| 156 | + ) |
| 157 | +}) |
| 158 | + |
| 159 | +// TODO: Modify the runBackup function to inject the filtered error logs to avoid repeating code with this |
| 160 | +func runHCPBackup(brCase BackupRestoreCase, backupName string, h *libhcp.HCHandler, namespaces []string, includedResources, excludedResources []string) bool { |
| 161 | + nsRequiresResticDCWorkaround, err := lib.NamespaceRequiresResticDCWorkaround(h.Client, brCase.Namespace) |
| 162 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 163 | + |
| 164 | + // create backup |
| 165 | + log.Printf("Creating backup %s for case %s", backupName, brCase.Name) |
| 166 | + err = lib.CreateCustomBackupForNamespaces(h.Client, namespace, backupName, namespaces, includedResources, excludedResources, brCase.BackupRestoreType == lib.RESTIC || brCase.BackupRestoreType == lib.KOPIA, brCase.BackupRestoreType == lib.CSIDataMover) |
| 167 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 168 | + |
| 169 | + // wait for backup to not be running |
| 170 | + gomega.Eventually(lib.IsBackupDone(h.Client, namespace, backupName), brCase.BackupTimeout, time.Second*10).Should(gomega.BeTrue()) |
| 171 | + // TODO only log on fail? |
| 172 | + describeBackup := lib.DescribeBackup(h.Client, namespace, backupName) |
| 173 | + ginkgo.GinkgoWriter.Println(describeBackup) |
| 174 | + |
| 175 | + backupLogs := lib.BackupLogs(kubernetesClientForSuiteRun, h.Client, namespace, backupName) |
| 176 | + backupErrorLogs := lib.BackupErrorLogs(kubernetesClientForSuiteRun, h.Client, namespace, backupName) |
| 177 | + accumulatedTestLogs = append(accumulatedTestLogs, describeBackup, backupLogs) |
| 178 | + |
| 179 | + // Check error logs for non-relevant errors |
| 180 | + filteredBackupErrorLogs := libhcp.FilterErrorLogs(backupErrorLogs) |
| 181 | + |
| 182 | + if !brCase.SkipVerifyLogs { |
| 183 | + gomega.Expect(filteredBackupErrorLogs).Should(gomega.Equal([]string{})) |
| 184 | + } |
| 185 | + |
| 186 | + // check if backup succeeded |
| 187 | + succeeded, err := lib.IsBackupCompletedSuccessfully(kubernetesClientForSuiteRun, h.Client, namespace, backupName) |
| 188 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 189 | + gomega.Expect(succeeded).To(gomega.Equal(true)) |
| 190 | + log.Printf("Backup for case %s succeeded", brCase.Name) |
| 191 | + |
| 192 | + if brCase.BackupRestoreType == lib.CSI { |
| 193 | + // wait for volume snapshot to be Ready |
| 194 | + gomega.Eventually(lib.AreVolumeSnapshotsReady(h.Client, backupName), time.Minute*4, time.Second*10).Should(gomega.BeTrue()) |
| 195 | + } |
| 196 | + |
| 197 | + return nsRequiresResticDCWorkaround |
| 198 | +} |
| 199 | + |
| 200 | +// TODO: Modify the runRestore function to inject the filtered error logs to avoid repeating code with this |
| 201 | +func runHCPRestore(brCase BackupRestoreCase, backupName string, restoreName string, nsRequiresResticDCWorkaround bool) { |
| 202 | + log.Printf("Creating restore %s for case %s", restoreName, brCase.Name) |
| 203 | + err := lib.CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName) |
| 204 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 205 | + gomega.Eventually(lib.IsRestoreDone(dpaCR.Client, namespace, restoreName), time.Minute*60, time.Second*10).Should(gomega.BeTrue()) |
| 206 | + // TODO only log on fail? |
| 207 | + describeRestore := lib.DescribeRestore(dpaCR.Client, namespace, restoreName) |
| 208 | + ginkgo.GinkgoWriter.Println(describeRestore) |
| 209 | + |
| 210 | + restoreLogs := lib.RestoreLogs(kubernetesClientForSuiteRun, dpaCR.Client, namespace, restoreName) |
| 211 | + restoreErrorLogs := lib.RestoreErrorLogs(kubernetesClientForSuiteRun, dpaCR.Client, namespace, restoreName) |
| 212 | + accumulatedTestLogs = append(accumulatedTestLogs, describeRestore, restoreLogs) |
| 213 | + |
| 214 | + // Check error logs for non-relevant errors |
| 215 | + filteredRestoreErrorLogs := libhcp.FilterErrorLogs(restoreErrorLogs) |
| 216 | + |
| 217 | + if !brCase.SkipVerifyLogs { |
| 218 | + gomega.Expect(filteredRestoreErrorLogs).Should(gomega.Equal([]string{})) |
| 219 | + } |
| 220 | + |
| 221 | + // Check if restore succeeded |
| 222 | + succeeded, err := lib.IsRestoreCompletedSuccessfully(kubernetesClientForSuiteRun, dpaCR.Client, namespace, restoreName) |
| 223 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 224 | + gomega.Expect(succeeded).To(gomega.Equal(true)) |
| 225 | + |
| 226 | + if nsRequiresResticDCWorkaround { |
| 227 | + // We run the dc-post-restore.sh script for both restic and |
| 228 | + // kopia backups and for any DCs with attached volumes, |
| 229 | + // regardless of whether it was restic or kopia backup. |
| 230 | + // The script is designed to work with labels set by the |
| 231 | + // openshift-velero-plugin and can be run without pre-conditions. |
| 232 | + log.Printf("Running dc-post-restore.sh script.") |
| 233 | + err = lib.RunDcPostRestoreScript(restoreName) |
| 234 | + gomega.Expect(err).ToNot(gomega.HaveOccurred()) |
| 235 | + } |
| 236 | +} |
0 commit comments