Delete mounted indices after test case in ESRestTestCase (#73650)

This commit adds some clean up logic to ESRestTestCase so 
that searchable snapshots indices are deleted after test case 
executions, before the snapshot and repositories are wipe out.

Backport of #73555

Author: tlrx

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -539,6 +539,15 @@ protected boolean preserveSLMPoliciesUponCompletion() {
         return false;
     }
 
+    /**
+     * Returns whether to preserve searchable snapshots indices. Defaults to not
+     * preserving them. Only runs at all if xpack is installed on the cluster
+     * being tested.
+     */
+    protected boolean preserveSearchableSnapshotsIndicesUponCompletion() {
+        return false;
+    }
+
     /**
      * Returns whether to wait to make absolutely certain that all snapshots
      * have been deleted.
@@ -560,6 +569,11 @@ private void wipeCluster() throws Exception {
             deleteAllSLMPolicies();
         }
 
+        // Clean up searchable snapshots indices before deleting snapshots and repositories
+        if (hasXPack() && nodeVersions.first().onOrAfter(Version.V_7_8_0) && preserveSearchableSnapshotsIndicesUponCompletion() == false) {
+            wipeSearchableSnapshotsIndices();
+        }
+
         SetOnce<Map<String, List<Map<?,?>>>> inProgressSnapshots = new SetOnce<>();
         if (waitForAllSnapshotsWiped()) {
             AtomicReference<Map<String, List<Map<?,?>>>> snapshots = new AtomicReference<>();
@@ -787,6 +801,22 @@ protected static void wipeDataStreams() throws IOException {
         }
     }
 
+    protected void wipeSearchableSnapshotsIndices() throws IOException {
+        // retrieves all indices with a type of store equals to "snapshot"
+        final Request request = new Request("GET", "_cluster/state/metadata");
+        request.addParameter("filter_path", "metadata.indices.*.settings.index.store.snapshot");
+
+        final Response response = adminClient().performRequest(request);
+        @SuppressWarnings("unchecked")
+        Map<String, ?> indices = (Map<String, ?>) XContentMapValues.extractValue("metadata.indices", entityAsMap(response));
+        if (indices != null) {
+            for (String index : indices.keySet()) {
+                assertAcked("Failed to delete searchable snapshot index [" + index + ']',
+                    adminClient().performRequest(new Request("DELETE", index)));
+            }
+        }
+    }
+
     /**
      * Wipe fs snapshots we created one by one and all repositories so that the next test can create the repositories fresh and they'll
      * start empty. There isn't an API to delete all snapshots. There is an API to delete all snapshot repositories but that leaves all of

x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/clear_cache.yml

@@ -51,19 +51,6 @@ setup:
       indices.delete:
         index: docs
 
----
-teardown:
-
-  - do:
-      snapshot.delete:
-        repository: repository-fs
-        snapshot: snapshot
-        ignore: 404
-
-  - do:
-      snapshot.delete_repository:
-        repository: repository-fs
-
 ---
 "Clear searchable snapshots cache":
   - skip:

x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/indices_stats.yml

@@ -53,18 +53,6 @@ setup:
   - do:
       indices.delete:
         index: docs
----
-teardown:
-
-  - do:
-      snapshot.delete:
-        repository: repository-fs
-        snapshot: snapshot
-        ignore: 404
-
-  - do:
-      snapshot.delete_repository:
-        repository: repository-fs
 
 ---
 "Tests Indices Stats API for snapshot backed indices":

x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/pit.yml

@@ -50,18 +50,6 @@ setup:
   - do:
       indices.delete:
         index: docs
----
-teardown:
-
-  - do:
-      snapshot.delete:
-        repository: repository-fs
-        snapshot: snapshot
-        ignore: 404
-
-  - do:
-      snapshot.delete_repository:
-        repository: repository-fs
 
 ---
 "Tests searches vs default-storage index":

x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/shared_cache_stats.yml

@@ -53,18 +53,6 @@ setup:
   - do:
       indices.delete:
         index: docs
----
-teardown:
-
-  - do:
-      snapshot.delete:
-        repository: repository-fs
-        snapshot: snapshot
-        ignore: 404
-
-  - do:
-      snapshot.delete_repository:
-        repository: repository-fs
 
 ---
 "Node Cache Stats API with Frozen Indices":

x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stats.yml

@@ -50,18 +50,6 @@ setup:
   - do:
       indices.delete:
         index: docs
----
-teardown:
-
-  - do:
-      snapshot.delete:
-        repository: repository-fs
-        snapshot: snapshot
-        ignore: 404
-
-  - do:
-      snapshot.delete_repository:
-        repository: repository-fs
 
 ---
 "Tests searchable snapshots stats":

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java

@@ -65,6 +65,11 @@ protected boolean preserveDataStreamsUponCompletion() {
         return true;
     }
 
+    @Override
+    protected boolean preserveSearchableSnapshotsIndicesUponCompletion() {
+        return true;
+    }
+
     enum ClusterType {
         OLD,
         MIXED,

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java

@@ -106,6 +106,11 @@ protected boolean preserveSnapshotsUponCompletion() {
         return true;
     }
 
+    @Override
+    protected boolean preserveSearchableSnapshotsIndicesUponCompletion() {
+        return true;
+    }
+
     public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }