Fix testDeleteActionDeletesSearchableSnapshot (#68751)

It could happen for ILM to run so fast the test did not get to pick up
the snapshot name from the ILM execution state.
This changes the implementation of the test to not rely on that snapshot
name, but to assert that the test repository is empty after ILM completes
the cycle for the first generation backing index.

Author: andreidan

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java

@@ -9,8 +9,6 @@
 
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.metadata.DataStream;
@@ -22,23 +20,20 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.client.NoOpClient;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.ilm.DeleteAction;
 import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
 import org.elasticsearch.xpack.core.ilm.FreezeAction;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
-import org.elasticsearch.xpack.core.ilm.MountSnapshotStep;
 import org.elasticsearch.xpack.core.ilm.Phase;
 import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep;
 import org.elasticsearch.xpack.core.ilm.RolloverAction;
 import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction;
 import org.elasticsearch.xpack.core.ilm.SetPriorityAction;
 import org.elasticsearch.xpack.core.ilm.ShrinkAction;
 import org.elasticsearch.xpack.core.ilm.Step;
-import org.elasticsearch.xpack.core.ilm.StepKeyTests;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
 import org.junit.Before;
 
@@ -163,8 +158,8 @@ public void testSearchableSnapshotForceMergesIndexToOneSegment() throws Exceptio
             TimeUnit.SECONDS);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/54433")
-    public void testDeleteActionDeletesSearchableSnapshot() throws Exception {
+       @SuppressWarnings("unchecked")
+       public void testDeleteActionDeletesSearchableSnapshot() throws Exception {
         createSnapshotRepo(client(), snapshotRepo, randomBoolean());
 
         // create policy with cold and delete phases
@@ -192,32 +187,29 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception {
         // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index
         rolloverMaxOneDocCondition(client(), dataStream);
 
-        String[] snapshotName = new String[1];
         String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L);
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        assertTrue(waitUntil(() -> {
-            try {
-                Map<String, Object> explainIndex = explainIndex(client(), backingIndexName);
-                if (explainIndex == null) {
-                    // in case we missed the original index and it was deleted
-                    explainIndex = explainIndex(client(), restoredIndexName);
-                }
-                snapshotName[0] = (String) explainIndex.get("snapshot_name");
-                return snapshotName[0] != null;
-            } catch (IOException e) {
-                return false;
-            }
-        }, 30, TimeUnit.SECONDS));
-        assertBusy(() -> assertFalse(indexExists(restoredIndexName)));
+
+        // let's wait for ILM to finish
+        assertBusy(() -> assertFalse(indexExists(backingIndexName)), 60, TimeUnit.SECONDS);
+        assertBusy(() -> assertFalse(indexExists(restoredIndexName)), 60, TimeUnit.SECONDS);
 
         assertTrue("the snapshot we generate in the cold phase should be deleted by the delete phase", waitUntil(() -> {
-            try {
-                Request getSnapshotsRequest = new Request("GET", "_snapshot/" + snapshotRepo + "/" + snapshotName[0]);
-                Response getSnapshotsResponse = client().performRequest(getSnapshotsRequest);
-                return EntityUtils.toString(getSnapshotsResponse.getEntity()).contains("snapshot_missing_exception");
-            } catch (IOException e) {
-                return false;
-            }
+           try {
+               Request getSnapshotsRequest = new Request("GET", "_snapshot/" + snapshotRepo + "/_all");
+               Response getSnapshotsResponse = client().performRequest(getSnapshotsRequest);
+
+               Map<String, Object> responseMap;
+               try (InputStream is = getSnapshotsResponse.getEntity().getContent()) {
+                   responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+               }
+               List<Object> responses = (List<Object>) responseMap.get("responses");
+               Object snapshots = ((Map<String, Object>) responses.get(0)).get("snapshots");
+               return ((List<Map<String, Object>>) snapshots).size() == 0;
+           } catch (Exception e) {
+               logger.error(e.getMessage(), e);
+               return false;
+           }
         }, 30, TimeUnit.SECONDS));
     }