[Test] Update test in SharedClusterSnapshotRestoreIT (#30200)

The `testDeleteSnapshotWithMissingIndexAndShardMetadata` test uses an
obsolete repository directory structure based on index names instead of
UUIDs. Because it swallows exceptions when deleting test files the test
never failed when the directory structure changed.

This commit fixes the test to use the right directory structure and file
 names and to not swallow exceptions anymore.

Author: tlrx

server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java

@@ -85,13 +85,15 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.StoredScriptsIT;
 import org.elasticsearch.snapshots.mockstore.MockRepository;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 
+import java.io.IOException;
 import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -1241,30 +1243,44 @@ public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exceptio
                         .put("compress", false)
                         .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
 
-        createIndex("test-idx-1", "test-idx-2");
+        final String[] indices = {"test-idx-1", "test-idx-2"};
+        createIndex(indices);
         logger.info("--> indexing some data");
         indexRandom(true,
                 client().prepareIndex("test-idx-1", "_doc").setSource("foo", "bar"),
                 client().prepareIndex("test-idx-2", "_doc").setSource("foo", "bar"));
 
         logger.info("--> creating snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1")
+            .setWaitForCompletion(true).setIndices(indices).get();
+        final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
+        assertThat(snapshotInfo.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
+
+        RepositoriesService service = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
+        Repository repository = service.repository("test-repo");
+
+        final Map<String, IndexId> indexIds = repository.getRepositoryData().getIndices();
+        final Path indicesPath = repo.resolve("indices");
 
         logger.info("--> delete index metadata and shard metadata");
-        Path indices = repo.resolve("indices");
-        Path testIndex1 = indices.resolve("test-idx-1");
-        Path testIndex2 = indices.resolve("test-idx-2");
-        Path testIndex2Shard0 = testIndex2.resolve("0");
-        IOUtils.deleteFilesIgnoringExceptions(testIndex1.resolve("snapshot-test-snap-1"));
-        IOUtils.deleteFilesIgnoringExceptions(testIndex2Shard0.resolve("snapshot-test-snap-1"));
+        for (String index : indices) {
+            Path shardZero = indicesPath.resolve(indexIds.get(index).getId()).resolve("0");
+            if (randomBoolean()) {
+                Files.delete(shardZero.resolve("index-0"));
+            }
+            Files.delete(shardZero.resolve("snap-" + snapshotInfo.snapshotId().getUUID() + ".dat"));
+        }
 
         logger.info("--> delete snapshot");
         client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get();
 
         logger.info("--> make sure snapshot doesn't exist");
         assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class);
+
+        for (String index : indices) {
+            assertTrue(Files.notExists(indicesPath.resolve(indexIds.get(index).getId())));
+        }
     }
 
     public void testDeleteSnapshotWithMissingMetadata() throws Exception {
@@ -1357,9 +1373,13 @@ public void testSnapshotWithMissingShardLevelIndexFile() throws Exception {
 
         logger.info("--> deleting shard level index file");
         try (Stream<Path> files = Files.list(repo.resolve("indices"))) {
-            files.forEach(indexPath ->
-                IOUtils.deleteFilesIgnoringExceptions(indexPath.resolve("0").resolve("index-0"))
-            );
+            files.forEach(indexPath -> {
+                try {
+                    Files.delete(indexPath.resolve("0").resolve("index-0"));
+                } catch (IOException e) {
+                    throw new RuntimeException("Failed to delete expected file", e);
+                }
+            });
         }
 
         logger.info("--> creating another snapshot");