Fix Index Deletion During Partial Snapshot Create

We can simply filter out shard generation updates for indices
that were removed from the cluster state concurrently to fix
index deletes during partial snapshots as that completely removes
any reference to those shards from the snapshot.

Follow up to #50202
Closes #50200

Author: original-brownbear

server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java

@@ -568,16 +568,17 @@ public void onNoLongerMaster() {
         private void cleanupAfterError(Exception exception) {
             threadPool.generic().execute(() -> {
                 if (snapshotCreated) {
+                    final MetaData metaData = clusterService.state().metaData();
                     repositoriesService.repository(snapshot.snapshot().getRepository())
                         .finalizeSnapshot(snapshot.snapshot().getSnapshotId(),
-                            buildGenerations(snapshot),
+                            buildGenerations(snapshot, metaData),
                             snapshot.startTime(),
                             ExceptionsHelper.stackTrace(exception),
                             0,
                             Collections.emptyList(),
                             snapshot.repositoryStateId(),
                             snapshot.includeGlobalState(),
-                            metaDataForSnapshot(snapshot, clusterService.state().metaData()),
+                            metaDataForSnapshot(snapshot, metaData),
                             snapshot.userMetadata(),
                             snapshot.useShardGenerations(),
                             ActionListener.runAfter(ActionListener.wrap(ignored -> {
@@ -593,11 +594,22 @@ private void cleanupAfterError(Exception exception) {
         }
     }
 
-    private static ShardGenerations buildGenerations(SnapshotsInProgress.Entry snapshot) {
+    private static ShardGenerations buildGenerations(SnapshotsInProgress.Entry snapshot, MetaData metaData) {
         ShardGenerations.Builder builder = ShardGenerations.builder();
         final Map<String, IndexId> indexLookup = new HashMap<>();
-        snapshot.indices().forEach(idx -> indexLookup.put(idx.getName(), idx));
-        snapshot.shards().forEach(c -> builder.put(indexLookup.get(c.key.getIndexName()), c.key.id(), c.value.generation()));
+        snapshot.indices().forEach(idx -> {
+            if (metaData.index(idx.getName()) != null) {
+                indexLookup.put(idx.getName(), idx);
+            } else {
+                assert snapshot.partial() : "Index [" + idx + "] was deleted during a snapshot but snapshot was not partial.";
+            }
+        });
+        snapshot.shards().forEach(c -> {
+            final IndexId indexId = indexLookup.get(c.key.getIndexName());
+            if (indexId != null) {
+                builder.put(indexId, c.key.id(), c.value.generation());
+            }
+        });
         return builder.build();
     }
 
@@ -1034,7 +1046,7 @@ protected void doRun() {
                 }
                 repository.finalizeSnapshot(
                     snapshot.getSnapshotId(),
-                    buildGenerations(entry),
+                    buildGenerations(entry, metaData),
                     entry.startTime(),
                     failure,
                     entry.shards().size(),

server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java

@@ -211,6 +211,7 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.mockito.Mockito.mock;
 
@@ -514,11 +515,11 @@ public void testConcurrentSnapshotDeleteAndDeleteIndex() {
             testClusterNodes.currentMaster(testClusterNodes.nodes.values().iterator().next().clusterService.state());
 
         final StepListener<Collection<CreateIndexResponse>> createIndicesListener = new StepListener<>();
+        final int indices = randomIntBetween(5, 20);
 
         continueOrDie(createRepoAndIndex(repoName, index, 1), createIndexResponse -> {
             // create a few more indices to make it more likely that the subsequent index delete operation happens before snapshot
             // finalization
-            final int indices = randomIntBetween(5, 20);
             final GroupedActionListener<CreateIndexResponse> listener = new GroupedActionListener<>(createIndicesListener, indices);
             for (int i = 0; i < indices; ++i) {
                 client().admin().indices().create(new CreateIndexRequest("index-" + i), listener);
@@ -527,9 +528,11 @@ public void testConcurrentSnapshotDeleteAndDeleteIndex() {
 
         final StepListener<CreateSnapshotResponse> createSnapshotResponseStepListener = new StepListener<>();
 
+        final boolean partialSnapshot = randomBoolean();
+
         continueOrDie(createIndicesListener, createIndexResponses ->
             client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setWaitForCompletion(false)
-                .execute(createSnapshotResponseStepListener));
+                .setPartial(partialSnapshot).execute(createSnapshotResponseStepListener));
 
         continueOrDie(createSnapshotResponseStepListener,
             createSnapshotResponse -> client().admin().indices().delete(new DeleteIndexRequest(index), noopListener()));
@@ -544,6 +547,13 @@ public void testConcurrentSnapshotDeleteAndDeleteIndex() {
 
         final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotIds.iterator().next());
         assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
+        if (partialSnapshot) {
+            // Single shard for each index so we either get all indices or all except for the deleted index
+            assertThat(snapshotInfo.successfulShards(), either(is(indices + 1)).or(is(indices)));
+        } else {
+            // Index delete must be blocked for non-partial snapshots and we get a snapshot for every index
+            assertEquals(snapshotInfo.successfulShards(), indices + 1);
+        }
         assertEquals(0, snapshotInfo.failedShards());
     }