Fix testMasterFailoverDuringCloneStep1 (#63580) (#64126)

Assuming the clone failed when the request failed is not sufficient.
There are failure modes where the request fails but the clone still works out
because the data node resent the requeest after the first clone had already been
failed and removed from the cluster state when master was restarted.

Closes #63473

Author: original-brownbear

server/src/internalClusterTest/java/org/elasticsearch/snapshots/CloneSnapshotIT.java

@@ -358,8 +358,9 @@ public void testMasterFailoverDuringCloneStep1() throws Exception {
         createFullSnapshot(repoName, sourceSnapshot);
 
         blockMasterOnReadIndexMeta(repoName);
+        final String cloneName = "target-snapshot";
         final ActionFuture<AcknowledgedResponse> cloneFuture =
-                startCloneFromDataNode(repoName, sourceSnapshot, "target-snapshot", testIndex);
+                startCloneFromDataNode(repoName, sourceSnapshot, cloneName, testIndex);
         awaitNumberOfSnapshotsInProgress(1);
         final String masterNode = internalCluster().getMasterName();
         waitForBlock(masterNode, repoName);
@@ -375,6 +376,9 @@ public void testMasterFailoverDuringCloneStep1() throws Exception {
 
         awaitNoMoreRunningOperations();
 
+        // Check if the clone operation worked out by chance as a result of the clone request being retried because of the master failover
+        cloneSucceeded = cloneSucceeded ||
+            getRepositoryData(repoName).getSnapshotIds().stream().anyMatch(snapshotId -> snapshotId.getName().equals(cloneName));
         assertAllSnapshotsSuccessful(getRepositoryData(repoName), cloneSucceeded ? 2 : 1);
     }
 

server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java

@@ -359,6 +359,8 @@ public void createSnapshot(final CreateSnapshotRequest request, final ActionList
         final String repositoryName = request.repository();
         final String snapshotName = indexNameExpressionResolver.resolveDateMathExpression(request.snapshot());
         validate(repositoryName, snapshotName);
+        // TODO: create snapshot UUID in CreateSnapshotRequest and make this operation idempotent to cleanly deal with transport layer
+        //       retries
         final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot
         Repository repository = repositoriesService.repository(request.repository());
         if (repository.isReadOnly()) {
@@ -486,6 +488,8 @@ public void cloneSnapshot(CloneSnapshotRequest request, ActionListener<Void> lis
         }
         final String snapshotName = indexNameExpressionResolver.resolveDateMathExpression(request.target());
         validate(repositoryName, snapshotName);
+        // TODO: create snapshot UUID in CloneSnapshotRequest and make this operation idempotent to cleanly deal with transport layer
+        //       retries
         final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID());
         final Snapshot snapshot = new Snapshot(repositoryName, snapshotId);
         initializingClones.add(snapshot);