Fix InternalSnapshotsInfoServiceTests.testErroneousSnapshotShardSizes (elastic#66532)

This test failed few times since elastic#65436 has been merged. 
Now InternalSnapshotsInfoService now also reroutes after
 shard fetching failures and because those reroute are 
executed in the generic thread pool the test should wait 
for all reroutes to be processed.

Author: tlrx

server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java

@@ -43,6 +43,7 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
@@ -179,10 +180,18 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, In
     }
 
     public void testErroneousSnapshotShardSizes() throws Exception {
-        final AtomicInteger reroutes = new AtomicInteger();
+        final int maxShardsToCreate = scaledRandomIntBetween(10, 500);
+
+        final PlainActionFuture<Void> waitForAllReroutesProcessed = new PlainActionFuture<>();
+        final CountDown reroutes = new CountDown(maxShardsToCreate);
         final RerouteService rerouteService = (reason, priority, listener) -> {
-            reroutes.incrementAndGet();
-            listener.onResponse(clusterService.state());
+            try {
+                listener.onResponse(clusterService.state());
+            } finally {
+                if (reroutes.countDown()) {
+                    waitForAllReroutesProcessed.onResponse(null);
+                }
+            }
         };
 
         final InternalSnapshotsInfoService snapshotsInfoService =
@@ -208,7 +217,6 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, In
         };
         when(repositoriesService.repository("_repo")).thenReturn(mockRepository);
 
-        final int maxShardsToCreate = scaledRandomIntBetween(10, 500);
         final Thread addSnapshotRestoreIndicesThread = new Thread(() -> {
             int remainingShards = maxShardsToCreate;
             while (remainingShards > 0) {
@@ -251,8 +259,9 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, In
                 success ? equalTo(results.get(snapshotShard.getKey())) : equalTo(defaultValue));
         }
 
+        waitForAllReroutesProcessed.get(60L, TimeUnit.SECONDS);
         assertThat("Expecting all snapshot shard size fetches to provide a size", results.size(), equalTo(maxShardsToCreate));
-        assertThat("Expecting all snapshot shard size fetches to execute a Reroute", reroutes.get(), equalTo(maxShardsToCreate));
+        assertTrue("Expecting all snapshot shard size fetches to execute a Reroute", reroutes.isCountedDown());
     }
 
     public void testNoLongerMaster() throws Exception {