Wait for no initializing shards in CCR tests (elastic#52327)

This commit restores the wait_for_no_initializing_shards check that was
removed in elastic#52284. We need this condition when waiting for the yellow
status on the follower because we bootstrap the primary shards of the
follower using the snapshot/restore.

Relates elastic#52284
Closes elastic#52126

Author: dnhatn

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -969,14 +969,18 @@ protected static void ensureHealth(Consumer<Request> requestConsumer) throws IOE
     }
 
     protected static void ensureHealth(String index, Consumer<Request> requestConsumer) throws IOException {
+        ensureHealth(client(), index, requestConsumer);
+    }
+
+    protected static void ensureHealth(RestClient client, String index, Consumer<Request> requestConsumer) throws IOException {
         Request request = new Request("GET", "/_cluster/health" + (index.trim().isEmpty() ? "" : "/" + index));
         requestConsumer.accept(request);
         try {
-            client().performRequest(request);
+            client.performRequest(request);
         } catch (ResponseException e) {
             if (e.getResponse().getStatusLine().getStatusCode() == HttpStatus.SC_REQUEST_TIMEOUT) {
                 try {
-                    final Response clusterStateResponse = client().performRequest(new Request("GET", "/_cluster/state?pretty"));
+                    final Response clusterStateResponse = client.performRequest(new Request("GET", "/_cluster/state?pretty"));
                     fail("timed out waiting for green state for index [" + index + "] " +
                         "cluster state [" + EntityUtils.toString(clusterStateResponse.getEntity()) + "]");
                 } catch (Exception inner) {

x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java

@@ -19,6 +19,7 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.common.xcontent.ObjectPath.eval;
@@ -48,7 +49,7 @@ public void testDowngradeRemoteClusterToBasic() throws Exception {
         assertBusy(() -> {
             ensureYellow(index1);
             verifyDocuments(index1, 5, "filtered_field:true");
-        });
+        }, 60, TimeUnit.SECONDS);
 
         String index2 = "logs-20190102";
         try (RestClient leaderClient = buildLeaderClient()) {
@@ -88,7 +89,7 @@ public void testDowngradeRemoteClusterToBasic() throws Exception {
                     }
                 });
             }
-        });
+        }, 60, TimeUnit.SECONDS);
 
         // Manually following index2 also does not work after the downgrade:
         Exception e = expectThrows(ResponseException.class, () -> followIndex("leader_cluster", index2));

x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java

@@ -215,14 +215,15 @@ protected static void ensureYellow(final String index) throws IOException {
     }
 
     protected static void ensureYellow(final String index, final RestClient client) throws IOException {
-        final Request request = new Request("GET", "/_cluster/health/" + index);
-        request.addParameter("wait_for_status", "yellow");
-        request.addParameter("wait_for_active_shards", "1");
-        request.addParameter("wait_for_no_relocating_shards", "true");
-        request.addParameter("wait_for_no_initializing_shards", "true");
-        request.addParameter("timeout", "5s");
-        request.addParameter("level", "shards");
-        client.performRequest(request);
+        ensureHealth(client, index, request -> {
+            request.addParameter("wait_for_status", "yellow");
+            request.addParameter("wait_for_active_shards", "1");
+            request.addParameter("wait_for_no_relocating_shards", "true");
+            // follower index can be yellow even when its primary shards are still initializing as we bootstrap them using snapshot/restore.
+            request.addParameter("wait_for_no_initializing_shards", "true");
+            request.addParameter("timeout", "5s");
+            request.addParameter("level", "shards");
+        });
     }
 
     protected int countCcrNodeTasks() throws IOException {