Fix timeout in testDowngradeRemoteClusterToBasic (#52322)

- ESCCRRestTestCase#ensureYellow does not work well with assertBusy
- Increases timeout to 60s

Closes #52036

Author: dnhatn

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -182,7 +182,7 @@ public void initClient() throws IOException {
         assert hasXPack != null;
         assert nodeVersions != null;
     }
-    
+
     /**
      * Helper class to check warnings in REST responses with sensitivity to versions
      * used in the target cluster.
@@ -191,26 +191,26 @@ public static class VersionSensitiveWarningsHandler implements WarningsHandler {
         Set<String> requiredSameVersionClusterWarnings = new HashSet<>();
         Set<String> allowedWarnings = new HashSet<>();
         final Set<Version> testNodeVersions;
-        
+
         public VersionSensitiveWarningsHandler(Set<Version> nodeVersions) {
             this.testNodeVersions = nodeVersions;
         }
 
         /**
          * Adds to the set of warnings that are all required in responses if the cluster
-         * is formed from nodes all running the exact same version as the client. 
+         * is formed from nodes all running the exact same version as the client.
          * @param requiredWarnings a set of required warnings
          */
         public void current(String... requiredWarnings) {
             requiredSameVersionClusterWarnings.addAll(Arrays.asList(requiredWarnings));
         }
 
         /**
-         * Adds to the set of warnings that are permissible (but not required) when running 
+         * Adds to the set of warnings that are permissible (but not required) when running
          * in mixed-version clusters or those that differ in version from the test client.
          * @param allowedWarnings optional warnings that will be ignored if received
          */
-        public void compatible(String... allowedWarnings) {            
+        public void compatible(String... allowedWarnings) {
             this.allowedWarnings.addAll(Arrays.asList(allowedWarnings));
         }
 
@@ -231,13 +231,13 @@ public boolean warningsShouldFailRequest(List<String> warnings) {
                 return false;
             }
         }
-        
+
         private boolean isExclusivelyTargetingCurrentVersionCluster() {
             assertFalse("Node versions running in the cluster are missing", testNodeVersions.isEmpty());
-            return testNodeVersions.size() == 1 && 
+            return testNodeVersions.size() == 1 &&
                     testNodeVersions.iterator().next().equals(Version.CURRENT);
-        } 
-        
+        }
+
     }
 
     /**
@@ -250,14 +250,14 @@ private boolean isExclusivelyTargetingCurrentVersionCluster() {
     public static RequestOptions expectWarnings(String... warnings) {
         return expectVersionSpecificWarnings(consumer -> consumer.current(warnings));
     }
-    
+
     public static RequestOptions expectVersionSpecificWarnings(Consumer<VersionSensitiveWarningsHandler> expectationsSetter) {
         Builder builder = RequestOptions.DEFAULT.toBuilder();
         VersionSensitiveWarningsHandler warningsHandler = new VersionSensitiveWarningsHandler(nodeVersions);
         expectationsSetter.accept(warningsHandler);
         builder.setWarningsHandler(warningsHandler);
         return builder.build();
-    }    
+    }
 
     /**
      * Construct an HttpHost from the given host and port
@@ -442,7 +442,7 @@ private void wipeCluster() throws Exception {
         // Cleanup rollup before deleting indices.  A rollup job might have bulks in-flight,
         // so we need to fully shut them down first otherwise a job might stall waiting
         // for a bulk to finish against a non-existing index (and then fail tests)
-        // 
+        //
         // Rollups were introduced in 6.3.0 so any cluster that contains older
         // nodes won't be able to do *anything* with rollups, including cleanup.
         if (hasXPack && nodeVersions.first().onOrAfter(Version.V_6_3_0)
@@ -777,17 +777,31 @@ protected static void assertOK(Response response) {
      * @param index index to test for
      **/
     protected static void ensureGreen(String index) throws IOException {
-        Request request = new Request("GET", "/_cluster/health/" + index);
-        request.addParameter("wait_for_status", "green");
-        request.addParameter("wait_for_no_relocating_shards", "true");
-        request.addParameter("timeout", "70s");
-        request.addParameter("level", "shards");
+        ensureHealth(index, (request) -> {
+            request.addParameter("wait_for_status", "green");
+            request.addParameter("wait_for_no_relocating_shards", "true");
+            request.addParameter("timeout", "70s");
+            request.addParameter("level", "shards");
+        });
+    }
+
+    protected static void ensureHealth(Consumer<Request> requestConsumer) throws IOException {
+        ensureHealth("", requestConsumer);
+    }
+
+    protected static void ensureHealth(String index, Consumer<Request> requestConsumer) throws IOException {
+        ensureHealth(client(), index, requestConsumer);
+    }
+
+    protected static void ensureHealth(RestClient client, String index, Consumer<Request> requestConsumer) throws IOException {
+        Request request = new Request("GET", "/_cluster/health" + (index.trim().isEmpty() ? "" : "/" + index));
+        requestConsumer.accept(request);
         try {
-            client().performRequest(request);
+            client.performRequest(request);
         } catch (ResponseException e) {
             if (e.getResponse().getStatusLine().getStatusCode() == HttpStatus.SC_REQUEST_TIMEOUT) {
                 try {
-                    final Response clusterStateResponse = client().performRequest(new Request("GET", "/_cluster/state"));
+                    final Response clusterStateResponse = client.performRequest(new Request("GET", "/_cluster/state?pretty"));
                     fail("timed out waiting for green state for index [" + index + "] " +
                         "cluster state [" + EntityUtils.toString(clusterStateResponse.getEntity()) + "]");
                 } catch (Exception inner) {

x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java

@@ -18,6 +18,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.common.xcontent.ObjectPath.eval;
 import static org.hamcrest.Matchers.containsString;
@@ -46,7 +47,7 @@ public void testDowngradeRemoteClusterToBasic() throws Exception {
         assertBusy(() -> {
             ensureYellow(index1);
             verifyDocuments(index1, 5, "filtered_field:true");
-        });
+        }, 60, TimeUnit.SECONDS);
 
         String index2 = "logs-20190102";
         try (RestClient leaderClient = buildLeaderClient()) {
@@ -101,7 +102,7 @@ public void testDowngradeRemoteClusterToBasic() throws Exception {
                             "the license mode [BASIC] on cluster [leader_cluster] does not enable [ccr]"));
                 });
             }
-        });
+        }, 60, TimeUnit.SECONDS);
 
         // Manually following index2 also does not work after the downgrade:
         Exception e = expectThrows(ResponseException.class, () -> followIndex("leader_cluster", index2));

x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java

@@ -208,14 +208,15 @@ protected static void ensureYellow(final String index) throws IOException {
     }
 
     protected static void ensureYellow(final String index, final RestClient client) throws IOException {
-        final Request request = new Request("GET", "/_cluster/health/" + index);
-        request.addParameter("wait_for_status", "yellow");
-        request.addParameter("wait_for_active_shards", "1");
-        request.addParameter("wait_for_no_relocating_shards", "true");
-        request.addParameter("wait_for_no_initializing_shards", "true");
-        request.addParameter("timeout", "5s");
-        request.addParameter("level", "shards");
-        client.performRequest(request);
+        ensureHealth(client, index, request -> {
+            request.addParameter("wait_for_status", "yellow");
+            request.addParameter("wait_for_active_shards", "1");
+            request.addParameter("wait_for_no_relocating_shards", "true");
+            // follower index can be yellow even when its primary shards are still initializing as we bootstrap them using snapshot/restore.
+            request.addParameter("wait_for_no_initializing_shards", "true");
+            request.addParameter("timeout", "5s");
+            request.addParameter("level", "shards");
+        });
     }
 
     protected int countCcrNodeTasks() throws IOException {