Fix testCannotShrinkLeaderIndex (#38529)

This test should no longer pass when the functionality it is intended to
test is broken, as it now indexes a number of documents and verifies
that the index is staying on the same step until after indexing and
replication of those documents is finished. This prevents the test from
passing if the leader index progresses in its lifecycle during that time.

Author: gwbrown

x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java

@@ -358,11 +358,7 @@ public void testCannotShrinkLeaderIndex() throws Exception {
                 changePolicyRequest.setEntity(changePolicyEntity);
                 assertOK(leaderClient.performRequest(changePolicyRequest));
 
-                index(leaderClient, indexName, "1");
-                assertDocumentExists(leaderClient, indexName, "1");
-
                 assertBusy(() -> {
-                    assertDocumentExists(client(), indexName, "1");
                     // Sanity check that following_index setting has been set, so that we can verify later that this setting has been unset:
                     assertThat(getIndexSetting(client(), indexName, "index.xpack.ccr.following_index"), equalTo("true"));
 
@@ -371,6 +367,20 @@ public void testCannotShrinkLeaderIndex() throws Exception {
                     assertILMPolicy(client(), indexName, policyName, "hot", "unfollow", "wait-for-indexing-complete");
                 });
 
+                // Index a bunch of documents and wait for them to be replicated
+                for (int i = 0; i < 50; i++) {
+                    index(leaderClient, indexName, Integer.toString(i));
+                }
+                assertBusy(() -> {
+                    for (int i = 0; i < 50; i++) {
+                        assertDocumentExists(client(), indexName, Integer.toString(i));
+                    }
+                });
+
+                // Then make sure both leader and follower are still both waiting
+                assertILMPolicy(leaderClient, indexName, policyName, "warm", "shrink", "wait-for-shard-history-leases");
+                assertILMPolicy(client(), indexName, policyName, "hot", "unfollow", "wait-for-indexing-complete");
+
                 // Manually set this to kick the process
                 updateIndexSettings(leaderClient, indexName, Settings.builder()
                     .put("index.lifecycle.indexing_complete", true)