[ML] making test more determinate (#40374)

* [ML] making test more determinate

* unmuting test

Author: benwtrent

x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointServiceTests.java

@@ -82,7 +82,6 @@ public void testExtractIndexCheckpointsLostPrimaries() {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/40368")
     public void testExtractIndexCheckpointsInconsistentGlobalCheckpoints() {
         Map<String, long[]> expectedCheckpoints = new HashMap<>();
         Set<String> indices = randomUserIndices();
@@ -141,9 +140,19 @@ private static ShardStats[] createRandomShardStats(Map<String, long[]> expectedC
                 // we need at least one replica for testing
                 int numShardCopies = randomIntBetween(2, 4);
 
+                int primaryShard = 0;
+                if (skipPrimaries) {
+                    primaryShard = randomInt(numShardCopies - 1);
+                }
                 int inconsistentReplica = -1;
                 if (inconsistentGlobalCheckpoints) {
-                    inconsistentReplica = randomIntBetween(0, numShardCopies - 1);
+                    List<Integer> replicas = new ArrayList<>(numShardCopies - 1);
+                    for (int i = 0; i < numShardCopies; i++) {
+                        if (primaryShard != i) {
+                            replicas.add(i);
+                        }
+                    }
+                    inconsistentReplica = randomFrom(replicas);
                 }
 
                 // SeqNoStats asserts that checkpoints are logical
@@ -156,11 +165,7 @@ private static ShardStats[] createRandomShardStats(Map<String, long[]> expectedC
 
                 for (int replica = 0;  replica < numShardCopies; replica++) {
                     ShardId shardId = new ShardId(index, shardIndex);
-                    boolean primary = (replica == 0);
-
-                    if (skipPrimaries) {
-                        primary = randomBoolean();
-                    }
+                    boolean primary = (replica == primaryShard);
 
                     Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(shardIndex));
                     ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, primary,