[Transform] Further increase the timeout and improve message for testStopWaitForCheckpoint (#64865)

improve timeout when starting a transform with retry, improve transform
ids in tests to avoid clashes in logs

fixes #63365

Author: Hendrik Muhs

x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java

@@ -56,6 +56,7 @@ public void cleanTransforms() throws IOException {
 
     public void testTransformCrud() throws Exception {
         String indexName = "basic-crud-reviews";
+        String transformId = "transform-crud";
         createReviewsIndex(indexName, 100);
 
         Map<String, SingleGroupSource> groups = new HashMap<>();
@@ -67,7 +68,7 @@ public void testTransformCrud() throws Exception {
             .addAggregator(AggregationBuilders.avg("review_score").field("stars"))
             .addAggregator(AggregationBuilders.max("timestamp").field("timestamp"));
 
-        TransformConfig config = createTransformConfig("transform-crud", groups, aggs, "reviews-by-user-business-day", indexName);
+        TransformConfig config = createTransformConfig(transformId, groups, aggs, "reviews-by-user-business-day", indexName);
 
         assertTrue(putTransform(config, RequestOptions.DEFAULT).isAcknowledged());
         assertTrue(startTransform(config.getId(), RequestOptions.DEFAULT).isAcknowledged());
@@ -85,6 +86,7 @@ public void testTransformCrud() throws Exception {
 
     public void testContinuousTransformCrud() throws Exception {
         String indexName = "continuous-crud-reviews";
+        String transformId = "transform-continuous-crud";
         createReviewsIndex(indexName, 100);
 
         Map<String, SingleGroupSource> groups = new HashMap<>();
@@ -97,7 +99,7 @@ public void testContinuousTransformCrud() throws Exception {
             .addAggregator(AggregationBuilders.max("timestamp").field("timestamp"));
 
         TransformConfig config = createTransformConfigBuilder(
-            "transform-crud",
+            transformId,
             groups,
             aggs,
             "reviews-by-user-business-day",
@@ -289,6 +291,8 @@ public void testStopWaitForCheckpoint() throws Exception {
 
     public void testContinuousTransformRethrottle() throws Exception {
         String indexName = "continuous-crud-reviews-throttled";
+        String transformId = "transform-continuous-crud-throttled";
+
         createReviewsIndex(indexName, 1000);
 
         Map<String, SingleGroupSource> groups = new HashMap<>();
@@ -301,7 +305,7 @@ public void testContinuousTransformRethrottle() throws Exception {
             .addAggregator(AggregationBuilders.max("timestamp").field("timestamp"));
 
         TransformConfig config = createTransformConfigBuilder(
-            "transform-crud",
+            transformId,
             groups,
             aggs,
             "reviews-by-user-business-day",

x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIntegTestCase.java

@@ -145,6 +145,7 @@ protected StartTransformResponse startTransform(String id, RequestOptions option
     // workaround for https://github.com/elastic/elasticsearch/issues/62204
     protected StartTransformResponse startTransformWithRetryOnConflict(String id, RequestOptions options) throws Exception {
         final int totalRetries = 10;
+        long totalSleepTime = 0;
         ElasticsearchStatusException lastConflict = null;
         for (int retries = totalRetries; retries > 0; --retries) {
             try (RestHighLevelClient restClient = new TestRestHighLevelClient()) {
@@ -163,10 +164,15 @@ protected StartTransformResponse startTransformWithRetryOnConflict(String id, Re
                 }
 
                 lastConflict = e;
-                Thread.sleep(5 * (1 + totalRetries - retries));
+
+                // wait between some ms max 5s, between a check,
+                // with 10 retries the total retry should not be longer than 10s
+                final long sleepTime = 5 * Math.round((Math.min(Math.pow(2, 1 + totalRetries - retries), 1000)));
+                totalSleepTime += sleepTime;
+                Thread.sleep(sleepTime);
             }
         }
-        throw lastConflict;
+        throw new AssertionError("startTransformWithRetryOnConflict timed out after " + totalSleepTime + "ms", lastConflict);
     }
 
     protected AcknowledgedResponse deleteTransform(String id) throws IOException {