changing hard coded 10k page size limit to 65k (#74651)

changes hard code 10k page size limit to 65k(default for `search.max_bucket`)

Closes #57719

Co-authored-by: Hendrik Muhs <[email protected]>

Author: Esduard

docs/reference/rest-api/common-parms.asciidoc

@@ -602,8 +602,8 @@ tag::bulk-dynamic-templates[]
 `dynamic_templates`::
 (Optional, map)
 A map from the full name of fields to the name of <<dynamic-templates, dynamic templates>.
-Defaults to an empty map. If a name matches a dynamic template, then that template will be 
-applied regardless of other match predicates defined in the template. And if a field is 
+Defaults to an empty map. If a name matches a dynamic template, then that template will be
+applied regardless of other match predicates defined in the template. And if a field is
 already defined in the mapping, then this parameter won't be used.
 end::bulk-dynamic-templates[]
 
@@ -896,11 +896,11 @@ end::source-transforms[]
 tag::source-index-transforms[]
 The _source indices_ for the {transform}. It can be a single index, an index
 pattern (for example, `"my-index-*"`), an array of indices (for example,
-`["my-index-000001", "my-index-000002"]`), or an array of index patterns (for 
-example, `["my-index-*", "my-other-index-*"]`. For remote indices use the syntax 
+`["my-index-000001", "my-index-000002"]`), or an array of index patterns (for
+example, `["my-index-*", "my-other-index-*"]`. For remote indices use the syntax
 `"remote_name:index_name"`.
 
-NOTE: If any indices are in remote clusters then the master node and at least 
+NOTE: If any indices are in remote clusters then the master node and at least
 one transform node must have the `remote_cluster_client` node role.
 end::source-index-transforms[]
 
@@ -910,8 +910,8 @@ A query clause that retrieves a subset of data from the source index. See
 end::source-query-transforms[]
 
 tag::source-runtime-mappings-transforms[]
-Definitions of search-time runtime fields that can be used by the transform. For 
-search runtime fields all data nodes, including remote nodes, must be 7.12 or 
+Definitions of search-time runtime fields that can be used by the transform. For
+search runtime fields all data nodes, including remote nodes, must be 7.12 or
 later.
 end::source-runtime-mappings-transforms[]
 
@@ -971,7 +971,7 @@ unique key.
 end::transform-latest[]
 
 tag::transform-retention[]
-Defines a retention policy for the {transform}. Data that meets the defined 
+Defines a retention policy for the {transform}. Data that meets the defined
 criteria is deleted from the destination index.
 end::transform-retention[]
 
@@ -984,7 +984,7 @@ The date field that is used to calculate the age of the document.
 end::transform-retention-time-field[]
 
 tag::transform-retention-time-max-age[]
-Specifies the maximum age of a document in the destination index. Documents that 
+Specifies the maximum age of a document in the destination index. Documents that
 are older than the configured value are removed from the destination index.
 end::transform-retention-time-max-age[]
 
@@ -1008,7 +1008,7 @@ end::transform-settings-docs-per-second[]
 tag::transform-settings-max-page-search-size[]
 Defines the initial page size to use for the composite aggregation for each
 checkpoint. If circuit breaker exceptions occur, the page size is dynamically
-adjusted to a lower value. The minimum value is `10` and the maximum is `10,000`.
+adjusted to a lower value. The minimum value is `10` and the maximum is `65,536`.
 The default value is `500`.
 end::transform-settings-max-page-search-size[]
 

x-pack/plugin/build.gradle

@@ -167,6 +167,7 @@ tasks.named("yamlRestCompatTest").configure {
     'transform/transforms_cat_apis/Test cat transform stats hiding headers',
     'transform/transforms_cat_apis/Test cat transform stats with column selection',
     'transform/transforms_cat_apis/Test cat transform stats with continuous transform',
+    'transform/transforms_crud/Test put config with invalid pivot size', // disabled until backport of #74651 has finished
     'vectors/10_dense_vector_basic/Deprecated function signature',
     'vectors/30_sparse_vector_basic/Cosine Similarity',
     'vectors/30_sparse_vector_basic/Deprecated function signature',

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.xpack.core.transform.TransformField;
 
 import java.io.IOException;
@@ -97,10 +98,12 @@ public Integer getDatesAsEpochMillisForUpdate() {
     }
 
     public ActionRequestValidationException validate(ActionRequestValidationException validationException) {
-        // TODO: make this dependent on search.max_buckets
-        if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 10_000)) {
+        if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) {
             validationException = addValidationError(
-                "settings.max_page_search_size [" + maxPageSearchSize + "] must be greater than 10 and less than 10,000",
+                "settings.max_page_search_size ["
+                    + maxPageSearchSize
+                    + "] is out of range. The minimum value is 10 and the maximum is "
+                    + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS,
                 validationException
             );
         }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper;
@@ -170,9 +171,13 @@ public int hashCode() {
     }
 
     public ActionRequestValidationException validate(ActionRequestValidationException validationException) {
-        if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 10_000)) {
+
+        if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) {
             validationException = addValidationError(
-                "pivot.max_page_search_size [" + maxPageSearchSize + "] must be greater than 10 and less than 10,000",
+                "pivot.max_page_search_size ["
+                    + maxPageSearchSize
+                    + "] is out of range. The minimum value is 10 and the maximum is "
+                    + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS,
                 validationException
             );
         }

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_crud.yml

@@ -480,7 +480,7 @@ setup:
 ---
 "Test put config with invalid pivot size":
   - do:
-      catch: /pivot\.max_page_search_size \[5\] must be greater than 10 and less than 10,000/
+      catch: /pivot\.max_page_search_size \[5\] is out of range. The minimum value is 10 and the maximum is 65536/
       transform.put_transform:
         transform_id: "airline-transform"
         body: >
@@ -494,15 +494,16 @@ setup:
             }
           }
   - do:
-      catch: /pivot\.max_page_search_size \[15000\] must be greater than 10 and less than 10,000/
+      catch: /pivot\.max_page_search_size \[75000\] is out of range. The minimum value is 10 and the maximum is 65536/
+
       transform.put_transform:
         transform_id: "airline-transform"
         body: >
           {
             "source": { "index": "airline-data" },
             "dest": { "index": "airline-dest-index" },
             "pivot": {
-              "max_page_search_size": 15000,
+              "max_page_search_size": 75000,
               "group_by": { "airline": {"terms": {"field": "airline"}}},
               "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
             }