Merge branch 'master' into 2021-11-10-jna-5.10

Author: DaveCTurner

.ci/bwcVersions

@@ -44,6 +44,7 @@ BWC_VERSION:
   - "7.15.0"
   - "7.15.1"
   - "7.15.2"
+  - "7.15.3"
   - "7.16.0"
   - "8.0.0"
   - "8.1.0"

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelConfig.java

@@ -45,7 +45,9 @@ public class TrainedModelConfig implements ToXContentObject {
     public static final ParseField TAGS = new ParseField("tags");
     public static final ParseField METADATA = new ParseField("metadata");
     public static final ParseField INPUT = new ParseField("input");
+    @Deprecated
     public static final ParseField ESTIMATED_HEAP_MEMORY_USAGE_BYTES = new ParseField("estimated_heap_memory_usage_bytes");
+    public static final ParseField MODEL_SIZE_BYTES = new ParseField("model_size_bytes", "estimated_heap_memory_usage_bytes");
     public static final ParseField ESTIMATED_OPERATIONS = new ParseField("estimated_operations");
     public static final ParseField LICENSE_LEVEL = new ParseField("license_level");
     public static final ParseField DEFAULT_FIELD_MAP = new ParseField("default_field_map");
@@ -70,7 +72,7 @@ public class TrainedModelConfig implements ToXContentObject {
         PARSER.declareStringArray(TrainedModelConfig.Builder::setTags, TAGS);
         PARSER.declareObject(TrainedModelConfig.Builder::setMetadata, (p, c) -> p.map(), METADATA);
         PARSER.declareObject(TrainedModelConfig.Builder::setInput, (p, c) -> TrainedModelInput.fromXContent(p), INPUT);
-        PARSER.declareLong(TrainedModelConfig.Builder::setEstimatedHeapMemory, ESTIMATED_HEAP_MEMORY_USAGE_BYTES);
+        PARSER.declareLong(TrainedModelConfig.Builder::setModelSize, MODEL_SIZE_BYTES);
         PARSER.declareLong(TrainedModelConfig.Builder::setEstimatedOperations, ESTIMATED_OPERATIONS);
         PARSER.declareString(TrainedModelConfig.Builder::setLicenseLevel, LICENSE_LEVEL);
         PARSER.declareObject(TrainedModelConfig.Builder::setDefaultFieldMap, (p, c) -> p.mapStrings(), DEFAULT_FIELD_MAP);
@@ -101,7 +103,7 @@ public static TrainedModelConfig fromXContent(XContentParser parser) throws IOEx
     private final List<String> tags;
     private final Map<String, Object> metadata;
     private final TrainedModelInput input;
-    private final Long estimatedHeapMemory;
+    private final Long modelSize;
     private final Long estimatedOperations;
     private final String licenseLevel;
     private final Map<String, String> defaultFieldMap;
@@ -120,7 +122,7 @@ public static TrainedModelConfig fromXContent(XContentParser parser) throws IOEx
         List<String> tags,
         Map<String, Object> metadata,
         TrainedModelInput input,
-        Long estimatedHeapMemory,
+        Long modelSize,
         Long estimatedOperations,
         String licenseLevel,
         Map<String, String> defaultFieldMap,
@@ -138,7 +140,7 @@ public static TrainedModelConfig fromXContent(XContentParser parser) throws IOEx
         this.tags = tags == null ? null : Collections.unmodifiableList(tags);
         this.metadata = metadata == null ? null : Collections.unmodifiableMap(metadata);
         this.input = input;
-        this.estimatedHeapMemory = estimatedHeapMemory;
+        this.modelSize = modelSize;
         this.estimatedOperations = estimatedOperations;
         this.licenseLevel = licenseLevel;
         this.defaultFieldMap = defaultFieldMap == null ? null : Collections.unmodifiableMap(defaultFieldMap);
@@ -195,16 +197,36 @@ public TrainedModelInput getInput() {
         return input;
     }
 
+    /**
+     * @deprecated use {@link TrainedModelConfig#getModelSize()} instead
+     * @return the {@link ByteSizeValue} of the model size if available.
+     */
+    @Deprecated
     public ByteSizeValue getEstimatedHeapMemory() {
-        return estimatedHeapMemory == null ? null : new ByteSizeValue(estimatedHeapMemory);
+        return modelSize == null ? null : new ByteSizeValue(modelSize);
     }
 
+    /**
+     * @deprecated use {@link TrainedModelConfig#getModelSizeBytes()} instead
+     * @return the model size in bytes if available.
+     */
+    @Deprecated
     public Long getEstimatedHeapMemoryBytes() {
-        return estimatedHeapMemory;
+        return modelSize;
     }
 
-    public Long getEstimatedOperations() {
-        return estimatedOperations;
+    /**
+     * @return the {@link ByteSizeValue} of the model size if available.
+     */
+    public ByteSizeValue getModelSize() {
+        return modelSize == null ? null : new ByteSizeValue(modelSize);
+    }
+
+    /**
+     * @return the model size in bytes if available.
+     */
+    public Long getModelSizeBytes() {
+        return modelSize;
     }
 
     public String getLicenseLevel() {
@@ -256,8 +278,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (input != null) {
             builder.field(INPUT.getPreferredName(), input);
         }
-        if (estimatedHeapMemory != null) {
-            builder.field(ESTIMATED_HEAP_MEMORY_USAGE_BYTES.getPreferredName(), estimatedHeapMemory);
+        if (modelSize != null) {
+            builder.field(MODEL_SIZE_BYTES.getPreferredName(), modelSize);
         }
         if (estimatedOperations != null) {
             builder.field(ESTIMATED_OPERATIONS.getPreferredName(), estimatedOperations);
@@ -301,7 +323,7 @@ public boolean equals(Object o) {
             && Objects.equals(compressedDefinition, that.compressedDefinition)
             && Objects.equals(tags, that.tags)
             && Objects.equals(input, that.input)
-            && Objects.equals(estimatedHeapMemory, that.estimatedHeapMemory)
+            && Objects.equals(modelSize, that.modelSize)
             && Objects.equals(estimatedOperations, that.estimatedOperations)
             && Objects.equals(licenseLevel, that.licenseLevel)
             && Objects.equals(defaultFieldMap, that.defaultFieldMap)
@@ -322,7 +344,7 @@ public int hashCode() {
             compressedDefinition,
             description,
             tags,
-            estimatedHeapMemory,
+            modelSize,
             estimatedOperations,
             metadata,
             licenseLevel,
@@ -346,7 +368,7 @@ public static class Builder {
         private TrainedModelDefinition definition;
         private String compressedDefinition;
         private TrainedModelInput input;
-        private Long estimatedHeapMemory;
+        private Long modelSize;
         private Long estimatedOperations;
         private String licenseLevel;
         private Map<String, String> defaultFieldMap;
@@ -431,8 +453,8 @@ public Builder setInput(TrainedModelInput input) {
             return this;
         }
 
-        private Builder setEstimatedHeapMemory(Long estimatedHeapMemory) {
-            this.estimatedHeapMemory = estimatedHeapMemory;
+        private Builder setModelSize(Long modelSize) {
+            this.modelSize = modelSize;
             return this;
         }
 
@@ -469,7 +491,7 @@ public TrainedModelConfig build() {
                 tags,
                 metadata,
                 input,
-                estimatedHeapMemory,
+                modelSize,
                 estimatedOperations,
                 licenseLevel,
                 defaultFieldMap,

docs/reference/how-to/fix-common-cluster-issues.asciidoc

@@ -671,3 +671,70 @@ CPU usage or high JVM memory pressure. For tips, see <<high-cpu-usage>> and
 
 If you regularly trigger circuit breaker errors, see <<circuit-breaker-errors>>
 for tips on diagnosing and preventing them.
+
+[discrete]
+[[task-queue-backlog]]
+=== Task queue backlog
+
+A backlogged task queue can prevent tasks from completing and 
+put the cluster into an unhealthy state. 
+Resource constraints, a large number of tasks being triggered at once,
+and long running tasks can all contribute to a backlogged task queue.
+
+[discrete]
+[[diagnose-task-queue-backlog]]
+==== Diagnose a task queue backlog
+
+**Check the thread pool status**
+
+A <<high-cpu-usage,depleted thread pool>> can result in <<rejected-requests,rejected requests>>. 
+
+You can use the <<cat-thread-pool,cat thread pool API>> to 
+see the number of active threads in each thread pool and
+how many tasks are queued, how many have been rejected, and how many have completed. 
+
+[source,console]
+---
+GET /_cat/thread_pool?v&s=t,n&h=type,name,node_name,active,queue,rejected,completed
+---
+
+**Inspect the hot threads on each node**
+
+If a particular thread pool queue is backed up, 
+you can periodically poll the <<cluster-nodes-hot-threads,Nodes hot threads>> API 
+to determine if the thread has sufficient 
+resources to progress and gauge how quickly it is progressing.
+
+[source,console]
+---
+GET /_nodes/hot_threads
+---
+
+**Look for long running tasks**
+
+Long-running tasks can also cause a backlog. 
+You can use the <<tasks,task management>> API to get information about the tasks that are running. 
+Check the `running_time_in_nanos` to identify tasks that are taking an excessive amount of time to complete. 
+
+[source,console]
+----
+GET /_tasks?filter_path=nodes.*.tasks
+----
+
+[discrete]
+[[resolve-task-queue-backlog]]
+==== Resolve a task queue backlog
+
+**Increase available resources** 
+
+If tasks are progressing slowly and the queue is backing up, 
+you might need to take steps to <<reduce-cpu-usage>>. 
+
+In some cases, increasing the thread pool size might help.
+For example, the `force_merge` thread pool defaults to a single thread.
+Increasing the size to 2 might help reduce a backlog of force merge requests.
+
+**Cancel stuck tasks**
+
+If you find the active task's hot thread isn't progressing and there's a backlog, 
+consider canceling the task. 

docs/reference/ilm/actions/ilm-forcemerge.asciidoc

@@ -6,16 +6,36 @@ Phases allowed: hot, warm.
 
 <<indices-forcemerge,Force merges>> the index into 
 the specified maximum number of <<indices-segments,segments>>.
-
 This action makes the index <<dynamic-index-settings,read-only>>.
 
-To use the `forcemerge` action in the `hot` phase, the `rollover` action *must* be present.
-If no rollover action is configured, {ilm-init} will reject the policy. 
-
 [NOTE]
 The `forcemerge` action is best effort. It might happen that some of the
 shards are relocating, in which case they will not be merged.
 
+To use the `forcemerge` action in the `hot` phase, the `rollover` action *must* be present.
+If no rollover action is configured, {ilm-init} will reject the policy. 
+
+[[ilm-forcemerge-performance]]
+.Performance considerations
+****
+Force merge is a resource-intensive operation. 
+If too many force merges are triggered at once, it can negatively impact your cluster. 
+This can happen when you apply an {ilm-init} policy that includes a force merge action 
+to existing indices.
+If they meet the `min_age` criteria, they can immediately proceed through multiple phases. 
+You can prevent this by increasing the `min_age` or setting `index.lifecycle.origination_date` 
+to change how the index age is calculated. 
+
+If you experience a force merge task queue backlog,
+you might need to increase the size of the force merge threadpool so  
+indices can be force merged in parallel. 
+To do this, configure the `thread_pool.force_merge.size` <<cluster-get-settings,cluster setting>>.
+
+IMPORTANT: This can have cascading performance impacts. 
+Monitor cluster performance and increment the size of the thread pool slowly to reduce the backlog.
+****
+
+
 [[ilm-forcemerge-options]]
 ==== Options
 

docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc

@@ -109,9 +109,9 @@ The time when the trained model was created.
 
 `default_field_map` :::
 (object)
-A string object that contains the default field map to use when inferring 
-against the model. For example, {dfanalytics} may train the model on a specific 
-multi-field `foo.keyword`. The analytics job would then supply a default field 
+A string object that contains the default field map to use when inferring
+against the model. For example, {dfanalytics} may train the model on a specific
+multi-field `foo.keyword`. The analytics job would then supply a default field
 map entry for `"foo" : "foo.keyword"`.
 +
 Any field map described in the inference configuration takes precedence.
@@ -120,9 +120,9 @@ Any field map described in the inference configuration takes precedence.
 (string)
 The free-text description of the trained model.
 
-`estimated_heap_memory_usage_bytes`:::
+`model_size_bytes`:::
 (integer)
-The estimated heap usage in bytes to keep the trained model in memory.
+The estimated model size in bytes to keep the trained model in memory.
 
 `estimated_operations`:::
 (integer)
@@ -131,7 +131,7 @@ The estimated number of operations to use the trained model.
 `inference_config`:::
 (object)
 The default configuration for inference. This can be either a `regression`
-or `classification` configuration. It must match the `target_type` of the 
+or `classification` configuration. It must match the `target_type` of the
 underlying `definition.trained_model`.
 +
 .Properties of `inference_config`

docs/reference/ml/df-analytics/apis/put-trained-models.asciidoc

@@ -48,7 +48,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id]
 (Optional, boolean)
 If set to `true` and a `compressed_definition` is provided, the request defers
 definition decompression and skips relevant validations.
-This deferral is useful for systems or users that know a good JVM heap size estimate for their
+This deferral is useful for systems or users that know a good byte size estimate for their
 model and know that their model is valid and likely won't fail during inference.
 
 
@@ -373,10 +373,7 @@ An array of `trained_model` objects. Supported trained models are `tree` and
 A human-readable description of the {infer} trained model.
 
 `estimated_heap_memory_usage_bytes`::
-(Optional, integer)
-The estimated heap usage in bytes to keep the trained model in memory. This
-property is supported only if `defer_definition_decompression` is `true` or the
-model definition is not supplied.
+(Optional, integer) deprecated:[7.16.0,Replaced by `model_size_bytes`]
 
 `estimated_operations`::
 (Optional, integer)
@@ -743,6 +740,12 @@ empty as the index for storing model definitions is configured automatically.
 (Optional, object)
 An object map that contains metadata about the model.
 
+`model_size_bytes`::
+(Optional, integer)
+The estimated memory usage in bytes to keep the trained model in memory. This
+property is supported only if `defer_definition_decompression` is `true` or the
+model definition is not supplied.
+
 `model_type`::
 (Optional, string)
 The created model type. By default the model type is `tree_ensemble`.

docs/reference/rest-api/usage.asciidoc

@@ -161,7 +161,7 @@ GET /_xpack/usage
           "prepackaged": 1,
           "other": 0
         },
-        "estimated_heap_memory_usage_bytes": {
+        "model_size_bytes": {
           "min": 0.0,
           "max": 0.0,
           "avg": 0.0,

rest-api-spec/build.gradle

@@ -82,7 +82,6 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task ->
   task.skipTest("search.aggregation/20_terms/string profiler via map", "The profiler results aren't backwards compatible.")
   task.skipTest("search.aggregation/20_terms/numeric profiler", "The profiler results aren't backwards compatible.")
   task.skipTest("migration/10_get_feature_upgrade_status/Get feature upgrade status", "Awaits backport")
-  task.skipTest("cluster.put_settings/10_basic/Test put and reset transient settings", "Awaits backport")
 
   task.replaceValueInMatch("_type", "_doc")
   task.addAllowedWarningRegex("\\[types removal\\].*")

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml

@@ -1,13 +1,6 @@
 ---
 "Test put and reset transient settings":
-  - skip:
-      version: " - 7.99.99"
-      reason:  "transient settings deprecation"
-      features: "allowed_warnings"
-
   - do:
-      allowed_warnings:
-        - "[transient settings removal] Updating cluster settings through transientSettings is deprecated. Use persistent settings instead."
       cluster.put_settings:
         body:
           transient:
@@ -23,8 +16,6 @@
   - match: {transient: {cluster.routing.allocation.enable: "none"}}
 
   - do:
-      allowed_warnings:
-        - "[transient settings removal] Updating cluster settings through transientSettings is deprecated. Use persistent settings instead."
       cluster.put_settings:
         body:
           transient:

server/src/main/java/org/elasticsearch/Version.java

@@ -91,6 +91,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_7_15_0 = new Version(7150099, org.apache.lucene.util.Version.LUCENE_8_9_0);
     public static final Version V_7_15_1 = new Version(7150199, org.apache.lucene.util.Version.LUCENE_8_9_0);
     public static final Version V_7_15_2 = new Version(7150299, org.apache.lucene.util.Version.LUCENE_8_9_0);
+    public static final Version V_7_15_3 = new Version(7150399, org.apache.lucene.util.Version.LUCENE_8_9_0);
     public static final Version V_7_16_0 = new Version(7160099, org.apache.lucene.util.Version.LUCENE_8_10_1);
     public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_9_0_0);
     public static final Version V_8_1_0 = new Version(8010099, org.apache.lucene.util.Version.LUCENE_9_0_0);

x-pack/plugin/build.gradle

@@ -113,7 +113,6 @@ tasks.named("yamlRestTestV7CompatTransform").configure{ task ->
   task.skipTest("indices.freeze/20_stats/Translog stats on frozen indices", "#70192 -- the freeze index API is removed from 8.0")
   task.skipTest("indices.freeze/10_basic/Basic", "#70192 -- the freeze index API is removed from 8.0")
   task.skipTest("indices.freeze/10_basic/Test index options", "#70192 -- the freeze index API is removed from 8.0")
-  task.skipTest("ml/categorization_agg/Test categorization aggregation with poor settings", "https://github.com/elastic/elasticsearch/pull/79586")
 
   task.replaceValueInMatch("_type", "_doc")
   task.addAllowedWarningRegex("\\[types removal\\].*")