Merge pull request #170 from elastic/master

🤖 ESQL: Merge upstream

Author: github-actions[bot]

build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/MissingJavadocTypeCheck.java

@@ -32,9 +32,9 @@
 import com.puppycrawl.tools.checkstyle.utils.ScopeUtil;
 
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 /**
  * This is a copy of Checkstyle's {@link com.puppycrawl.tools.checkstyle.checks.javadoc.MissingJavadocTypeCheck},
@@ -59,10 +59,10 @@ public class MissingJavadocTypeCheck extends AbstractCheck {
     private Pattern ignorePattern = Pattern.compile("^$");
 
     /**
-     * Specify the list of annotations that allow missed documentation.
+     * Specify the set of annotations that allow missed documentation.
      * Only short names are allowed, e.g. {@code Generated}.
      */
-    private List<String> skipAnnotations = Collections.singletonList("Generated");
+    private Set<String> skipAnnotations = Set.of("Generated");
 
     /**
      * Setter to specify the visibility scope where Javadoc comments are checked.
@@ -89,7 +89,7 @@ public void setExcludeScope(Scope excludeScope) {
      * @param userAnnotations user's value.
      */
     public void setSkipAnnotations(String... userAnnotations) {
-        skipAnnotations = Arrays.asList(userAnnotations);
+        skipAnnotations = Arrays.stream(userAnnotations).collect(Collectors.toSet());
     }
 
     /**
@@ -149,10 +149,7 @@ private boolean shouldCheck(final DetailAST ast) {
 
         return customScope.isIn(scope)
             && (surroundingScope == null || surroundingScope.isIn(scope))
-            && (excludeScope == null
-                || !customScope.isIn(excludeScope)
-                || surroundingScope != null
-                && !surroundingScope.isIn(excludeScope))
+            && (excludeScope == null || !customScope.isIn(excludeScope) || surroundingScope != null && !surroundingScope.isIn(excludeScope))
             && !AnnotationUtil.containsAnnotation(ast, skipAnnotations)
             && ignorePattern.matcher(outerTypeName).find() == false;
     }

docs/changelog/88187.yaml

@@ -0,0 +1,5 @@
+pr: 88187
+summary: Add deployed native models to `inference_stats` in trained model stats response
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/88271.yaml

@@ -0,0 +1,6 @@
+pr: 88271
+summary: "Fix: extract matrix stats using `bucket_selector` `buckets_path`"
+area: Aggregations
+type: bug
+issues:
+ - 87454

docs/changelog/88297.yaml

@@ -0,0 +1,5 @@
+pr: 88297
+summary: Print full exception when console is non-interactive
+area: Infra/Core
+type: enhancement
+issues: []

docs/reference/aggregations/bucket/composite-aggregation.asciidoc

@@ -736,7 +736,7 @@ GET /_search
 
 NOTE: The `after_key` is *usually* the key to the last bucket returned in
 the response, but that isn't guaranteed. Always use the returned `after_key` instead
-of derriving it from the buckets.
+of deriving it from the buckets.
 
 ==== Early termination
 

docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc

@@ -237,3 +237,24 @@ POST /sales/_search?size=0
 --------------------------------------------------
 // TEST[setup:sales]
 <1> Documents without a value in the `tag` field will fall into the same bucket as documents that have the value `N/A`.
+==== Execution Hint
+
+There are different mechanisms by which cardinality aggregations can be executed:
+
+ - by using field values directly (`direct`)
+ - by using global ordinals of the field and resolving those values after
+   finishing a shard (`global_ordinals`)
+ - by using segment ordinal values and resolving those values after each
+   segment (`segment_ordinals`)
+
+Additionally, there are two "heuristic based" modes.  These modes will cause
+Elasticsearch to use some data about the state of the index to choose an
+appropriate execution method.  The two heuristics are: 
+ - `save_time_heuristic` - this is the default in Elasticsearch 8.4 and later.
+ - `save_memory_heuristic` - this was the default in Elasticsearch 8.3 and
+   earlier
+
+When not specified, Elasticsearch will apply a heuristic to chose the
+appropriate mode.  Also note that some data (i.e. non-ordinal fields), `direct`
+is the only option, and the hint will be ignored in these cases.  Generally
+speaking, it should not be necessary to set this value.

docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc

@@ -85,6 +85,10 @@ Bytes of storage required per node.
 (integer)
 Bytes of memory required per node.
 
+`processors`::
+(float)
+Number of processors (vCPUs) required per node.
+
 =======
 `total`::
 (object)
@@ -100,6 +104,11 @@ Total bytes of storage required for the policy.
 `memory`::
 (integer)
 Total bytes of memory required for the policy.
+
+`processors`::
+(float)
+Total number of processors (vCPUs) required for the policy.
+
 =======
 ======
 `current_capacity`::
@@ -125,6 +134,10 @@ Maximum bytes of storage of a node.
 (integer)
 Maximum bytes of memory of a node.
 
+`processors`::
+(float)
+Maximum number of processors (vCPUs) of a node.
+
 =======
 `total`::
 (object)
@@ -140,6 +153,10 @@ Current bytes of storage available for the policy.
 `memory`::
 (integer)
 Current bytes of memory available for the policy.
+
+`processors`::
+Current number of processors (vCPUs) available for the policy.
+
 =======
 ======
 `current_nodes`::
@@ -191,6 +208,10 @@ Bytes of storage required per node.
 (integer)
 Bytes of memory required per node.
 
+`processors`::
+(float)
+Number of processors (vCPUs) required per node.
+
 ==========
 `total`::
 (object)
@@ -206,6 +227,11 @@ Total bytes of storage required for the policy.
 `memory`::
 (integer)
 Total bytes of memory required for the policy.
+
+`processors`::
+(float)
+Total number of processors (vCPUs) required for the policy.
+
 ==========
 =========
 `reason_summary`::

docs/reference/autoscaling/deciders/fixed-decider.asciidoc

@@ -19,6 +19,10 @@ Required amount of node-level storage. Defaults to `-1` (disabled).
 (Optional, <<byte-units,byte value>>)
 Required amount of node-level memory. Defaults to `-1` (disabled).
 
+`processors`::
+(Optional, float)
+Required number of processors. Defaults to disabled.
+
 `nodes`::
 (Optional, integer)
 Number of nodes to use when calculating capacity. Defaults to `1`.
@@ -38,6 +42,7 @@ PUT /_autoscaling/policy/my_autoscaling_policy
     "fixed": {
       "storage": "1tb",
       "memory": "32gb",
+      "processors": 2.3,
       "nodes": 8
     }
   }

docs/reference/index-modules/allocation/data_tier_allocation.asciidoc

@@ -32,4 +32,4 @@ mounted indices>> exclusively.
     are nodes with the `data_warm` role. If there are no nodes in the warm tier,
     but there are nodes with the `data_hot` role, the index is allocated to
     the hot tier.
-    Used in conjuction with <<data-tier-allocation,data tiers>>.
+    Used in conjunction with <<data-tier-allocation,data tiers>>.

docs/reference/mapping/params/format.asciidoc

@@ -38,7 +38,7 @@ https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.htm
 ==== Built In Formats
 
 Most of the below formats have a `strict` companion format, which means that
-year, month and day parts of the week must use respectively 4, 2 and 2 digits
+year, month and day parts of the month must use respectively 4, 2 and 2 digits
 exactly, potentially prepending zeros. For instance a date like `5/11/1` would
 be considered invalid and would need to be rewritten to `2005/11/01` to be
 accepted by the date parser.

docs/reference/mapping/runtime.asciidoc

@@ -110,9 +110,9 @@ against runtime fields.
 You map runtime fields by adding a `runtime` section under the mapping
 definition and defining
 <<modules-scripting-using,a Painless script>>. This script has access to the
-entire context of a document, including the original `_source` and any mapped
-fields plus their values. At query time, the script runs and generates values
-for each scripted field that is required for the query.
+entire context of a document, including the original `_source` via `params._source`
+and any mapped fields plus their values. At query time, the script runs and 
+generates values for each scripted field that is required for the query.
 
 .Emitting runtime field values
 ****
@@ -1358,9 +1358,9 @@ field within a document. A grok pattern is like a regular expression that
 supports aliased expressions that you can reuse.
 
 The script matches on the `%{COMMONAPACHELOG}` log pattern, which understands
-the structure of Apache logs. If the pattern matches, the script emits the
-value of the matching IP address. If the pattern doesn't match
-(`clientip != null`), the script just returns the field value without crashing.
+the structure of Apache logs. If the pattern matches (`clientip != null`),
+the script emits the value of the matching IP address. If the pattern doesn't
+match, the script just returns the field value without crashing.
 
 [source,console]
 ----

docs/reference/scripting/common-script-uses.asciidoc

@@ -76,9 +76,9 @@ field as a runtime field in the mapping. The following runtime script defines a
 grok pattern that extracts structured fields out of the `message` field. 
 
 The script matches on the `%{COMMONAPACHELOG}` log pattern, which understands
-the structure of Apache logs. If the pattern matches, the script emits the
-value matching the IP address. If the pattern doesn't match
-(`clientip != null`), the script just returns the field value without crashing.
+the structure of Apache logs. If the pattern matches (`clientip != null`), the
+script emits the value of the matching IP address. If the pattern doesn't match,
+the script just returns the field value without crashing.
 
 [source,console]
 ----
@@ -418,4 +418,4 @@ defined it in the dissect pattern!
 }
 ----
 // TESTRESPONSE[s/"took" : 2/"took": "$body.took"/]
-// TESTRESPONSE[s/"_id" : "GXx3H3kBKGE42WRNlddJ"/"_id": $body.hits.hits.0._id/]
+// TESTRESPONSE[s/"_id" : "GXx3H3kBKGE42WRNlddJ"/"_id": $body.hits.hits.0._id/]

docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc

@@ -63,6 +63,14 @@ You should run the analysis on a multi-node cluster of a similar size to your
 production cluster so that it can detect any problems that only arise when the
 repository is accessed by many nodes at once.
 
+If the analysis fails then {es} detected that your repository behaved
+unexpectedly. This usually means you are using a third-party storage system
+with an incorrect or incompatible implementation of the API it claims to
+support. If so, this storage system is not suitable for use as a snapshot
+repository. You will need to work with the supplier of your storage system to
+address the incompatibilities that {es} detects. See
+<<self-managed-repo-types>> for more information.
+
 If the analysis is successful this API returns details of the testing process,
 optionally including how long each operation took. You can use this information
 to determine the performance of your storage system. If any operation fails or

docs/reference/sql/concepts.asciidoc

@@ -48,7 +48,7 @@ In practice this means that while in SQL one can potentially have multiple catal
 
 |`cluster`
 |`cluster` (federated)
-|Traditionally in SQL, _cluster_ refers to a single RDMBS instance which contains a number of ++catalog++s or ++database++s (see above). The same word can be reused inside {es} as well however its semantic clarified a bit.
+|Traditionally in SQL, _cluster_ refers to a single RDBMS instance which contains a number of ++catalog++s or ++database++s (see above). The same word can be reused inside {es} as well however its semantic clarified a bit.
 
 While RDBMS tend to have only one running instance, on a single machine (_not_ distributed), {es} goes the opposite way and by default, is distributed and multi-instance.
 

docs/reference/sql/endpoints/odbc/configuration.asciidoc

@@ -290,7 +290,7 @@ Should the driver follow HTTP redirects of the requests to the server?
 +
 * Use local timezone
 +
-This setting controlls the timezone of:
+This setting controls the timezone of:
 +
 ** the context in which the query will execute (especially relevant for functions dealing with timestamp components);
 
@@ -302,7 +302,7 @@ timezone.
 * Auto-escape PVAs
 +
 The pattern-value arguments make use of `_` and `%` as special characters to
-build patern matching values. Some applications however use these chars as
+build pattern matching values. Some applications however use these chars as
 regular ones, which can lead to {es-sql} returning more data than the app
 intended. With the auto escaping, the driver will inspect the arguments and
 will escape these special characters if not already done by the application.

gradle/build.versions.toml

@@ -11,7 +11,7 @@ apache-rat = "org.apache.rat:apache-rat:0.11"
 asm = { group = "org.ow2.asm", name="asm", version.ref="asm" }
 asm-tree = { group = "org.ow2.asm", name="asm-tree", version.ref="asm" }
 bytebuddy = "net.bytebuddy:byte-buddy:1.12.10"
-checkstyle = "com.puppycrawl.tools:checkstyle:10.1"
+checkstyle = "com.puppycrawl.tools:checkstyle:10.3"
 commons-codec = "commons-codec:commons-codec:1.11"
 commmons-io = "commons-io:commons-io:2.2"
 docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.14.13"

modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java

@@ -204,6 +204,26 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     public Object getProperty(List<String> path) {
         if (path.isEmpty()) {
             return this;
+        } else if (path.size() == 2) {
+            if (results == null) {
+                return emptyMap();
+            }
+            final String field = path.get(0)
+                .replaceAll("^\"", "") // remove leading "
+                .replaceAll("^'", "") // remove leading '
+                .replaceAll("\"$", "") // remove trailing "
+                .replaceAll("'$", ""); // remove trailing '
+            final String element = path.get(1);
+            return switch (element) {
+                case "counts" -> results.getFieldCount(field);
+                case "means" -> results.getMean(field);
+                case "variances" -> results.getVariance(field);
+                case "skewness" -> results.getSkewness(field);
+                case "kurtosis" -> results.getKurtosis(field);
+                case "covariance" -> results.getCovariance(field, field);
+                case "correlation" -> results.getCorrelation(field, field);
+                default -> throw new IllegalArgumentException("Found unknown path element [" + element + "] in [" + getName() + "]");
+            };
         } else if (path.size() == 1) {
             String element = path.get(0);
             if (results == null) {

modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java

@@ -19,6 +19,7 @@
 import java.lang.invoke.MethodType;
 import java.util.BitSet;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -146,6 +147,8 @@ private ArrayLengthHelper() {}
     /** factory for arraylength MethodHandle (intrinsic) from Java 9 (pkg-private for tests) */
     static final MethodHandle JAVA9_ARRAY_LENGTH_MH_FACTORY;
 
+    public static final Map<Class<?>, MethodHandle> DEF_TO_BOXED_TYPE_IMPLICIT_CAST;
+
     static {
         final MethodHandles.Lookup methodHandlesLookup = MethodHandles.publicLookup();
 
@@ -182,6 +185,43 @@ private ArrayLengthHelper() {}
             arrayLengthMHFactory = null;
         }
         JAVA9_ARRAY_LENGTH_MH_FACTORY = arrayLengthMHFactory;
+
+        Map<Class<?>, MethodHandle> defToBoxedTypeImplicitCast = new HashMap<>();
+
+        try {
+            defToBoxedTypeImplicitCast.put(
+                Byte.class,
+                methodHandlesLookup.findStatic(Def.class, "defToByteImplicit", MethodType.methodType(Byte.class, Object.class))
+            );
+            defToBoxedTypeImplicitCast.put(
+                Short.class,
+                methodHandlesLookup.findStatic(Def.class, "defToShortImplicit", MethodType.methodType(Short.class, Object.class))
+            );
+            defToBoxedTypeImplicitCast.put(
+                Character.class,
+                methodHandlesLookup.findStatic(Def.class, "defToCharacterImplicit", MethodType.methodType(Character.class, Object.class))
+            );
+            defToBoxedTypeImplicitCast.put(
+                Integer.class,
+                methodHandlesLookup.findStatic(Def.class, "defToIntegerImplicit", MethodType.methodType(Integer.class, Object.class))
+            );
+            defToBoxedTypeImplicitCast.put(
+                Long.class,
+                methodHandlesLookup.findStatic(Def.class, "defToLongImplicit", MethodType.methodType(Long.class, Object.class))
+            );
+            defToBoxedTypeImplicitCast.put(
+                Float.class,
+                methodHandlesLookup.findStatic(Def.class, "defToFloatImplicit", MethodType.methodType(Float.class, Object.class))
+            );
+            defToBoxedTypeImplicitCast.put(
+                Double.class,
+                methodHandlesLookup.findStatic(Def.class, "defToDoubleImplicit", MethodType.methodType(Double.class, Object.class))
+            );
+        } catch (NoSuchMethodException | IllegalAccessException exception) {
+            throw new IllegalStateException(exception);
+        }
+
+        DEF_TO_BOXED_TYPE_IMPLICIT_CAST = Collections.unmodifiableMap(defToBoxedTypeImplicitCast);
     }
 
     /** Hack to rethrow unknown Exceptions from {@link MethodHandle#invokeExact}: */