Merge remote-tracking branch 'elastic/master' into geoshape-doc-values

Author: talevy

.ci/dockerOnLinuxExclusions

@@ -8,3 +8,8 @@ debian-8
 opensuse-15-1
 ol-7.7
 sles-12
+
+# These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded 
+# for PR checks
+centos-6
+ol-6.10

.ci/matrix-build-javas.yml

@@ -22,5 +22,5 @@ export JAVA8_HOME="${HOME}"/.java/java8
 export JAVA11_HOME="${HOME}"/.java/java11
 export JAVA12_HOME="${HOME}"/.java/openjdk12
 export JAVA13_HOME="${HOME}"/.java/openjdk13
-./gradlew --parallel clean --scan -Porg.elasticsearch.acceptScanTOS=true -s resolveAllDependencies
+./gradlew --parallel clean -s resolveAllDependencies
 

.ci/matrix-java-exclusions.yml

@@ -136,7 +136,7 @@ public void apply(Project project) {
 
         if (dockerComposeSupported() == false) {
             project.getLogger()
-                .warn(
+                .info(
                     "Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose "
                         + "but none could be found so these will be skipped",
                     project.getPath()

.ci/packer_cache.sh

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.5.0-snapshot-3333ce7da6d
+lucene            = 8.5.0-snapshot-d62f6307658
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 13.0.2+8

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -55,7 +55,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
             Supplier<DiscoveryNodes> nodesInCluster) {
         return Arrays.asList(
-                new RestNoopBulkAction(restController),
-                new RestNoopSearchAction(restController));
+                new RestNoopBulkAction(),
+                new RestNoopSearchAction());
     }
 }

buildSrc/version.properties

@@ -32,24 +32,28 @@
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 
 import java.io.IOException;
+import java.util.List;
 
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestNoopBulkAction extends BaseRestHandler {
 
-    public RestNoopBulkAction(RestController controller) {
-        controller.registerHandler(POST, "/_noop_bulk", this);
-        controller.registerHandler(PUT, "/_noop_bulk", this);
-        controller.registerHandler(POST, "/{index}/_noop_bulk", this);
-        controller.registerHandler(PUT, "/{index}/_noop_bulk", this);
+    @Override
+    public List<Route> routes() {
+        return unmodifiableList(asList(
+            new Route(POST, "/_noop_bulk"),
+            new Route(PUT, "/_noop_bulk"),
+            new Route(POST, "/{index}/_noop_bulk"),
+            new Route(PUT, "/{index}/_noop_bulk")));
     }
 
     @Override

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java

@@ -21,20 +21,25 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestNoopSearchAction extends BaseRestHandler {
 
-    public RestNoopSearchAction(RestController controller) {
-        controller.registerHandler(GET, "/_noop_search", this);
-        controller.registerHandler(POST, "/_noop_search", this);
-        controller.registerHandler(GET, "/{index}/_noop_search", this);
-        controller.registerHandler(POST, "/{index}/_noop_search", this);
+    @Override
+    public List<Route> routes() {
+        return unmodifiableList(asList(
+            new Route(GET, "/_noop_search"),
+            new Route(POST, "/_noop_search"),
+            new Route(GET, "/{index}/_noop_search"),
+            new Route(POST, "/{index}/_noop_search")));
     }
 
     @Override

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java

@@ -21,13 +21,9 @@
 
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.client.Validatable;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
-import java.io.IOException;
-
-public final class GetSourceRequest implements Validatable, ToXContentObject {
+public final class GetSourceRequest implements Validatable {
     private String routing;
     private String preference;
 
@@ -36,8 +32,8 @@ public final class GetSourceRequest implements Validatable, ToXContentObject {
 
     private FetchSourceContext fetchSourceContext;
 
-    private String index;
-    private String id;
+    private final String index;
+    private final String id;
 
     public GetSourceRequest(String index, String id) {
         this.index = index;
@@ -101,11 +97,6 @@ public GetSourceRequest fetchSourceContext(FetchSourceContext context) {
         return this;
     }
 
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return null;
-    }
-
     public String index() {
         return index;
     }

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java

@@ -103,7 +103,7 @@ public String getName() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject();
-        if (featureNames != null) {
+        if (featureNames != null && featureNames.isEmpty() == false) {
             builder.field(FEATURE_NAMES.getPreferredName(), featureNames);
         }
         if (models != null) {
@@ -157,7 +157,7 @@ public static Builder builder() {
     }
 
     public static class Builder {
-        private List<String> featureNames;
+        private List<String> featureNames = Collections.emptyList();
         private List<TrainedModel> trainedModels;
         private OutputAggregator outputAggregator;
         private TargetType targetType;

client/rest-high-level/src/main/java/org/elasticsearch/client/core/GetSourceRequest.java

@@ -42,6 +42,7 @@ public class TreeNode implements ToXContentObject {
     public static final ParseField NODE_INDEX = new ParseField("node_index");
     public static final ParseField SPLIT_GAIN = new ParseField("split_gain");
     public static final ParseField LEAF_VALUE = new ParseField("leaf_value");
+    public static final ParseField NUMBER_SAMPLES = new ParseField("number_samples");
 
 
     private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>(
@@ -61,6 +62,7 @@ public class TreeNode implements ToXContentObject {
         PARSER.declareInt(Builder::setNodeIndex, NODE_INDEX);
         PARSER.declareDouble(Builder::setSplitGain, SPLIT_GAIN);
         PARSER.declareDouble(Builder::setLeafValue, LEAF_VALUE);
+        PARSER.declareLong(Builder::setNumberSamples, NUMBER_SAMPLES);
     }
 
     public static Builder fromXContent(XContentParser parser) {
@@ -76,6 +78,7 @@ public static Builder fromXContent(XContentParser parser) {
     private final Boolean defaultLeft;
     private final Integer leftChild;
     private final Integer rightChild;
+    private final Long numberSamples;
 
 
     TreeNode(Operator operator,
@@ -86,7 +89,8 @@ public static Builder fromXContent(XContentParser parser) {
              Double leafValue,
              Boolean defaultLeft,
              Integer leftChild,
-             Integer rightChild) {
+             Integer rightChild,
+             Long numberSamples) {
         this.operator = operator;
         this.threshold  = threshold;
         this.splitFeature = splitFeature;
@@ -96,6 +100,7 @@ public static Builder fromXContent(XContentParser parser) {
         this.defaultLeft = defaultLeft;
         this.leftChild  = leftChild;
         this.rightChild = rightChild;
+        this.numberSamples = numberSamples;
     }
 
     public Operator getOperator() {
@@ -134,6 +139,10 @@ public Integer getRightChild() {
         return rightChild;
     }
 
+    public Long getNumberSamples() {
+        return numberSamples;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -146,6 +155,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         addOptionalField(builder, DEFAULT_LEFT, defaultLeft );
         addOptionalField(builder, LEFT_CHILD, leftChild);
         addOptionalField(builder, RIGHT_CHILD, rightChild);
+        addOptionalField(builder, NUMBER_SAMPLES, numberSamples);
         builder.endObject();
         return builder;
     }
@@ -169,7 +179,8 @@ public boolean equals(Object o) {
             && Objects.equals(leafValue, that.leafValue)
             && Objects.equals(defaultLeft, that.defaultLeft)
             && Objects.equals(leftChild, that.leftChild)
-            && Objects.equals(rightChild, that.rightChild);
+            && Objects.equals(rightChild, that.rightChild)
+            && Objects.equals(numberSamples, that.numberSamples);
     }
 
     @Override
@@ -182,7 +193,8 @@ public int hashCode() {
             leafValue,
             defaultLeft,
             leftChild,
-            rightChild);
+            rightChild,
+            numberSamples);
     }
 
     @Override
@@ -204,6 +216,7 @@ public static class Builder {
         private Boolean defaultLeft;
         private Integer leftChild;
         private Integer rightChild;
+        private Long numberSamples;
 
         public Builder(int nodeIndex) {
             this.nodeIndex = nodeIndex;
@@ -265,6 +278,11 @@ public Integer getRightChild() {
             return rightChild;
         }
 
+        public Builder setNumberSamples(Long numberSamples) {
+            this.numberSamples = numberSamples;
+            return this;
+        }
+
         public TreeNode build() {
             return new TreeNode(operator,
                 threshold, 
@@ -274,7 +292,8 @@ public TreeNode build() {
                 leafValue, 
                 defaultLeft, 
                 leftChild, 
-                rightChild);
+                rightChild,
+                numberSamples);
         }
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java

@@ -318,6 +318,7 @@ public static class ClusterPrivilegeName {
         public static final String MANAGE_OIDC = "manage_oidc";
         public static final String MANAGE_TOKEN = "manage_token";
         public static final String MANAGE_PIPELINE = "manage_pipeline";
+        public static final String MANAGE_AUTOSCALING = "manage_autoscaling";
         public static final String MANAGE_CCR = "manage_ccr";
         public static final String READ_CCR = "read_ccr";
         public static final String MANAGE_ILM = "manage_ilm";
@@ -326,8 +327,8 @@ public static class ClusterPrivilegeName {
         public static final String[] ALL_ARRAY = new String[] { NONE, ALL, MONITOR, MONITOR_TRANSFORM_DEPRECATED, MONITOR_TRANSFORM,
             MONITOR_ML, MONITOR_WATCHER, MONITOR_ROLLUP, MANAGE, MANAGE_TRANSFORM_DEPRECATED, MANAGE_TRANSFORM,
             MANAGE_ML, MANAGE_WATCHER, MANAGE_ROLLUP, MANAGE_INDEX_TEMPLATES, MANAGE_INGEST_PIPELINES, TRANSPORT_CLIENT,
-                MANAGE_SECURITY, MANAGE_SAML, MANAGE_OIDC, MANAGE_TOKEN, MANAGE_PIPELINE, MANAGE_CCR, READ_CCR, MANAGE_ILM, READ_ILM,
-                MANAGE_ENRICH };
+            MANAGE_SECURITY, MANAGE_SAML, MANAGE_OIDC, MANAGE_TOKEN, MANAGE_PIPELINE, MANAGE_AUTOSCALING, MANAGE_CCR, READ_CCR,
+            MANAGE_ILM, READ_ILM, MANAGE_ENRICH };
     }
 
     /**

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/tree/TreeNode.java

@@ -86,7 +86,7 @@ public static Ensemble createRandom(TargetType targetType) {
                 .toArray() :
             null;
 
-        return new Ensemble(featureNames,
+        return new Ensemble(randomBoolean() ? featureNames : Collections.emptyList(),
             models,
             outputAggregator,
             targetType,

client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java

@@ -49,6 +49,7 @@ public static TreeNode createRandomLeafNode(double internalValue) {
         return TreeNode.builder(randomInt(100))
             .setDefaultLeft(randomBoolean() ? null : randomBoolean())
             .setLeafValue(internalValue)
+            .setNumberSamples(randomNonNegativeLong())
             .build();
     }
 
@@ -63,6 +64,7 @@ public static TreeNode.Builder createRandom(int nodeIndex,
             .setDefaultLeft(randomBoolean() ? null : randomBoolean())
             .setLeftChild(left)
             .setRightChild(right)
+            .setNumberSamples(randomBoolean() ? null : randomNonNegativeLong())
             .setThreshold(threshold)
             .setOperator(operator)
             .setSplitFeature(featureIndex)

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/EnsembleTests.java

@@ -44,6 +44,11 @@ task createPluginsDir(type: EmptyDirTask) {
   dir = "${pluginsDir}"
   dirMode = 0755
 }
+ext.jvmOptionsDir = new File(buildDir, 'jvm-options-hack/jvm.options.d')
+task createJvmOptionsDir(type: EmptyDirTask) {
+  dir = "${jvmOptionsDir}"
+  dirMode = 0750
+}
 
 CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
   return copySpec {
@@ -55,6 +60,10 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
         dirMode 0750
         fileMode 0660
         with configFiles(distributionType, oss, jdk)
+        from {
+          dirMode 0750
+          jvmOptionsDir.getParent()
+        }
       }
       into('bin') {
         with binFiles(distributionType, oss, jdk)
@@ -94,7 +103,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
 
 // common config across all zip/tar
 tasks.withType(AbstractArchiveTask) {
-  dependsOn createLogsDir, createPluginsDir
+  dependsOn createLogsDir, createPluginsDir, createJvmOptionsDir
   String subdir = it.name.substring('build'.size()).replaceAll(/[A-Z]/) { '-' + it.toLowerCase() }.substring(1)
   destinationDir = file("${subdir}/build/distributions")
   baseName = "elasticsearch${subdir.contains('oss') ? '-oss' : ''}"

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/tree/TreeNodeTests.java

@@ -30,8 +30,8 @@ ${source_elasticsearch}
 RUN tar zxf /opt/${elasticsearch} --strip-components=1
 RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env \
     && sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
-RUN mkdir -p config data logs
-RUN chmod 0775 config data logs
+RUN mkdir -p config config/jvm.options.d data logs
+RUN chmod 0775 config config/jvm.options.d data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/
 RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties