Merge remote-tracking branch 'elastic/master' into use-plugins-for-node-tool

Author: ywelsch

CONTRIBUTING.md

@@ -107,7 +107,9 @@ and `JAVA11_HOME` available so that the tests can pass.
 `jrunscript` for jdk distributions.
 
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
-using the wrapper via the `gradlew` script in the root of the repository.
+using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat` 
+script on Windows in the root of the repository. The examples below show the
+usage on Unix.
 
 We support development in the Eclipse and IntelliJ IDEs.
 For Eclipse, the minimum version that we support is [4.13][eclipse].

buildSrc/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 
+import org.gradle.internal.jvm.Jvm
 import org.gradle.util.GradleVersion
 
 plugins {
@@ -168,6 +169,9 @@ if (project != rootProject) {
   forbiddenApisTest.enabled = false
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
+  if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))){
+    test.enabled = false
+  }
 
   configurations {
     distribution
@@ -223,8 +227,13 @@ if (project != rootProject) {
   }
   check.dependsOn(integTest)
 
+  // for now we hardcode the tests for our build to use the gradle jvm.
+  tasks.withType(Test).configureEach {
+    it.executable = Jvm.current().getJavaExecutable()
+  }
+
   /*
-   * We alread configure publication and we don't need or want this one that
+   * We already configure publication and we don't need or want this one that
    * comes from the java-gradle-plugin.
    */
   afterEvaluate {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -720,6 +720,9 @@ class BuildPlugin implements Plugin<Project> {
                 // TODO: remove this once ctx isn't added to update script params in 7.0
                 test.systemProperty 'es.scripting.update.ctx_in_params', 'false'
 
+                // TODO: remove this property in 8.0
+                test.systemProperty 'es.search.rewrite_sort', 'true'
+
                 // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
                 test.systemProperty 'es.transport.cname_in_publish_address', 'true'
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -66,6 +66,8 @@ class SnippetsTask extends DefaultTask {
         exclude 'build.gradle'
         // That is where the snippets go, not where they come from!
         exclude 'build'
+        exclude 'build-idea'
+        exclude 'build-eclipse'
     }
 
     /**

buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java

@@ -41,14 +41,12 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
     private final RegularFileProperty outputFile;
     private final RegularFileProperty compilerVersionFile;
     private final RegularFileProperty runtimeVersionFile;
-    private final RegularFileProperty fipsJvmFile;
 
     @Inject
     public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) {
         this.outputFile = objectFactory.fileProperty();
         this.compilerVersionFile = objectFactory.fileProperty();
         this.runtimeVersionFile = objectFactory.fileProperty();
-        this.fipsJvmFile = objectFactory.fileProperty();
     }
 
     @Input
@@ -113,11 +111,6 @@ public RegularFileProperty getRuntimeVersionFile() {
         return runtimeVersionFile;
     }
 
-    @OutputFile
-    public RegularFileProperty getFipsJvmFile() {
-        return fipsJvmFile;
-    }
-
     @TaskAction
     public void generate() {
         String javaVendorVersion = System.getProperty("java.vendor.version", System.getProperty("java.vendor"));
@@ -130,7 +123,6 @@ public void generate() {
         String runtimeJavaVersionDetails = gradleJavaVersionDetails;
         JavaVersion runtimeJavaVersionEnum = JavaVersion.current();
         File gradleJavaHome = Jvm.current().getJavaHome();
-        boolean inFipsJvm = false;
 
         try {
             if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
@@ -146,8 +138,6 @@ public void generate() {
                 if (runtimeJavaHome.exists()) {
                     runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome);
                     runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome));
-
-                    inFipsJvm = Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
                 } else {
                     throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist");
                 }
@@ -213,7 +203,6 @@ public void generate() {
 
         writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name());
         writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name());
-        writeToFile(fipsJvmFile.getAsFile().get(), Boolean.toString(inFipsJvm));
     }
 
     private void writeToFile(File file, String content) {

buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -76,14 +76,12 @@ public void apply(Project project) {
                 task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info"));
                 task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version"));
                 task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version"));
-                task.getFipsJvmFile().set(new File(project.getBuildDir(), "in-fips-jvm"));
             });
 
         PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> {
             task.getBuildInfoFile().set(generateTask.getOutputFile());
             task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile());
             task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile());
-            task.getFipsJvmFile().set(generateTask.getFipsJvmFile());
             task.setGlobalInfoListeners(extension.listeners);
         });
 
@@ -103,6 +101,7 @@ public void apply(Project project) {
             params.setIsCi(System.getenv("JENKINS_URL") != null);
             params.setIsInternal(GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null);
             params.setDefaultParallel(findDefaultParallel(project));
+            params.setInFipsJvm(isInFipsJvm());
         });
 
         project.allprojects(p -> {
@@ -153,6 +152,10 @@ private static String getJavaHomeEnvVarName(String version) {
         return "JAVA" + version + "_HOME";
     }
 
+    private static boolean isInFipsJvm() {
+        return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
+    }
+
     private static String getResourceContents(String resourcePath) {
         try (BufferedReader reader = new BufferedReader(
             new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath))

buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java

@@ -16,15 +16,13 @@ public class PrintGlobalBuildInfoTask extends DefaultTask {
     private final RegularFileProperty buildInfoFile;
     private final RegularFileProperty compilerVersionFile;
     private final RegularFileProperty runtimeVersionFile;
-    private final RegularFileProperty fipsJvmFile;
     private List<Runnable> globalInfoListeners = new ArrayList<>();
 
     @Inject
     public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) {
         this.buildInfoFile = objectFactory.fileProperty();
         this.compilerVersionFile = objectFactory.fileProperty();
         this.runtimeVersionFile = objectFactory.fileProperty();
-        this.fipsJvmFile = objectFactory.fileProperty();
     }
 
     @InputFile
@@ -42,11 +40,6 @@ public RegularFileProperty getRuntimeVersionFile() {
         return runtimeVersionFile;
     }
 
-    @InputFile
-    public RegularFileProperty getFipsJvmFile() {
-        return fipsJvmFile;
-    }
-
     public void setGlobalInfoListeners(List<Runnable> globalInfoListeners) {
         this.globalInfoListeners = globalInfoListeners;
     }
@@ -57,6 +50,7 @@ public void print() {
         getLogger().quiet("Elasticsearch Build Hamster says Hello!");
         getLogger().quiet(getFileText(getBuildInfoFile()).asString());
         getLogger().quiet("  Random Testing Seed   : " + BuildParams.getTestSeed());
+        getLogger().quiet("  In FIPS 140 mode      : " + BuildParams.isInFipsJvm());
         getLogger().quiet("=======================================");
 
         setGlobalProperties();
@@ -76,7 +70,6 @@ private void setGlobalProperties() {
         BuildParams.init(params -> {
             params.setCompilerJavaVersion(JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString()));
             params.setRuntimeJavaVersion(JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString()));
-            params.setInFipsJvm(Boolean.parseBoolean(getFileText(getFipsJvmFile()).asString()));
         });
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -415,6 +415,11 @@ public synchronized void start() {
         } catch (IOException e) {
             throw new UncheckedIOException("Failed to create working directory for " + this, e);
         }
+
+        copyExtraJars();
+
+        copyExtraConfigFiles();
+
         createConfiguration();
 
         if (plugins.isEmpty() == false) {
@@ -453,10 +458,6 @@ public synchronized void start() {
 
         installModules();
 
-        copyExtraConfigFiles();
-
-        copyExtraJars();
-
         if (isSettingTrue("xpack.security.enabled")) {
             if (credentials.isEmpty()) {
                 user(Collections.emptyMap());
@@ -677,6 +678,10 @@ private Map<String, String> getESEnvironment() {
         if (systemProperties.isEmpty() == false) {
             systemPropertiesString = " " + systemProperties.entrySet().stream()
                 .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
+                // ES_PATH_CONF is also set as an environment variable and for a reference to ${ES_PATH_CONF}
+                // to work ES_JAVA_OPTS, we need to make sure that ES_PATH_CONF before ES_JAVA_OPTS. Instead,
+                // we replace the reference with the actual value in other environment variables
+                .map(p -> p.replace("${ES_PATH_CONF}", configFile.getParent().toString()))
                 .collect(Collectors.joining(" "));
         }
         String jvmArgsString = "";

buildSrc/version.properties

@@ -29,9 +29,8 @@ joda              = 2.10.4
 #  - distribution/tools/plugin-cli
 #  - x-pack/plugin/security
 bouncycastle      = 1.61
-
 # test dependencies
-randomizedrunner  = 2.7.1
+randomizedrunner  = 2.7.4
 junit             = 4.12
 httpclient        = 4.5.10
 httpcore          = 4.4.12

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -29,6 +29,8 @@
 import org.elasticsearch.client.RequestConverters.EndpointBuilder;
 import org.elasticsearch.client.core.PageParams;
 import org.elasticsearch.client.ml.CloseJobRequest;
+import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
+import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
 import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
 import org.elasticsearch.client.ml.DeleteCalendarRequest;
@@ -58,6 +60,8 @@
 import org.elasticsearch.client.ml.GetModelSnapshotsRequest;
 import org.elasticsearch.client.ml.GetOverallBucketsRequest;
 import org.elasticsearch.client.ml.GetRecordsRequest;
+import org.elasticsearch.client.ml.GetTrainedModelsRequest;
+import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest;
 import org.elasticsearch.client.ml.MlInfoRequest;
 import org.elasticsearch.client.ml.OpenJobRequest;
 import org.elasticsearch.client.ml.PostCalendarEventRequest;
@@ -700,15 +704,85 @@ static Request evaluateDataFrame(EvaluateDataFrameRequest evaluateRequest) throw
         return request;
     }
 
-    static Request estimateMemoryUsage(PutDataFrameAnalyticsRequest estimateRequest) throws IOException {
+    static Request explainDataFrameAnalytics(ExplainDataFrameAnalyticsRequest explainRequest) throws IOException {
+        EndpointBuilder endpoint = new EndpointBuilder().addPathPartAsIs("_ml", "data_frame", "analytics");
+        if (explainRequest.getId() != null) {
+            endpoint.addPathPart(explainRequest.getId());
+        }
+        endpoint.addPathPartAsIs("_explain");
+
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint.build());
+        if (explainRequest.getConfig() != null) {
+            request.setEntity(createEntity(explainRequest.getConfig(), REQUEST_BODY_CONTENT_TYPE));
+        }
+        return request;
+    }
+
+    static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) {
         String endpoint = new EndpointBuilder()
-            .addPathPartAsIs("_ml", "data_frame", "analytics", "_estimate_memory_usage")
+            .addPathPartAsIs("_ml", "inference")
+            .addPathPart(Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getIds()))
             .build();
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        request.setEntity(createEntity(estimateRequest, REQUEST_BODY_CONTENT_TYPE));
+        RequestConverters.Params params = new RequestConverters.Params();
+        if (getTrainedModelsRequest.getPageParams() != null) {
+            PageParams pageParams = getTrainedModelsRequest.getPageParams();
+            if (pageParams.getFrom() != null) {
+                params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString());
+            }
+            if (pageParams.getSize() != null) {
+                params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString());
+            }
+        }
+        if (getTrainedModelsRequest.getAllowNoMatch() != null) {
+            params.putParam(GetTrainedModelsRequest.ALLOW_NO_MATCH,
+                Boolean.toString(getTrainedModelsRequest.getAllowNoMatch()));
+        }
+        if (getTrainedModelsRequest.getDecompressDefinition() != null) {
+            params.putParam(GetTrainedModelsRequest.DECOMPRESS_DEFINITION,
+                Boolean.toString(getTrainedModelsRequest.getDecompressDefinition()));
+        }
+        if (getTrainedModelsRequest.getIncludeDefinition() != null) {
+            params.putParam(GetTrainedModelsRequest.INCLUDE_MODEL_DEFINITION,
+                Boolean.toString(getTrainedModelsRequest.getIncludeDefinition()));
+        }
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        request.addParameters(params.asMap());
+        return request;
+    }
+
+    static Request getTrainedModelsStats(GetTrainedModelsStatsRequest getTrainedModelsStatsRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "inference")
+            .addPathPart(Strings.collectionToCommaDelimitedString(getTrainedModelsStatsRequest.getIds()))
+            .addPathPart("_stats")
+            .build();
+        RequestConverters.Params params = new RequestConverters.Params();
+        if (getTrainedModelsStatsRequest.getPageParams() != null) {
+            PageParams pageParams = getTrainedModelsStatsRequest.getPageParams();
+            if (pageParams.getFrom() != null) {
+                params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString());
+            }
+            if (pageParams.getSize() != null) {
+                params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString());
+            }
+        }
+        if (getTrainedModelsStatsRequest.getAllowNoMatch() != null) {
+            params.putParam(GetTrainedModelsStatsRequest.ALLOW_NO_MATCH,
+                Boolean.toString(getTrainedModelsStatsRequest.getAllowNoMatch()));
+        }
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        request.addParameters(params.asMap());
         return request;
     }
 
+    static Request deleteTrainedModel(DeleteTrainedModelRequest deleteRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "inference")
+            .addPathPart(deleteRequest.getId())
+            .build();
+        return new Request(HttpDelete.METHOD_NAME, endpoint);
+    }
+
     static Request putFilter(PutFilterRequest putFilterRequest) throws IOException {
         String endpoint = new EndpointBuilder()
             .addPathPartAsIs("_ml")