Merge branch 'master' into feature-ml-data-frame-analytics

Author: dimitris-athanasiou

.ci/matrix-build-javas.yml

@@ -7,3 +7,4 @@
 
 ES_BUILD_JAVA:
   - java11
+  - openjdk12

.ci/matrix-runtime-javas.yml

@@ -9,5 +9,6 @@ ES_RUNTIME_JAVA:
   - java8
   - java8fips
   - java11
+  - openjdk12
   - zulu8
   - zulu11

benchmarks/build.gradle

@@ -24,7 +24,7 @@ mainClassName = 'org.openjdk.jmh.Main'
 assemble.enabled = false
 archivesBaseName = 'elasticsearch-benchmarks'
 
-test.enabled = false
+unitTest.enabled = false
 
 dependencies {
     compile("org.elasticsearch:elasticsearch:${version}") {
@@ -53,7 +53,7 @@ forbiddenApisMain.enabled = false
 dependencyLicenses.enabled = false
 dependenciesInfo.enabled = false
 
-thirdPartyAudit.excludes = [
+thirdPartyAudit.ignoreViolations (
         // these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees)
         'org.openjdk.jmh.profile.AbstractHotspotProfiler',
         'org.openjdk.jmh.profile.HotspotThreadProfiler',
@@ -62,4 +62,4 @@ thirdPartyAudit.excludes = [
         'org.openjdk.jmh.profile.HotspotMemoryProfiler',
         'org.openjdk.jmh.profile.HotspotRuntimeProfiler',
         'org.openjdk.jmh.util.Utils'
-]
+)

benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterBenchmark.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark.time;
+
+import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.time.DateFormatter;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.time.temporal.TemporalAccessor;
+import java.util.concurrent.TimeUnit;
+
+@Fork(3)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Benchmark)
+@SuppressWarnings("unused") //invoked by benchmarking framework
+public class DateFormatterBenchmark {
+
+    private final DateFormatter javaFormatter = DateFormatter.forPattern("8year_month_day||ordinal_date||epoch_millis");
+    private final DateFormatter jodaFormatter = Joda.forPattern("year_month_day||ordinal_date||epoch_millis");
+
+    @Benchmark
+    public TemporalAccessor parseJavaDate() {
+        return javaFormatter.parse("1234567890");
+    }
+
+    @Benchmark
+    public TemporalAccessor parseJodaDate() {
+        return jodaFormatter.parse("1234567890");
+    }
+}
+

build.gradle

@@ -24,13 +24,9 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
-import org.gradle.api.tasks.options.Option
-import  org.gradle.util.GradleVersion
-import  org.gradle.util.DistributionLocator
+import org.gradle.util.GradleVersion
+import org.gradle.util.DistributionLocator
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
-import com.carrotsearch.gradle.junit4.RandomizedTestingTask
-
-import java.util.function.Predicate
 
 plugins {
     id 'com.gradle.build-scan' version '2.0.2'
@@ -433,7 +429,7 @@ tasks.idea.doLast {
   ideaMarker.setText('', 'UTF-8')
 }
 if (System.getProperty('idea.active') != null && ideaMarker.exists() == false) {
-  throw new GradleException('You must run gradle idea from the root of elasticsearch before importing into IntelliJ')
+  throw new GradleException('You must run `./gradlew idea` from the root of elasticsearch before importing into IntelliJ')
 }
 
 // eclipse configuration

buildSrc/build.gradle

@@ -16,7 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import java.nio.file.Files
 import org.gradle.util.GradleVersion
 
 plugins {
@@ -179,9 +178,7 @@ if (project != rootProject) {
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
-  test {
-    include "**/*Tests.class"
-    exclude "**/*IT.class"
+  unitTest {
     // The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test
     // with compiler instead on the ones that are too old.
     if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_10) {
@@ -226,6 +223,18 @@ if (project != rootProject) {
     integTestClass = 'org.elasticsearch.gradle.test.GradleIntegrationTestCase'
   }
 
+  testingConventions {
+    naming.clear()
+    naming {
+      Tests {
+        baseClass 'org.elasticsearch.gradle.test.GradleUnitTestCase'
+      }
+      IT {
+        baseClass 'org.elasticsearch.gradle.test.GradleIntegrationTestCase'
+      }
+    }
+  }
+
   /*
    * We alread configure publication and we don't need or want this one that
    * comes from the java-gradle-plugin.

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy

@@ -4,30 +4,14 @@ import com.carrotsearch.ant.tasks.junit4.JUnit4
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
-import org.gradle.api.UnknownTaskException
-import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.tasks.TaskContainer
-import org.gradle.api.tasks.TaskProvider
-import org.gradle.api.tasks.testing.Test
 
 class RandomizedTestingPlugin implements Plugin<Project> {
 
     void apply(Project project) {
         setupSeed(project)
-        replaceTestTask(project.tasks)
+        createUnitTestTask(project.tasks)
         configureAnt(project.ant)
-        configureSanityCheck(project)
-    }
-
-    private static void configureSanityCheck(Project project) {
-        // Check the task graph to confirm tasks were indeed replaced
-        // https://github.com/elastic/elasticsearch/issues/31324
-        project.rootProject.getGradle().getTaskGraph().whenReady {
-            Task test = project.getTasks().findByName("test")
-            if (test != null && (test instanceof RandomizedTestingTask) == false) {
-                throw new IllegalStateException("Test task was not replaced in project ${project.path}. Found ${test.getClass()}")
-            }
-        }
     }
 
     /**
@@ -57,35 +41,15 @@ class RandomizedTestingPlugin implements Plugin<Project> {
         }
     }
 
-    static void replaceTestTask(TaskContainer tasks) {
-        // Gradle 4.8 introduced lazy tasks, thus we deal both with the `test` task as well as it's provider
-        // https://github.com/gradle/gradle/issues/5730#issuecomment-398822153
-        // since we can't be sure if the task was ever realized, we remove both the provider and the task
-        TaskProvider<Test> oldTestProvider
-        try {
-            oldTestProvider = tasks.named('test')
-        } catch (UnknownTaskException unused) {
-            // no test task, ok, user will use testing task on their own
-            return
+    static void createUnitTestTask(TaskContainer tasks) {
+        // only create a unitTest task if the `test` task exists as some project don't make use of it.
+        tasks.matching { it.name == "test" }.all {
+            // We don't want to run any tests with the Gradle test runner since we add our own randomized runner
+            it.enabled = false
+            RandomizedTestingTask unitTest = tasks.create('unitTest', RandomizedTestingTask)
+            unitTest.description = 'Runs unit tests with the randomized testing framework'
+            it.dependsOn unitTest
         }
-        Test oldTestTask = oldTestProvider.get()
-
-        // we still have to use replace here despite the remove above because the task container knows about the provider
-        // by the same name
-        RandomizedTestingTask newTestTask = tasks.replace('test', RandomizedTestingTask)
-        newTestTask.configure{
-            group =  JavaBasePlugin.VERIFICATION_GROUP
-            description = 'Runs unit tests with the randomized testing framework'
-            dependsOn oldTestTask.dependsOn, 'testClasses'
-            classpath = oldTestTask.classpath
-            testClassesDirs = oldTestTask.project.sourceSets.test.output.classesDirs
-        }
-
-        // hack so check task depends on custom test
-        Task checkTask = tasks.getByName('check')
-        checkTask.dependsOn.remove(oldTestProvider)
-        checkTask.dependsOn.remove(oldTestTask)
-        checkTask.dependsOn.add(newTestTask)
     }
 
     static void configureAnt(AntBuilder ant) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -40,6 +40,7 @@ import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.execution.TaskExecutionGraph
+import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.JavaPlugin
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
@@ -292,18 +293,7 @@ class BuildPlugin implements Plugin<Project> {
                                 it.standardOutput = dockerVersionOutput
                             })
                     final String dockerVersion = dockerVersionOutput.toString().trim()
-                    final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-ce)?, build [0-9a-f]{7}/
-                    assert matcher.matches() : dockerVersion
-                    final dockerMajorMinorVersion = matcher.group(1)
-                    final String[] majorMinor = dockerMajorMinorVersion.split("\\.")
-                    if (Integer.parseInt(majorMinor[0]) < 17
-                            || (Integer.parseInt(majorMinor[0]) == 17 && Integer.parseInt(majorMinor[1]) < 5)) {
-                        final String message = String.format(
-                                Locale.ROOT,
-                                "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]",
-                                dockerVersion)
-                        throwDockerRequiredException(message)
-                    }
+                    checkDockerVersionRecent(dockerVersion)
 
                     final ByteArrayOutputStream dockerImagesErrorOutput = new ByteArrayOutputStream()
                     // the Docker binary executes, check that we can execute a privileged command
@@ -338,6 +328,21 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
+    protected static void checkDockerVersionRecent(String dockerVersion) {
+        final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-ce)?, build [0-9a-f]{7,40}/
+        assert matcher.matches(): dockerVersion
+        final dockerMajorMinorVersion = matcher.group(1)
+        final String[] majorMinor = dockerMajorMinorVersion.split("\\.")
+        if (Integer.parseInt(majorMinor[0]) < 17
+                || (Integer.parseInt(majorMinor[0]) == 17 && Integer.parseInt(majorMinor[1]) < 5)) {
+            final String message = String.format(
+                    Locale.ROOT,
+                    "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]",
+                    dockerVersion)
+            throwDockerRequiredException(message)
+        }
+    }
+
     private static void throwDockerRequiredException(final String message) {
         throw new GradleException(
                 message + "\nyou can address this by attending to the reported issue, "
@@ -888,15 +893,22 @@ class BuildPlugin implements Plugin<Project> {
             parallelism System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel)
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
+            project.sourceSets.matching { it.name == "test" }.all { test ->
+                task.testClassesDirs = test.output.classesDirs
+                task.classpath = test.runtimeClasspath
+            }
+            group =  JavaBasePlugin.VERIFICATION_GROUP
+            dependsOn 'testClasses'
 
             // Make sure all test tasks are configured properly
             if (name != "test") {
                 project.tasks.matching { it.name == "test"}.all { testTask ->
-                    task.testClassesDirs = testTask.testClassesDirs
-                    task.classpath = testTask.classpath
                     task.shouldRunAfter testTask
                 }
             }
+            if (name == "unitTest") {
+                include("**/*Tests.class")
+            }
 
             // TODO: why are we not passing maxmemory to junit4?
             jvmArg '-Xmx' + System.getProperty('tests.heap.size', '512m')
@@ -986,8 +998,6 @@ class BuildPlugin implements Plugin<Project> {
 
             exclude '**/*$*.class'
 
-            dependsOn(project.tasks.testClasses)
-
             project.plugins.withType(ShadowPlugin).whenPluginAdded {
                 // Test against a shadow jar if we made one
                 classpath -= project.tasks.compileJava.outputs.files

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -78,6 +78,19 @@ public class PluginBuildPlugin extends BuildPlugin {
                 skipIntegTestInDisguise = true
             }
         }
+        project.testingConventions {
+            naming.clear()
+            naming {
+                Tests {
+                    baseClass 'org.apache.lucene.util.LuceneTestCase'
+                }
+                IT {
+                    baseClass 'org.elasticsearch.test.ESIntegTestCase'
+                    baseClass 'org.elasticsearch.test.rest.ESRestTestCase'
+                    baseClass 'org.elasticsearch.test.ESSingleNodeTestCase'
+                }
+            }
+        }
         createIntegTestTask(project)
         createBundleTask(project)
         project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime'))
@@ -101,7 +114,7 @@ public class PluginBuildPlugin extends BuildPlugin {
                 generatePOMTask.ext.pomFileName = "${project.archivesBaseName}-client-${project.versions.elasticsearch}.pom"
             }
         } else {
-            project.plugins.withType(MavenPublishPlugin).whenPluginAdded {
+            if (project.plugins.hasPlugin(MavenPublishPlugin)) {
                 project.publishing.publications.nebula(MavenPublication).artifactId(
                         project.pluginProperties.extension.name
                 )

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -91,7 +91,17 @@ class PrecommitTasks {
     }
 
     static Task configureTestingConventions(Project project) {
-        project.getTasks().create("testingConventions", TestingConventionsTasks.class)
+        TestingConventionsTasks task = project.getTasks().create("testingConventions", TestingConventionsTasks.class)
+        task.naming {
+            Tests {
+                baseClass "org.apache.lucene.util.LuceneTestCase"
+            }
+            IT {
+                baseClass "org.elasticsearch.test.ESIntegTestCase"
+                baseClass 'org.elasticsearch.test.rest.ESRestTestCase'
+            }
+        }
+        return task
     }
 
     private static Task configureJarHell(Project project) {
@@ -123,8 +133,16 @@ class PrecommitTasks {
         project.tasks.withType(CheckForbiddenApis) {
             dependsOn(buildResources)
             targetCompatibility = project.runtimeJavaVersion >= JavaVersion.VERSION_1_9 ?
-                    project.runtimeJavaVersion.getMajorVersion() :
-                    project.runtimeJavaVersion
+                    project.runtimeJavaVersion.getMajorVersion() : project.runtimeJavaVersion
+            if (project.runtimeJavaVersion > JavaVersion.VERSION_11) {
+                doLast {
+                    project.logger.info(
+                            "Forbidden APIs does not support java version past 11. Will use the signatures from 11 for ",
+                            project.runtimeJavaVersion
+                    )
+                }
+                targetCompatibility = JavaVersion.VERSION_11.getMajorVersion()
+            }
             bundledSignatures = [
                     "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"
             ]

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -55,8 +55,6 @@ public class RestIntegTestTask extends DefaultTask {
         super.dependsOn(runner)
         clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses)
         runner.dependsOn(clusterInit)
-        runner.classpath = project.sourceSets.test.runtimeClasspath
-        runner.testClassesDirs = project.sourceSets.test.output.classesDirs
         clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project)
 
         // override/add more for rest tests