Merge branch master into zen2

2 types of conflicts during the merge:
1) Line length fix
2) Classes no longer extend AbstractComponent

Author: Andrey Ershov

TESTING.asciidoc

@@ -150,8 +150,9 @@ Default value provided below in [brackets].
 
 === Load balancing and caches.
 
-By default the tests run on up to 4 JVMs based on the number of cores. If you
-want to explicitly specify the number of JVMs you can do so on the command
+By default the tests run on multiple processes using all the available cores on all 
+available CPUs. Not including hyper-threading.
+If you want to explicitly specify the number of JVMs you can do so on the command
 line:
 
 ----------------------------

build.gradle

@@ -25,6 +25,9 @@ import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
+import com.carrotsearch.gradle.junit4.RandomizedTestingTask
+
+import java.util.function.Predicate
 
 plugins {
     id 'com.gradle.build-scan' version '1.13.2'
@@ -622,3 +625,19 @@ allprojects {
     }
   }
 }
+
+allprojects {
+  task checkPart1
+  task checkPart2 
+  tasks.matching { it.name == "check" }.all { check ->
+    if (check.path.startsWith(":x-pack:")) {
+      checkPart2.dependsOn check
+    } else {
+      checkPart1.dependsOn check
+    }
+  } 
+}
+
+
+
+

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -773,9 +773,32 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     static void applyCommonTestConfig(Project project) {
+        String defaultParallel = 'auto'
+        // Count physical cores on any Linux distro ( don't count hyper-threading )
+        if (project.file("/proc/cpuinfo").exists()) {
+            Map<String, Integer> socketToCore = [:]
+            String currentID = ""
+            project.file("/proc/cpuinfo").readLines().forEach({ line ->
+                if (line.contains(":")) {
+                    List<String> parts = line.split(":", 2).collect({it.trim()})
+                    String name = parts[0], value = parts[1]
+                    // the ID of the CPU socket
+                    if (name == "physical id") {
+                        currentID = value
+                    }
+                    // Number  of cores not including hyper-threading
+                    if (name == "cpu cores") {
+                        assert currentID.isEmpty() == false
+                        socketToCore[currentID] = Integer.valueOf(value)
+                        currentID = ""
+                    }
+                }
+            })
+            defaultParallel = socketToCore.values().sum().toString();
+        }
         project.tasks.withType(RandomizedTestingTask) {
             jvm "${project.runtimeJavaHome}/bin/java"
-            parallelism System.getProperty('tests.jvms', 'auto')
+            parallelism System.getProperty('tests.jvms', defaultParallel)
             ifNoTests System.getProperty('tests.ifNoTests', 'fail')
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -24,7 +24,9 @@ import org.apache.rat.license.SimpleLicenseFamily
 import org.elasticsearch.gradle.AntTask
 import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.SkipWhenEmpty
 import org.gradle.api.tasks.SourceSet
 
 import java.nio.file.Files
@@ -39,12 +41,6 @@ public class LicenseHeadersTask extends AntTask {
     @OutputFile
     File reportFile = new File(project.buildDir, 'reports/licenseHeaders/rat.log')
 
-    /**
-     * The list of java files to check. protected so the afterEvaluate closure in the
-     * constructor can write to it.
-     */
-    protected List<FileCollection> javaFiles
-
     /** Allowed license families for this project. */
     @Input
     List<String> approvedLicenses = ['Apache', 'Generated']
@@ -65,11 +61,16 @@ public class LicenseHeadersTask extends AntTask {
 
     LicenseHeadersTask() {
         description = "Checks sources for missing, incorrect, or unacceptable license headers"
-        // Delay resolving the dependencies until after evaluation so we pick up generated sources
-        project.afterEvaluate {
-            javaFiles = project.sourceSets.collect({it.allJava})
-            inputs.files(javaFiles)
-        }
+    }
+
+    /**
+     * The list of java files to check. protected so the afterEvaluate closure in the
+     * constructor can write to it.
+     */
+    @InputFiles
+    @SkipWhenEmpty
+    public List<FileCollection> getJavaFiles() {
+        return project.sourceSets.collect({it.allJava})
     }
 
     /**
@@ -97,9 +98,8 @@ public class LicenseHeadersTask extends AntTask {
         Files.deleteIfExists(reportFile.toPath())
 
         // run rat, going to the file
-        List<FileCollection> input = javaFiles
         ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
-            for (FileCollection dirSet : input) {
+            for (FileCollection dirSet : javaFiles) {
                for (File dir: dirSet.srcDirs) {
                    // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
                    if (dir.exists()) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -69,7 +69,7 @@ class ClusterConfiguration {
      */
     @Input
     Closure<Integer> minimumMasterNodes = {
-        if (bwcVersion != null && bwcVersion.before("6.5.0-SNAPSHOT")) {
+        if (bwcVersion != null && bwcVersion.before("6.5.0")) {
             return numNodes > 1 ? numNodes : -1
         } else {
             return numNodes > 1 ? numNodes.intdiv(2) + 1 : -1

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -127,7 +127,7 @@ class ClusterFormationTasks {
             nodes.add(node)
             Closure<Map> writeConfigSetup
             Object dependsOn
-            if (node.nodeVersion.onOrAfter("6.5.0-SNAPSHOT")) {
+            if (node.nodeVersion.onOrAfter("6.5.0")) {
                 writeConfigSetup = { Map esConfig ->
                     // Don't force discovery provider if one is set by the test cluster specs already
                     if (esConfig.containsKey('discovery.zen.hosts_provider') == false) {
@@ -140,7 +140,7 @@ class ClusterFormationTasks {
             } else {
                 dependsOn = startTasks.empty ? startDependencies : startTasks.get(0)
                 writeConfigSetup = { Map esConfig ->
-                    String unicastTransportUri = node.config.unicastTransportUri(nodes.get(0), node, project.ant)
+                    String unicastTransportUri = node.config.unicastTransportUri(nodes.get(0), node, project.createAntBuilder())
                     if (unicastTransportUri == null) {
                         esConfig['discovery.zen.ping.unicast.hosts'] = []
                     } else {
@@ -717,7 +717,7 @@ class ClusterFormationTasks {
             Collection<String> unicastHosts = new HashSet<>()
             nodes.forEach { node ->
                 unicastHosts.addAll(node.config.otherUnicastHostAddresses.call())
-                String unicastHost = node.config.unicastTransportUri(node, null, project.ant)
+                String unicastHost = node.config.unicastTransportUri(node, null, project.createAntBuilder())
                 if (unicastHost != null) {
                     unicastHosts.add(unicastHost)
                 }
@@ -913,9 +913,10 @@ class ClusterFormationTasks {
                 outputPrintStream: outputStream,
                 messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO)
 
-        project.ant.project.addBuildListener(listener)
-        Object retVal = command(project.ant)
-        project.ant.project.removeBuildListener(listener)
+        AntBuilder ant = project.createAntBuilder()
+        ant.project.addBuildListener(listener)
+        Object retVal = command(ant)
+        ant.project.removeBuildListener(listener)
         return retVal
     }