Merge branch 'master' into ccr

* master:
  [TEST] REST client request without leading '/' (#29471)
  Using ObjectParser in UpdateRequest (#29293)
  Prevent accidental changes of default values (#29528)
  [Docs] Add definitions to glossary  (#29127)
  Avoid self-deadlock in the translog (#29520)
  Minor cleanup in NodeInfo.groovy
  Lazy configure build tasks that require older JDKs (#29519)
  Simplify snapshot check in root build file
  Make NodeInfo#nodeVersion strongly-typed as Version (#29515)
  Enable license header exclusions (#29379)
  Use proper Java version for BWC builds (#29493)
  Mute TranslogTests#testFatalIOExceptionsWhileWritingConcurrently
  Enable skipping fetching latest for BWC builds (#29497)

Author: jasontedor

TESTING.asciidoc

@@ -498,6 +498,13 @@ will contain your change.
 . Push both branches to your remote repository.
 . Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x`.
 
+== Skip fetching latest
+
+For some BWC testing scenarios, you want to use the local clone of the
+repository without fetching latest. For these use cases, you can set the system
+property `tests.bwc.git_fetch_latest` to `false` and the BWC builds will skip
+fetching the latest from the remote.
+
 == Test coverage analysis
 
 Generating test coverage reports for Elasticsearch is currently not possible through Gradle.

build.gradle

@@ -36,7 +36,7 @@ import java.security.MessageDigest
 // common maven publishing configuration
 subprojects {
   group = 'org.elasticsearch'
-  version = VersionProperties.elasticsearch
+  version = VersionProperties.elasticsearch.toString()
   description = "Elasticsearch subproject ${project.path}"
 }
 
@@ -80,7 +80,7 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
  * in a branch if there are only betas and rcs in the branch so we have
  * *something* to test against. */
 VersionCollection versions = new VersionCollection(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
-if (versions.currentVersion.toString() != VersionProperties.elasticsearch) {
+if (versions.currentVersion != VersionProperties.elasticsearch) {
   throw new GradleException("The last version in Versions.java [${versions.currentVersion}] does not match " +
           "VersionProperties.elasticsearch [${VersionProperties.elasticsearch}]")
 }
@@ -245,7 +245,7 @@ subprojects {
     // other packages (e.g org.elasticsearch.client) will point to server rather than
     // their own artifacts.
     if (project.plugins.hasPlugin(BuildPlugin)) {
-      String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
+      String artifactsHost = VersionProperties.elasticsearch.isSnapshot() ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
       Closure sortClosure = { a, b -> b.group <=> a.group }
       Closure depJavadocClosure = { dep ->
         if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -97,6 +97,12 @@ class BuildPlugin implements Plugin<Project> {
             String compilerJavaHome = findCompilerJavaHome()
             String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome)
             File gradleJavaHome = Jvm.current().javaHome
+
+            final Map<Integer, String> javaVersions = [:]
+            for (int version = 7; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) {
+                javaVersions.put(version, findJavaHome(version));
+            }
+
             String javaVendor = System.getProperty('java.vendor')
             String javaVersion = System.getProperty('java.version')
             String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
@@ -158,10 +164,32 @@ class BuildPlugin implements Plugin<Project> {
                 throw new GradleException(message)
             }
 
+            for (final Map.Entry<Integer, String> javaVersionEntry : javaVersions.entrySet()) {
+                final String javaHome = javaVersionEntry.getValue()
+                if (javaHome == null) {
+                    continue
+                }
+                JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome))
+                final JavaVersion expectedJavaVersionEnum
+                final int version = javaVersionEntry.getKey()
+                if (version < 9) {
+                    expectedJavaVersionEnum = JavaVersion.toVersion("1." + version)
+                } else {
+                    expectedJavaVersionEnum = JavaVersion.toVersion(Integer.toString(version))
+                }
+                if (javaVersionEnum != expectedJavaVersionEnum) {
+                    final String message =
+                            "the environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java" +
+                                    " ${expectedJavaVersionEnum} but is [${javaHome}] corresponding to [${javaVersionEnum}]"
+                    throw new GradleException(message)
+                }
+            }
+
             project.rootProject.ext.compilerJavaHome = compilerJavaHome
             project.rootProject.ext.runtimeJavaHome = runtimeJavaHome
             project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum
             project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum
+            project.rootProject.ext.javaVersions = javaVersions
             project.rootProject.ext.buildChecksDone = true
         }
 
@@ -173,6 +201,7 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome
         project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
         project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
+        project.ext.javaVersions = project.rootProject.ext.javaVersions
     }
 
     private static String findCompilerJavaHome() {
@@ -188,6 +217,27 @@ class BuildPlugin implements Plugin<Project> {
         return javaHome
     }
 
+    private static String findJavaHome(int version) {
+        return System.getenv('JAVA' + version + '_HOME')
+    }
+
+    /**
+     * Get Java home for the project for the specified version. If the specified version is not configured, an exception with the specified
+     * message is thrown.
+     *
+     * @param project the project
+     * @param version the version of Java home to obtain
+     * @param message the exception message if Java home for the specified version is not configured
+     * @return Java home for the specified version
+     * @throws GradleException if Java home for the specified version is not configured
+     */
+    static String getJavaHome(final Project project, final int version, final String message) {
+        if (project.javaVersions.get(version) == null) {
+            throw new GradleException(message)
+        }
+        return project.javaVersions.get(version)
+    }
+
     private static String findRuntimeJavaHome(final String compilerJavaHome) {
         assert compilerJavaHome != null
         return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
@@ -517,17 +567,18 @@ class BuildPlugin implements Plugin<Project> {
             jarTask.destinationDir = new File(project.buildDir, 'distributions')
             // fixup the jar manifest
             jarTask.doFirst {
-                boolean isSnapshot = VersionProperties.elasticsearch.endsWith("-SNAPSHOT");
-                String version = VersionProperties.elasticsearch;
-                if (isSnapshot) {
-                    version = version.substring(0, version.length() - 9)
-                }
+                final Version versionWithoutSnapshot = new Version(
+                        VersionProperties.elasticsearch.major,
+                        VersionProperties.elasticsearch.minor,
+                        VersionProperties.elasticsearch.revision,
+                        VersionProperties.elasticsearch.suffix,
+                        false)
                 // this doFirst is added before the info plugin, therefore it will run
                 // after the doFirst added by the info plugin, and we can override attributes
                 jarTask.manifest.attributes(
-                        'X-Compile-Elasticsearch-Version': version,
+                        'X-Compile-Elasticsearch-Version': versionWithoutSnapshot,
                         'X-Compile-Lucene-Version': VersionProperties.lucene,
-                        'X-Compile-Elasticsearch-Snapshot': isSnapshot,
+                        'X-Compile-Elasticsearch-Snapshot': VersionProperties.elasticsearch.isSnapshot(),
                         'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),
                         'Build-Java-Version': project.compilerJavaVersion)
                 if (jarTask.manifest.attributes.containsKey('Change') == false) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy

@@ -74,20 +74,36 @@ public class Version {
         return "${major}.${minor}.${revision}${suffix}${snapshotStr}"
     }
 
+    public boolean before(Version compareTo) {
+        return id < compareTo.id
+    }
+
     public boolean before(String compareTo) {
-        return id < fromString(compareTo).id
+        return before(fromString(compareTo))
+    }
+
+    public boolean onOrBefore(Version compareTo) {
+        return id <= compareTo.id
     }
 
     public boolean onOrBefore(String compareTo) {
-        return id <= fromString(compareTo).id
+        return onOrBefore(fromString(compareTo))
+    }
+
+    public boolean onOrAfter(Version compareTo) {
+        return id >= compareTo.id
     }
 
     public boolean onOrAfter(String compareTo) {
-        return id >= fromString(compareTo).id
+        return onOrAfter(fromString(compareTo))
+    }
+
+    public boolean after(Version compareTo) {
+        return id > compareTo.id
     }
 
     public boolean after(String compareTo) {
-        return id > fromString(compareTo).id
+        return after(fromString(compareTo))
     }
 
     public boolean onOrBeforeIncludingSuffix(Version otherVersion) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy

@@ -22,7 +22,7 @@ package org.elasticsearch.gradle
  * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
  */
 class VersionProperties {
-    static final String elasticsearch
+    static final Version elasticsearch
     static final String lucene
     static final Map<String, String> versions = new HashMap<>()
     static {
@@ -32,7 +32,7 @@ class VersionProperties {
             throw new RuntimeException('/version.properties resource missing')
         }
         props.load(propsStream)
-        elasticsearch = props.getProperty('elasticsearch')
+        elasticsearch = Version.fromString(props.getProperty('elasticsearch'))
         lucene = props.getProperty('lucene')
         for (String property : props.stringPropertyNames()) {
             versions.put(property, props.getProperty(property))

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -41,7 +41,7 @@ public class DocsTestPlugin extends RestTestPlugin {
              * to the version being built for testing but needs to resolve to
              * the last released version for docs. */
             '\\{version\\}':
-                VersionProperties.elasticsearch.replace('-SNAPSHOT', ''),
+                VersionProperties.elasticsearch.toString().replace('-SNAPSHOT', ''),
             '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
         ]
         Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -77,7 +77,7 @@ class PluginPropertiesTask extends Copy {
             'name': extension.name,
             'description': extension.description,
             'version': stringSnap(extension.version),
-            'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
+            'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch.toString()),
             'javaVersion': project.targetCompatibility as String,
             'classname': extension.classname,
             'extendedPlugins': extension.extendedPlugins.join(','),

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -49,6 +49,14 @@ public class LicenseHeadersTask extends AntTask {
     @Input
     List<String> approvedLicenses = ['Apache', 'Generated']
 
+    /**
+     * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the
+     * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the
+     * possibility of inadvertently using the license on our own source files).
+     */
+    @Input
+    List<String> excludes = []
+
     /**
      * Additional license families that may be found. The key is the license category name (5 characters),
      * followed by the family name and the value list of patterns to search for.
@@ -95,7 +103,7 @@ public class LicenseHeadersTask extends AntTask {
                for (File dir: dirSet.srcDirs) {
                    // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
                    if (dir.exists()) {
-                       ant.fileset(dir: dir)
+                       ant.fileset(dir: dir, excludes: excludes.join(' '))
                    }
                }
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.test
 
+import org.elasticsearch.gradle.Version
 import org.gradle.api.GradleException
 import org.gradle.api.Project
 import org.gradle.api.tasks.Input
@@ -37,7 +38,7 @@ class ClusterConfiguration {
     int numBwcNodes = 0
 
     @Input
-    String bwcVersion = null
+    Version bwcVersion = null
 
     @Input
     int httpPort = 0

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -107,11 +107,14 @@ class ClusterFormationTasks {
         for (int i = 0; i < config.numNodes; i++) {
             // we start N nodes and out of these N nodes there might be M bwc nodes.
             // for each of those nodes we might have a different configuration
-            String elasticsearchVersion = VersionProperties.elasticsearch
-            Configuration distro = currentDistro
+            final Configuration distro
+            final Version elasticsearchVersion
             if (i < config.numBwcNodes) {
                 elasticsearchVersion = config.bwcVersion
                 distro = bwcDistro
+            } else {
+                elasticsearchVersion = VersionProperties.elasticsearch
+                distro = currentDistro
             }
             NodeInfo node = new NodeInfo(config, i, project, prefix, elasticsearchVersion, sharedDir)
             nodes.add(node)
@@ -126,7 +129,7 @@ class ClusterFormationTasks {
     }
 
     /** Adds a dependency on the given distribution */
-    static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
+    static void configureDistributionDependency(Project project, String distro, Configuration configuration, Version elasticsearchVersion) {
         String packaging = distro
         if (distro == 'tar') {
             packaging = 'tar.gz'
@@ -137,7 +140,7 @@ class ClusterFormationTasks {
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
-    static void configureBwcPluginDependency(String name, Project project, Project pluginProject, Configuration configuration, String elasticsearchVersion) {
+    static void configureBwcPluginDependency(String name, Project project, Project pluginProject, Configuration configuration, Version elasticsearchVersion) {
         verifyProjectHasBuildPlugin(name, elasticsearchVersion, project, pluginProject)
         final String pluginName = findPluginName(pluginProject)
         project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
@@ -303,7 +306,7 @@ class ClusterFormationTasks {
         // Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space
         esConfig['cluster.routing.allocation.disk.watermark.low'] = '1b'
         esConfig['cluster.routing.allocation.disk.watermark.high'] = '1b'
-        if (Version.fromString(node.nodeVersion).major >= 6) {
+        if (node.nodeVersion.major >= 6) {
             esConfig['cluster.routing.allocation.disk.watermark.flood_stage'] = '1b'
         }
         // increase script compilation limit since tests can rapid-fire script compilations
@@ -803,7 +806,7 @@ class ClusterFormationTasks {
         return retVal
     }
 
-    static void verifyProjectHasBuildPlugin(String name, String version, Project project, Project pluginProject) {
+    static void verifyProjectHasBuildPlugin(String name, Version version, Project project, Project pluginProject) {
         if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false && pluginProject.plugins.hasPlugin(MetaPluginBuildPlugin) == false) {
             throw new GradleException("Task [${name}] cannot add plugin [${pluginProject.path}] with version [${version}] to project's " +
                     "[${project.path}] dependencies: the plugin is not an esplugin or es_meta_plugin")

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.gradle.test
 
 import com.sun.jna.Native
@@ -29,6 +30,8 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
 
+import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
+
 /**
  * A container for the files and configuration associated with a single node in a test cluster.
  */
@@ -100,10 +103,10 @@ class NodeInfo {
     ByteArrayOutputStream buffer = new ByteArrayOutputStream()
 
     /** the version of elasticsearch that this node runs */
-    String nodeVersion
+    Version nodeVersion
 
     /** Holds node configuration for part of a test cluster. */
-    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, String nodeVersion, File sharedDir) {
+    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, Version nodeVersion, File sharedDir) {
         this.config = config
         this.nodeNum = nodeNum
         this.sharedDir = sharedDir
@@ -162,7 +165,14 @@ class NodeInfo {
             args.add("${esScript}")
         }
 
-        env = ['JAVA_HOME': project.runtimeJavaHome]
+        if (nodeVersion.before("6.2.0")) {
+            env = ['JAVA_HOME': "${-> getJavaHome(project, 8, "JAVA8_HOME must be set to run BWC tests against [" + nodeVersion + "]")}"]
+        } else if (nodeVersion.onOrAfter("6.2.0") && nodeVersion.before("6.3.0")) {
+            env = ['JAVA_HOME': "${-> getJavaHome(project, 9, "JAVA9_HOME must be set to run BWC tests against [" + nodeVersion + "]")}"]
+        } else {
+            env = ['JAVA_HOME': (String) project.runtimeJavaHome]
+        }
+
         args.addAll("-E", "node.portsfile=true")
         String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
         String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
@@ -284,7 +294,7 @@ class NodeInfo {
     }
 
     /** Returns the directory elasticsearch home is contained in for the given distribution */
-    static File homeDir(File baseDir, String distro, String nodeVersion) {
+    static File homeDir(File baseDir, String distro, Version nodeVersion) {
         String path
         switch (distro) {
             case 'integ-test-zip':
@@ -302,7 +312,7 @@ class NodeInfo {
         return new File(baseDir, path)
     }
 
-    static File pathConf(File baseDir, String distro, String nodeVersion) {
+    static File pathConf(File baseDir, String distro, Version nodeVersion) {
         switch (distro) {
             case 'integ-test-zip':
             case 'zip':