Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (68 commits)
  Add note to migration docs on silent batch mode (#29365)
  Allow using distance measure in the geo context precision (#29273)
  Disable failing query in QueryBuilderBWCIT.
  Improve similarity integration. (#29187)
  Fix some query extraction bugs. (#29283)
  Fixed quote_field_suffix in query_string (#29332)
  TEST: Update negative byte size setting error msg
  Fix bwc in GeoDistanceQuery serialization (#29325)
  Move testMappingConflictRootCause to different class
  Enhance error for out of bounds byte size settings (#29338)
  [Docs] Correct javadoc of GetIndexRequest (#29364)
  Check presence of multi-types before validating new mapping (#29316)
  Make TransportRankEvalAction members final
  Pass through script params in scripted metric agg (#29154)
  Remove silent batch mode from install plugin (#29359)
  Track Lucene operations in engine explicitly (#29357)
  Build: Fix Java9 MR build (#29312)
  Reindex: Fix error in delete-by-query rest spec (#29318)
  Move Nullable into core (#29341)
  [Docs] Correct experimental note formatting
  ...

Author: martijnvg

.ci/java-versions.properties

@@ -0,0 +1,8 @@
+# This file is used with all of the non-matrix tests in Jenkins.
+
+# This .properties file defines the versions of Java with which to
+# build and test Elasticsearch for this branch. Valid Java versions
+# are 'java' or 'openjdk' followed by the major release number.
+
+ES_BUILD_JAVA=java10
+ES_RUNTIME_JAVA=java8

.ci/matrix-build-javas.yml

@@ -0,0 +1,9 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are included as an axis of the matrix.
+
+# This axis of the build matrix represents the versions of Java with
+# which Elasticsearch will be built.  Valid Java versions are 'java'
+# or 'openjdk' followed by the major release number.
+
+ES_BUILD_JAVA:
+  - java10

.ci/matrix-java-exclusions.yml

@@ -0,0 +1,14 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are excluded from the test matrix.
+
+# The yaml mapping below represents a single intersection on the build
+# matrix where a test *should not* be run.  The value of the exclude
+# key is a list of maps.
+
+# In this example all of the combinations defined in the matrix will
+# run except for the test that builds with java10 and runs with java8.
+# exclude:
+#   - ES_BUILD_JAVA: java10
+#     ES_RUNTIME_JAVA: java8
+
+exclude:

.ci/matrix-runtime-javas.yml

@@ -0,0 +1,10 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are included as an axis of the matrix.
+
+# This axis of the build matrix represents the versions of Java on
+# which Elasticsearch will be tested.  Valid Java versions are 'java'
+# or 'openjdk' followed by the major release number.
+
+ES_RUNTIME_JAVA:
+  - java8
+  - java10

TESTING.asciidoc

@@ -296,7 +296,6 @@ e.g. -Dtests.rest.suite=index,get,create/10_with_id
 * `tests.rest.blacklist`: comma separated globs that identify tests that are
 blacklisted and need to be skipped
 e.g. -Dtests.rest.blacklist=index/*/Index document,get/10_basic/*
-* `tests.rest.spec`: REST spec path (default /rest-api-spec/api)
 
 Note that the REST tests, like all the integration tests, can be run against an external
 cluster by specifying the `tests.cluster` property, which if present needs to contain a
@@ -477,12 +476,12 @@ branch. Finally, on a release branch, it will test against the most recent relea
 === BWC Testing against a specific remote/branch
 
 Sometimes a backward compatibility change spans two versions. A common case is a new functionality
-that needs a BWC bridge in and an unreleased versioned of a release branch (for example, 5.x).
+that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
 To test the changes, you can instruct Gradle to build the BWC version from a another remote/branch combination instead of
-pulling the release branch from GitHub. You do so using the `tests.bwc.remote` and `tests.bwc.refspec` system properties:
+pulling the release branch from GitHub. You do so using the `tests.bwc.remote` and `tests.bwc.refspec.BRANCH` system properties:
 
 -------------------------------------------------
-./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec=index_req_bwc_5.x
+./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x
 -------------------------------------------------
 
 The branch needs to be available on the remote that the BWC makes of the
@@ -497,7 +496,7 @@ will need to:
 will contain your change.
 . Create a branch called `index_req_bwc_5.x` off `5.x`. This will contain your bwc layer.
 . Push both branches to your remote repository.
-. Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec=index_req_bwc_5.x`.
+. Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x`.
 
 == Test coverage analysis
 

build.gradle

@@ -200,7 +200,8 @@ subprojects {
     "org.elasticsearch:elasticsearch:${version}": ':server',
     "org.elasticsearch:elasticsearch-cli:${version}": ':server:cli',
     "org.elasticsearch:elasticsearch-core:${version}": ':libs:elasticsearch-core',
-    "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',    
+    "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
+    "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',

buildSrc/build.gradle

@@ -94,7 +94,7 @@ dependencies {
   compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
   compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
-  compile 'de.thetaphi:forbiddenapis:2.4.1'
+  compile 'de.thetaphi:forbiddenapis:2.5'
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -311,8 +311,8 @@ class BuildPlugin implements Plugin<Project> {
     /** Adds repositories used by ES dependencies */
     static void configureRepositories(Project project) {
         RepositoryHandler repos = project.repositories
-        if (System.getProperty("repos.mavenlocal") != null) {
-            // with -Drepos.mavenlocal=true we can force checking the local .m2 repo which is
+        if (System.getProperty("repos.mavenLocal") != null) {
+            // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
             // useful for development ie. bwc tests where we install stuff in the local repository
             // such that we don't have to pass hardcoded files to gradle
             repos.mavenLocal()
@@ -551,7 +551,7 @@ class BuildPlugin implements Plugin<Project> {
                 if (project.licenseFile == null || project.noticeFile == null) {
                     throw new GradleException("Must specify license and notice file for project ${project.path}")
                 }
-                jarTask.into('META-INF') {
+                jarTask.metaInf {
                     from(project.licenseFile.parent) {
                         include project.licenseFile.name
                     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -95,7 +95,7 @@ public class PluginBuildPlugin extends BuildPlugin {
             // we "upgrade" these optional deps to provided for plugins, since they will run
             // with a full elasticsearch server that includes optional deps
             compileOnly "org.locationtech.spatial4j:spatial4j:${project.versions.spatial4j}"
-            compileOnly "com.vividsolutions:jts:${project.versions.jts}"
+            compileOnly "org.locationtech.jts:jts-core:${project.versions.jts}"
             compileOnly "org.apache.logging.log4j:log4j-api:${project.versions.log4j}"
             compileOnly "org.apache.logging.log4j:log4j-core:${project.versions.log4j}"
             compileOnly "org.elasticsearch:jna:${project.versions.jna}"

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.precommit
 
+import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.gradle.api.Project
 import org.gradle.api.Task
@@ -83,17 +84,14 @@ class PrecommitTasks {
                               getClass().getResource('/forbidden/es-all-signatures.txt')]
             suppressAnnotations = ['**.SuppressForbidden']
         }
-        Task mainForbidden = project.tasks.findByName('forbiddenApisMain')
-        if (mainForbidden != null) {
-            mainForbidden.configure {
-                signaturesURLs += getClass().getResource('/forbidden/es-server-signatures.txt')
-            }
-        }
-        Task testForbidden = project.tasks.findByName('forbiddenApisTest')
-        if (testForbidden != null) {
-            testForbidden.configure {
-                signaturesURLs += getClass().getResource('/forbidden/es-test-signatures.txt')
-                signaturesURLs += getClass().getResource('/forbidden/http-signatures.txt')
+        project.tasks.withType(CheckForbiddenApis) {
+            // we do not use the += operator to add signatures, as conventionMappings of Gradle do not work when it's configured using withType:
+            if (name.endsWith('Test')) {
+                signaturesURLs = project.forbiddenApis.signaturesURLs +
+                    [ getClass().getResource('/forbidden/es-test-signatures.txt'), getClass().getResource('/forbidden/http-signatures.txt') ]
+            } else {
+                signaturesURLs = project.forbiddenApis.signaturesURLs +
+                    [ getClass().getResource('/forbidden/es-server-signatures.txt') ]
             }
         }
         Task forbiddenApis = project.tasks.findByName('forbiddenApis')
@@ -144,21 +142,15 @@ class PrecommitTasks {
             ]
             toolVersion = 7.5
         }
-        for (String taskName : ['checkstyleMain', 'checkstyleJava9', 'checkstyleTest']) {
-            Task task = project.tasks.findByName(taskName)
-            if (task != null) {
-                project.tasks['check'].dependsOn.remove(task)
-                checkstyleTask.dependsOn(task)
-                task.dependsOn(copyCheckstyleConf)
-                task.inputs.file(checkstyleSuppressions)
-                task.reports {
-                    html.enabled false
-                }
-            }
-        }
 
-        project.tasks.withType(Checkstyle) {
-            dependsOn(copyCheckstyleConf)
+        project.tasks.withType(Checkstyle) { task ->
+            project.tasks[JavaBasePlugin.CHECK_TASK_NAME].dependsOn.remove(task)
+            checkstyleTask.dependsOn(task)
+            task.dependsOn(copyCheckstyleConf)
+            task.inputs.file(checkstyleSuppressions)
+            task.reports {
+                html.enabled false
+            }
         }
 
         return checkstyleTask

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -498,7 +498,7 @@ class ClusterFormationTasks {
          * the short name requiring the path to already exist.
          */
         final Object esPluginUtil = "${-> node.binPath().resolve('elasticsearch-plugin').toString()}"
-        final Object[] args = [esPluginUtil, 'install', file]
+        final Object[] args = [esPluginUtil, 'install', '--batch', file]
         return configureExecTask(name, project, setup, node, args)
     }
 

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -248,9 +248,7 @@
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]EsExecutors.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]ThreadBarrier.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]concurrent[/\\]ThreadContext.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]XContentFactory.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]XContentHelper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]smile[/\\]SmileXContent.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]Discovery.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoverySettings.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ZenDiscovery.java" checks="LineLength" />

buildSrc/version.properties

@@ -1,9 +1,9 @@
 elasticsearch     = 6.3.0
-lucene            = 7.2.1
+lucene            = 7.3.0-snapshot-98a6b3d
 
 # optional dependencies
-spatial4j         = 0.6
-jts               = 1.13
+spatial4j         = 0.7
+jts               = 1.15.0
 jackson           = 2.8.10
 snakeyaml         = 1.17
 # when updating log4j, please update also docs/java-api/index.asciidoc