Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x:
  Fix building Javadoc JARs on JDK for client JARs (#29274)
  Require JDK 10 to build Elasticsearch (#29174)
  Decouple NamedXContentRegistry from ElasticsearchException (#29253)
  Docs: Update generating test coverage reports (#29255)
  [TEST] Fix issue with HttpInfo passed invalid parameter
  Remove all dependencies from XContentBuilder (#29225)
  Fix sporadic failure in CompositeValuesCollectorQueueTests
  Propagate ignore_unmapped to inner_hits (#29261)
  TEST: Increase timeout for testPrimaryReplicaResyncFailed
  REST client: hosts marked dead for the first time should not be immediately retried (#29230)
  Make SearchStats implement Writeable (#29258)
  [Docs] Spelling and grammar changes to reindex.asciidoc (#29232)
  [test] packaging: gradle tasks for groovy tests (#29046)
  remove testUnassignedShardAndEmptyNodesInRoutingTable
  Add file permissions checks to precommit task
  Remove execute mode bit from source files
  #28745: remove 7.x option in the composite rest tests.
  Optimize the composite aggregation for match_all and range queries (#28745)
  Clarify deprecation warning for auto_generate_phrase_query (#29204)

Author: martijnvg

CONTRIBUTING.md

@@ -92,11 +92,11 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
-JDK 9 is required to build Elasticsearch. You must have a JDK 9 installation
+JDK 10 is required to build Elasticsearch. You must have a JDK 10 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
-your JDK 9 installation. By default, tests use the same runtime as `JAVA_HOME`.
+your JDK 10 installation. By default, tests use the same runtime as `JAVA_HOME`.
 However, since Elasticsearch, supports JDK 8 the build supports compiling with
-JDK 9 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
+JDK 10 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 8 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 8.
 

TESTING.asciidoc

@@ -414,16 +414,16 @@ and in another window:
 
 ----------------------------------------------------
 vagrant up centos-7 --provider virtualbox && vagrant ssh centos-7
-cd $BATS_ARCHIVES
+cd $PACKAGING_ARCHIVES
 sudo -E bats $BATS_TESTS/*rpm*.bats
 ----------------------------------------------------
 
 If you wanted to retest all the release artifacts on a single VM you could:
 
 -------------------------------------------------
-./gradlew setupBats
+./gradlew setupPackagingTest
 cd qa/vagrant; vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404
-cd $BATS_ARCHIVES
+cd $PACKAGING_ARCHIVES
 sudo -E bats $BATS_TESTS/*.bats
 -------------------------------------------------
 
@@ -499,32 +499,21 @@ will contain your change.
 . Push both branches to your remote repository.
 . Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec=index_req_bwc_5.x`.
 
-== Coverage analysis
+== Test coverage analysis
 
-Tests can be run instrumented with jacoco to produce a coverage report in
-`target/site/jacoco/`.
+Generating test coverage reports for Elasticsearch is currently not possible through Gradle.
+However, it _is_ possible to gain insight in code coverage using IntelliJ's built-in coverage 
+analysis tool that can measure coverage upon executing specific tests. Eclipse may also be able
+to do the same using the EclEmma plugin.
 
-Unit test coverage:
-
----------------------------------------------------------------------------
-mvn -Dtests.coverage test jacoco:report
----------------------------------------------------------------------------
-
-Integration test coverage:
-
----------------------------------------------------------------------------
-mvn -Dtests.coverage -Dskip.unit.tests verify jacoco:report
----------------------------------------------------------------------------
-
-Combined (Unit+Integration) coverage:
-
----------------------------------------------------------------------------
-mvn -Dtests.coverage verify jacoco:report
----------------------------------------------------------------------------
+Test coverage reporting used to be possible with JaCoCo when Elasticsearch was using Maven
+as its build system. Since the switch to Gradle though, this is no longer possible, seeing as
+the code currently used to build Elasticsearch does not allow JaCoCo to recognize its tests.
+For more information on this, see the discussion in https://github.com/elastic/elasticsearch/issues/28867[issue #28867].
 
 == Launching and debugging from an IDE
 
-If you want to run elasticsearch from your IDE, the `./gradlew run` task
+If you want to run Elasticsearch from your IDE, the `./gradlew run` task
 supports a remote debugging option:
 
 ---------------------------------------------------------------------------

Vagrantfile

@@ -334,9 +334,9 @@ export TAR=/elasticsearch/distribution/tar/build/distributions
 export RPM=/elasticsearch/distribution/rpm/build/distributions
 export DEB=/elasticsearch/distribution/deb/build/distributions
 export BATS=/project/build/bats
-export BATS_UTILS=/project/build/bats/utils
-export BATS_TESTS=/project/build/bats/tests
-export BATS_ARCHIVES=/project/build/bats/archives
+export BATS_UTILS=/project/build/packaging/bats/utils
+export BATS_TESTS=/project/build/packaging/bats/tests
+export PACKAGING_ARCHIVES=/project/build/packaging/archives
 VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
 Defaults   env_keep += "ZIP"
@@ -346,7 +346,7 @@ Defaults   env_keep += "DEB"
 Defaults   env_keep += "BATS"
 Defaults   env_keep += "BATS_UTILS"
 Defaults   env_keep += "BATS_TESTS"
-Defaults   env_keep += "BATS_ARCHIVES"
+Defaults   env_keep += "PACKAGING_ARCHIVES"
 SUDOERS_VARS
     chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -58,7 +58,7 @@ import java.time.ZonedDateTime
 class BuildPlugin implements Plugin<Project> {
 
     static final JavaVersion minimumRuntimeVersion = JavaVersion.VERSION_1_8
-    static final JavaVersion minimumCompilerVersion = JavaVersion.VERSION_1_9
+    static final JavaVersion minimumCompilerVersion = JavaVersion.VERSION_1_10
 
     @Override
     void apply(Project project) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -168,12 +168,10 @@ public class PluginBuildPlugin extends BuildPlugin {
             Files.copy(jarFile.resolveSibling(sourcesFileName), jarFile.resolveSibling(clientSourcesFileName),
                     StandardCopyOption.REPLACE_EXISTING)
 
-            if (project.compilerJavaVersion < JavaVersion.VERSION_1_10) {
-                String javadocFileName = jarFile.fileName.toString().replace('.jar', '-javadoc.jar')
-                String clientJavadocFileName = clientFileName.replace('.jar', '-javadoc.jar')
-                Files.copy(jarFile.resolveSibling(javadocFileName), jarFile.resolveSibling(clientJavadocFileName),
-                        StandardCopyOption.REPLACE_EXISTING)
-            }
+            String javadocFileName = jarFile.fileName.toString().replace('.jar', '-javadoc.jar')
+            String clientJavadocFileName = clientFileName.replace('.jar', '-javadoc.jar')
+            Files.copy(jarFile.resolveSibling(javadocFileName), jarFile.resolveSibling(clientJavadocFileName),
+                    StandardCopyOption.REPLACE_EXISTING)
         }
         project.assemble.dependsOn(clientJar)
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/FilePermissionsTask.groovy

@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.GradleException
+import org.gradle.api.file.FileCollection
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.SourceSet
+import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.util.PatternSet
+import org.gradle.api.tasks.util.PatternFilterable
+import org.apache.tools.ant.taskdefs.condition.Os
+
+import java.nio.file.Files
+import java.nio.file.attribute.PosixFilePermission
+import java.nio.file.attribute.PosixFileAttributeView
+
+import static java.nio.file.attribute.PosixFilePermission.OTHERS_EXECUTE
+import static java.nio.file.attribute.PosixFilePermission.GROUP_EXECUTE
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE
+
+/**
+ * Checks source files for correct file permissions.
+ */
+public class FilePermissionsTask extends DefaultTask {
+
+    /** A pattern set of which files should be checked. */
+    private PatternFilterable filesFilter = new PatternSet()
+
+    @OutputFile
+    File outputMarker = new File(project.buildDir, 'markers/filePermissions')
+
+    FilePermissionsTask() {
+        onlyIf { !Os.isFamily(Os.FAMILY_WINDOWS) }
+        description = "Checks java source files for correct file permissions"
+        // we always include all source files, and exclude what should not be checked
+        filesFilter.include('**')
+        // exclude sh files that might have the executable bit set
+        filesFilter.exclude('**/*.sh')
+    }
+
+    /** Returns the files this task will check */
+    @InputFiles
+    FileCollection files() {
+        List<FileCollection> collections = new ArrayList<>()
+        for (SourceSet sourceSet : project.sourceSets) {
+            collections.add(sourceSet.allSource.matching(filesFilter))
+        }
+        return project.files(collections.toArray())
+    }
+
+    @TaskAction
+    void checkInvalidPermissions() {
+        List<String> failures = new ArrayList<>()
+        for (File f : files()) {
+            PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(f.toPath(), PosixFileAttributeView.class)
+            Set<PosixFilePermission> permissions = fileAttributeView.readAttributes().permissions()
+            if (permissions.contains(OTHERS_EXECUTE) || permissions.contains(OWNER_EXECUTE) ||
+                permissions.contains(GROUP_EXECUTE)) {
+                failures.add("Source file is executable: " + f)
+            }
+        }
+        if (failures.isEmpty() == false) {
+            throw new GradleException('Found invalid file permissions:\n' + failures.join('\n'))
+        }
+        outputMarker.setText('done', 'UTF-8')
+    }
+
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -37,6 +37,7 @@ class PrecommitTasks {
             configureNamingConventions(project),
             project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
             project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
+            project.tasks.create('filepermissions', FilePermissionsTask.class),
             project.tasks.create('jarHell', JarHellTask.class),
             project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy

@@ -37,9 +37,6 @@ class VagrantPropertiesExtension {
     @Input
     Boolean inheritTests
 
-    @Input
-    Boolean inheritTestArchives
-
     @Input
     Boolean inheritTestUtils
 
@@ -60,10 +57,6 @@ class VagrantPropertiesExtension {
         this.inheritTests = inheritTests
     }
 
-    void setInheritTestArchives(Boolean inheritTestArchives) {
-        this.inheritTestArchives = inheritTestArchives
-    }
-
     void setInheritTestUtils(Boolean inheritTestUtils) {
         this.inheritTestUtils = inheritTestUtils
     }