Merge branch 'master' into bump-compiler-to-jdk-10

* master: (40 commits)
  Do not optimize append-only if seen normal op with higher seqno (elastic#28787)
  [test] packaging: gradle tasks for groovy tests (elastic#29046)
  Prune only gc deletes below local checkpoint (elastic#28790)
  remove testUnassignedShardAndEmptyNodesInRoutingTable
  elastic#28745: remove extra option in the composite rest tests
  Fold EngineDiskUtils into Store, for better lock semantics (elastic#29156)
  Add file permissions checks to precommit task
  Remove execute mode bit from source files
  Optimize the composite aggregation for match_all and range queries (elastic#28745)
  [Docs] Add rank_eval size parameter k (elastic#29218)
  [DOCS] Remove ignore_z_value parameter link
  Docs: Update docs/index_.asciidoc (elastic#29172)
  Docs: Link C++ client lib elasticlient (elastic#28949)
  [DOCS] Unregister repository instead of deleting it (elastic#29206)
  Docs: HighLevelRestClient#multiSearch (elastic#29144)
  Add Z value support to geo_shape
  Remove type casts in logging in server component (elastic#28807)
  Change BroadcastResponse from ToXContentFragment to ToXContentObject (elastic#28878)
  REST : Split `RestUpgradeAction` into two actions (elastic#29124)
  Add error file docs to important settings
  ...

Author: jasontedor

README.textile

@@ -27,7 +27,6 @@ Elasticsearch is a distributed RESTful search engine built for the cloud. Featur
 ** All the power of Lucene easily exposed through simple configuration / plugins.
 * Per operation consistency
 ** Single document level operations are atomic, consistent, isolated and durable.
-* Open Source under the Apache License, version 2 ("ALv2")
 
 h2. Getting Started
 
@@ -217,23 +216,3 @@ Elasticsearch (1.x), it is required to perform a full cluster restart. Please
 see the "setup reference":
 https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html
 for more details on the upgrade process.
-
-h1. License
-
-<pre>
-This software is licensed under the Apache License, version 2 ("ALv2"), quoted below.
-
-Copyright 2009-2016 Elasticsearch <https://www.elastic.co>
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not
-use this file except in compliance with the License. You may obtain a copy of
-the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations under
-the License.
-</pre>

TESTING.asciidoc

@@ -414,16 +414,16 @@ and in another window:
 
 ----------------------------------------------------
 vagrant up centos-7 --provider virtualbox && vagrant ssh centos-7
-cd $BATS_ARCHIVES
+cd $PACKAGING_ARCHIVES
 sudo -E bats $BATS_TESTS/*rpm*.bats
 ----------------------------------------------------
 
 If you wanted to retest all the release artifacts on a single VM you could:
 
 -------------------------------------------------
-./gradlew setupBats
+./gradlew setupPackagingTest
 cd qa/vagrant; vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404
-cd $BATS_ARCHIVES
+cd $PACKAGING_ARCHIVES
 sudo -E bats $BATS_TESTS/*.bats
 -------------------------------------------------
 

Vagrantfile

@@ -334,9 +334,9 @@ export TAR=/elasticsearch/distribution/tar/build/distributions
 export RPM=/elasticsearch/distribution/rpm/build/distributions
 export DEB=/elasticsearch/distribution/deb/build/distributions
 export BATS=/project/build/bats
-export BATS_UTILS=/project/build/bats/utils
-export BATS_TESTS=/project/build/bats/tests
-export BATS_ARCHIVES=/project/build/bats/archives
+export BATS_UTILS=/project/build/packaging/bats/utils
+export BATS_TESTS=/project/build/packaging/bats/tests
+export PACKAGING_ARCHIVES=/project/build/packaging/archives
 VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
 Defaults   env_keep += "ZIP"
@@ -346,7 +346,7 @@ Defaults   env_keep += "DEB"
 Defaults   env_keep += "BATS"
 Defaults   env_keep += "BATS_UTILS"
 Defaults   env_keep += "BATS_TESTS"
-Defaults   env_keep += "BATS_ARCHIVES"
+Defaults   env_keep += "PACKAGING_ARCHIVES"
 SUDOERS_VARS
     chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -475,14 +475,18 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     static void configureJavadoc(Project project) {
-        project.tasks.withType(Javadoc) {
-            executable = new File(project.compilerJavaHome, 'bin/javadoc')
+        // remove compiled classes from the Javadoc classpath: http://mail.openjdk.java.net/pipermail/javadoc-dev/2018-January/000400.html
+        final List<File> classes = new ArrayList<>()
+        project.tasks.withType(JavaCompile) { javaCompile ->
+            classes.add(javaCompile.destinationDir)
         }
-        configureJavadocJar(project)
-        if (project.compilerJavaVersion == JavaVersion.VERSION_1_10) {
-            project.tasks.withType(Javadoc) { it.enabled = false }
-            project.tasks.getByName('javadocJar').each { it.enabled = false }
+        project.tasks.withType(Javadoc) { javadoc ->
+            javadoc.executable = new File(project.compilerJavaHome, 'bin/javadoc')
+            javadoc.classpath = javadoc.getClasspath().filter { f ->
+                return classes.contains(f) == false
+            }
         }
+        configureJavadocJar(project)
     }
 
     /** Adds a javadocJar task to generate a jar containing javadocs. */

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/FilePermissionsTask.groovy

@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.GradleException
+import org.gradle.api.file.FileCollection
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.SourceSet
+import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.util.PatternSet
+import org.gradle.api.tasks.util.PatternFilterable
+import org.apache.tools.ant.taskdefs.condition.Os
+
+import java.nio.file.Files
+import java.nio.file.attribute.PosixFilePermission
+import java.nio.file.attribute.PosixFileAttributeView
+
+import static java.nio.file.attribute.PosixFilePermission.OTHERS_EXECUTE
+import static java.nio.file.attribute.PosixFilePermission.GROUP_EXECUTE
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE
+
+/**
+ * Checks source files for correct file permissions.
+ */
+public class FilePermissionsTask extends DefaultTask {
+
+    /** A pattern set of which files should be checked. */
+    private PatternFilterable filesFilter = new PatternSet()
+
+    @OutputFile
+    File outputMarker = new File(project.buildDir, 'markers/filePermissions')
+
+    FilePermissionsTask() {
+        onlyIf { !Os.isFamily(Os.FAMILY_WINDOWS) }
+        description = "Checks java source files for correct file permissions"
+        // we always include all source files, and exclude what should not be checked
+        filesFilter.include('**')
+        // exclude sh files that might have the executable bit set
+        filesFilter.exclude('**/*.sh')
+    }
+
+    /** Returns the files this task will check */
+    @InputFiles
+    FileCollection files() {
+        List<FileCollection> collections = new ArrayList<>()
+        for (SourceSet sourceSet : project.sourceSets) {
+            collections.add(sourceSet.allSource.matching(filesFilter))
+        }
+        return project.files(collections.toArray())
+    }
+
+    @TaskAction
+    void checkInvalidPermissions() {
+        List<String> failures = new ArrayList<>()
+        for (File f : files()) {
+            PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(f.toPath(), PosixFileAttributeView.class)
+            Set<PosixFilePermission> permissions = fileAttributeView.readAttributes().permissions()
+            if (permissions.contains(OTHERS_EXECUTE) || permissions.contains(OWNER_EXECUTE) ||
+                permissions.contains(GROUP_EXECUTE)) {
+                failures.add("Source file is executable: " + f)
+            }
+        }
+        if (failures.isEmpty() == false) {
+            throw new GradleException('Found invalid file permissions:\n' + failures.join('\n'))
+        }
+        outputMarker.setText('done', 'UTF-8')
+    }
+
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -37,6 +37,7 @@ class PrecommitTasks {
             configureNamingConventions(project),
             project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
             project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
+            project.tasks.create('filepermissions', FilePermissionsTask.class),
             project.tasks.create('jarHell', JarHellTask.class),
             project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy

@@ -37,9 +37,6 @@ class VagrantPropertiesExtension {
     @Input
     Boolean inheritTests
 
-    @Input
-    Boolean inheritTestArchives
-
     @Input
     Boolean inheritTestUtils
 
@@ -60,10 +57,6 @@ class VagrantPropertiesExtension {
         this.inheritTests = inheritTests
     }
 
-    void setInheritTestArchives(Boolean inheritTestArchives) {
-        this.inheritTestArchives = inheritTestArchives
-    }
-
     void setInheritTestUtils(Boolean inheritTestUtils) {
         this.inheritTestUtils = inheritTestUtils
     }