Merge remote-tracking branch 'es/master' into ccr

* es/master: (38 commits)
  Build: Add pom generation to meta plugins (#28321)
  Add 6.3 version constant to master
  Minor improvements to translog docs (#28237)
  [Docs] Remove typo in painless-getting-started.asciidoc
  Build: Fix meta plugin usage in integ test clusters (#28307)
  Painless: Add spi jar that will be published for extending whitelists (#28302)
  mistyping in one of the highlighting examples comment -> content (#28139)
  Documents applicability of term query to range type (#28166)
  Build: Omit dependency licenses check for elasticsearch deps (#28304)
  Clean up commits when global checkpoint advanced (#28140)
  Implement socket and server ChannelContexts (#28275)
  Plugins: Fix meta plugins to install bundled plugins with their real name (#28285)
  Build: Fix meta plugin integ test installation (#28286)
  Modify Abstract transport tests to use impls (#28270)
  Fork Groovy compiler onto compile Java home
  [Docs] Update tophits-aggregation.asciidoc (#28273)
  Docs: match between snippet to its description (#28296)
  [TEST] fix RequestTests#testSearch in case search source is not set
  REST high-level client: remove index suffix from indices client method names (#28263)
  Fix simple_query_string on invalid input (#28219)
  ...

Author: martijnvg

CONTRIBUTING.md

@@ -126,8 +126,8 @@ Alternatively, `idea.no.launcher=true` can be set in the
 [`idea.properties`](https://www.jetbrains.com/help/idea/file-idea-properties.html)
 file which can be accessed under Help > Edit Custom Properties (this will require a
 restart of IDEA). For IDEA 2017.3 and above, in addition to the JVM option, you will need to go to
-`Run->Edit Configurations->...->Defaults->JUnit` and change the value for the `Shorten command line` setting from
-`user-local default: none` to `classpath file`. You may also need to [remove `ant-javafx.jar` from your
+`Run->Edit Configurations->...->Defaults->JUnit` and verify that the `Shorten command line` setting is set to
+`user-local default: none`. You may also need to [remove `ant-javafx.jar` from your
 classpath](https://github.com/elastic/elasticsearch/issues/14348) if that is
 reported as a source of jar hell.
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -43,6 +43,7 @@ import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
 import org.gradle.api.publish.maven.tasks.GenerateMavenPom
 import org.gradle.api.tasks.bundling.Jar
+import org.gradle.api.tasks.compile.GroovyCompile
 import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.api.tasks.javadoc.Javadoc
 import org.gradle.internal.jvm.Jvm
@@ -455,6 +456,13 @@ class BuildPlugin implements Plugin<Project> {
                 // TODO: use native Gradle support for --release when available (cf. https://github.com/gradle/gradle/issues/2510)
                 options.compilerArgs << '--release' << targetCompatibilityVersion.majorVersion
             }
+            // also apply release flag to groovy, which is used in build-tools
+            project.tasks.withType(GroovyCompile) {
+                final JavaVersion targetCompatibilityVersion = JavaVersion.toVersion(it.targetCompatibility)
+                options.fork = true
+                options.forkOptions.javaHome = new File(project.compilerJavaHome)
+                options.compilerArgs << '--release' << targetCompatibilityVersion.majorVersion
+            }
         }
     }
 
@@ -651,7 +659,10 @@ class BuildPlugin implements Plugin<Project> {
         Task precommit = PrecommitTasks.create(project, true)
         project.check.dependsOn(precommit)
         project.test.mustRunAfter(precommit)
-        project.dependencyLicenses.dependencies = project.configurations.runtime - project.configurations.provided
+        // only require dependency licenses for non-elasticsearch deps
+        project.dependencyLicenses.dependencies = project.configurations.runtime.fileCollection {
+            it.group.startsWith('org.elasticsearch') == false
+        } - project.configurations.provided
     }
 
     private static configureDependenciesInfo(Project project) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/MetaPluginBuildPlugin.groovy

@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.plugin
+
+import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.test.RestTestPlugin
+import org.elasticsearch.gradle.test.RunTask
+import org.elasticsearch.gradle.test.StandaloneRestTestPlugin
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.file.FileCopyDetails
+import org.gradle.api.file.RelativePath
+import org.gradle.api.tasks.bundling.Zip
+
+class MetaPluginBuildPlugin implements Plugin<Project> {
+
+    @Override
+    void apply(Project project) {
+        project.plugins.apply(StandaloneRestTestPlugin)
+        project.plugins.apply(RestTestPlugin)
+
+        createBundleTask(project)
+
+        project.integTestCluster {
+            dependsOn(project.bundlePlugin)
+            plugin(project.path)
+        }
+        BuildPlugin.configurePomGeneration(project)
+        project.afterEvaluate {
+            PluginBuildPlugin.addZipPomGeneration(project)
+        }
+
+        RunTask run = project.tasks.create('run', RunTask)
+        run.dependsOn(project.bundlePlugin)
+        run.clusterConfig.plugin(project.path)
+    }
+
+    private static void createBundleTask(Project project) {
+
+        MetaPluginPropertiesTask buildProperties = project.tasks.create('pluginProperties', MetaPluginPropertiesTask.class)
+
+        // create the actual bundle task, which zips up all the files for the plugin
+        Zip bundle = project.tasks.create(name: 'bundlePlugin', type: Zip, dependsOn: [buildProperties]) {
+            into('elasticsearch') {
+                from(buildProperties.descriptorOutput.parentFile) {
+                    // plugin properties file
+                    include(buildProperties.descriptorOutput.name)
+                }
+            }
+            // due to how the renames work for each bundled plugin, we must exclude empty dirs or every subdir
+            // within bundled plugin zips will show up at the root as an empty dir
+            includeEmptyDirs = false
+
+        }
+        project.assemble.dependsOn(bundle)
+
+        // also make the zip available as a configuration (used when depending on this project)
+        project.configurations.create('zip')
+        project.artifacts.add('zip', bundle)
+
+        // a super hacky way to inject code to run at the end of each of the bundled plugin's configuration
+        // to add itself back to this meta plugin zip
+        project.afterEvaluate {
+            buildProperties.extension.plugins.each { String bundledPluginProjectName ->
+                Project bundledPluginProject = project.project(bundledPluginProjectName)
+                bundledPluginProject.afterEvaluate {
+                    bundle.configure {
+                        dependsOn bundledPluginProject.bundlePlugin
+                        from(project.zipTree(bundledPluginProject.bundlePlugin.outputs.files.singleFile)) {
+                            eachFile { FileCopyDetails details ->
+                                // paths in the individual plugins begin with elasticsearch, and we want to add in the
+                                // bundled plugin name between that and each filename
+                                details.relativePath = new RelativePath(true, 'elasticsearch', bundledPluginProjectName,
+                                                                        details.relativePath.toString().replace('elasticsearch/', ''))
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

libs/elasticsearch-nio/src/main/java/org/elasticsearch/nio/WriteContext.java renamed to buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/MetaPluginPropertiesExtension.groovy

@@ -17,21 +17,30 @@
  * under the License.
  */
 
-package org.elasticsearch.nio;
+package org.elasticsearch.gradle.plugin
 
-import java.io.IOException;
-import java.util.function.BiConsumer;
-
-public interface WriteContext {
-
-    void sendMessage(Object message, BiConsumer<Void, Throwable> listener);
-
-    void queueWriteOperations(WriteOperation writeOperation);
-
-    void flushChannel() throws IOException;
-
-    boolean hasQueuedWriteOps();
-
-    void clearQueuedWriteOps(Exception e);
+import org.gradle.api.Project
+import org.gradle.api.tasks.Input
 
+/**
+ * A container for meta plugin properties that will be written to the meta plugin descriptor, for easy
+ * manipulation in the gradle DSL.
+ */
+class MetaPluginPropertiesExtension {
+    @Input
+    String name
+
+    @Input
+    String description
+
+    /**
+     * The plugins this meta plugin wraps.
+     * Note this is not written to the plugin descriptor, but used to setup the final zip file task.
+     */
+    @Input
+    List<String> plugins
+
+    MetaPluginPropertiesExtension(Project project) {
+        name = project.name
+    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/MetaPluginPropertiesTask.groovy

@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.plugin
+
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Task
+import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.OutputFile
+
+class MetaPluginPropertiesTask extends Copy {
+
+    MetaPluginPropertiesExtension extension
+
+    @OutputFile
+    File descriptorOutput = new File(project.buildDir, 'generated-resources/meta-plugin-descriptor.properties')
+
+    MetaPluginPropertiesTask() {
+        File templateFile = new File(project.buildDir, "templates/${descriptorOutput.name}")
+        Task copyPluginPropertiesTemplate = project.tasks.create('copyPluginPropertiesTemplate') {
+            doLast {
+                InputStream resourceTemplate = PluginPropertiesTask.getResourceAsStream("/${descriptorOutput.name}")
+                templateFile.parentFile.mkdirs()
+                templateFile.setText(resourceTemplate.getText('UTF-8'), 'UTF-8')
+            }
+        }
+
+        dependsOn(copyPluginPropertiesTemplate)
+        extension = project.extensions.create('es_meta_plugin', MetaPluginPropertiesExtension, project)
+        project.afterEvaluate {
+            // check require properties are set
+            if (extension.name == null) {
+                throw new InvalidUserDataException('name is a required setting for es_meta_plugin')
+            }
+            if (extension.description == null) {
+                throw new InvalidUserDataException('description is a required setting for es_meta_plugin')
+            }
+            // configure property substitution
+            from(templateFile.parentFile).include(descriptorOutput.name)
+            into(descriptorOutput.parentFile)
+            Map<String, String> properties = generateSubstitutions()
+            expand(properties)
+            inputs.properties(properties)
+        }
+    }
+
+    Map<String, String> generateSubstitutions() {
+        return ['name': extension.name,
+                'description': extension.description
+        ]
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.plugin
 
+import nebula.plugin.info.scm.ScmInfoPlugin
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.test.RestIntegTestTask
@@ -220,7 +221,8 @@ public class PluginBuildPlugin extends BuildPlugin {
     }
 
     /** Adds a task to generate a pom file for the zip distribution. */
-    protected void addZipPomGeneration(Project project) {
+    public static void addZipPomGeneration(Project project) {
+        project.plugins.apply(ScmInfoPlugin.class)
         project.plugins.apply(MavenPublishPlugin.class)
 
         project.publishing {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -23,6 +23,8 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.plugin.MetaPluginBuildPlugin
+import org.elasticsearch.gradle.plugin.MetaPluginPropertiesExtension
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
@@ -138,8 +140,8 @@ class ClusterFormationTasks {
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
     static void configureBwcPluginDependency(String name, Project project, Project pluginProject, Configuration configuration, String elasticsearchVersion) {
         verifyProjectHasBuildPlugin(name, elasticsearchVersion, project, pluginProject)
-        PluginPropertiesExtension extension = pluginProject.extensions.findByName('esplugin');
-        project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${extension.name}:${elasticsearchVersion}@zip")
+        final String pluginName = findPluginName(pluginProject)
+        project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
     }
 
     /**
@@ -449,7 +451,7 @@ class ClusterFormationTasks {
                 configuration = project.configurations.create(configurationName)
             }
 
-            final String depName = pluginProject.extensions.findByName('esplugin').name
+            final String depName = findPluginName(pluginProject)
 
             Dependency dep = bwcPlugins.dependencies.find {
                 it.name == depName
@@ -753,9 +755,19 @@ class ClusterFormationTasks {
     }
 
     static void verifyProjectHasBuildPlugin(String name, String version, Project project, Project pluginProject) {
-        if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false) {
+        if (pluginProject.plugins.hasPlugin(PluginBuildPlugin) == false && pluginProject.plugins.hasPlugin(MetaPluginBuildPlugin) == false) {
             throw new GradleException("Task [${name}] cannot add plugin [${pluginProject.path}] with version [${version}] to project's " +
-                    "[${project.path}] dependencies: the plugin is not an esplugin")
+                    "[${project.path}] dependencies: the plugin is not an esplugin or es_meta_plugin")
+        }
+    }
+
+    /** Find the plugin name in the given project, whether a regular plugin or meta plugin. */
+    static String findPluginName(Project pluginProject) {
+        PluginPropertiesExtension extension = pluginProject.extensions.findByName('esplugin')
+        if (extension != null) {
+            return extension.name
+        } else {
+            return pluginProject.extensions.findByName('es_meta_plugin').name
         }
     }
 }

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.es-meta-plugin.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.plugin.MetaPluginBuildPlugin