Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (46 commits)
  Docs: Add note about missing mapping for doc values field (#29036)
  [DOCS] Removed 6.1.4, 6.2.2, and 6.2.3 coming tags
  Remove BytesArray and BytesReference usage from XContentFactory (#29151)
  Fix BWC issue for PreSyncedFlushResponse
  Add pluggable XContentBuilder writers and human readable writers (#29120)
  Add unreleased version 6.2.4 (#29171)
  Add unreleased version 6.1.5 (#29168)
  Add a note about using the `retry_failed` flag before accepting data loss (#29160)
  Fix typo in percolate-query.asciidoc (#29155)
  Add release notes for 6.1.4 and 6.2.3
  Require HTTP::Tiny 0.070 for release notes script
  REST high-level client: add clear cache API (#28866)
  Relax remote check for bwc project checkouts (#28666)
  Set Java 9 checkstyle to depend on checkstyle conf (#28383)
  Docs: Add example of resetting index setting (#29048)
  Plugins: Fix module name conflict check for meta plugins (#29146)
  Build: Fix meta plugin bundled plugin names (#29147)
  Build: Simplify rest spec hack configuration (#29149)
  CLI: Close subcommands in MultiCommand (#28954)
  Build: Fix meta modules to not install as plugin in tests (#29150)
  ...

Author: martijnvg

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.gradle.doc
 
+import groovy.transform.PackageScope
 import org.elasticsearch.gradle.doc.SnippetsTask.Snippet
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.tasks.Input
@@ -99,6 +100,43 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         return snippet.language == 'js' || snippet.curl
     }
 
+    /**
+     * Converts Kibana's block quoted strings into standard JSON. These
+     * {@code """} delimited strings can be embedded in CONSOLE and can
+     * contain newlines and {@code "} without the normal JSON escaping.
+     * This has to add it.
+     */
+    @PackageScope
+    static String replaceBlockQuote(String body) {
+        int start = body.indexOf('"""');
+        if (start < 0) {
+            return body
+        }
+        /*
+         * 1.3 is a fairly wild guess of the extra space needed to hold
+         * the escaped string.
+         */
+        StringBuilder result = new StringBuilder((int) (body.length() * 1.3));
+        int startOfNormal = 0;
+        while (start >= 0) {
+            int end = body.indexOf('"""', start + 3);
+            if (end < 0) {
+                throw new InvalidUserDataException(
+                    "Invalid block quote starting at $start in:\n$body")
+            }
+            result.append(body.substring(startOfNormal, start));
+            result.append('"');
+            result.append(body.substring(start + 3, end)
+                .replace('"', '\\"')
+                .replace("\n", "\\n"));
+            result.append('"');
+            startOfNormal = end + 3;
+            start = body.indexOf('"""', startOfNormal);
+        }
+        result.append(body.substring(startOfNormal));
+        return result.toString();
+    }
+
     private class TestBuilder {
         private static final String SYNTAX = {
             String method = /(?<method>GET|PUT|POST|HEAD|OPTIONS|DELETE)/
@@ -259,6 +297,8 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             if (body != null) {
                 // Throw out the leading newline we get from parsing the body
                 body = body.substring(1)
+                // Replace """ quoted strings with valid json ones
+                body = replaceBlockQuote(body)
                 current.println("        body: |")
                 body.eachLine { current.println("          $it") }
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/MetaPluginBuildPlugin.groovy

@@ -37,19 +37,28 @@ class MetaPluginBuildPlugin implements Plugin<Project> {
         project.plugins.apply(RestTestPlugin)
 
         createBundleTask(project)
+        boolean isModule = project.path.startsWith(':modules:')
 
         project.integTestCluster {
             dependsOn(project.bundlePlugin)
-            plugin(project.path)
         }
         BuildPlugin.configurePomGeneration(project)
         project.afterEvaluate {
             PluginBuildPlugin.addZipPomGeneration(project)
+            if (isModule) {
+                if (project.integTestCluster.distribution == 'integ-test-zip') {
+                    project.integTestCluster.module(project)
+                }
+             } else {
+                project.integTestCluster.plugin(project.path)
+             }
         }
 
         RunTask run = project.tasks.create('run', RunTask)
         run.dependsOn(project.bundlePlugin)
-        run.clusterConfig.plugin(project.path)
+        if (isModule == false) {
+            run.clusterConfig.plugin(project.path)
+        }
     }
 
     private static void createBundleTask(Project project) {
@@ -79,12 +88,13 @@ class MetaPluginBuildPlugin implements Plugin<Project> {
             buildProperties.extension.plugins.each { String bundledPluginProjectName ->
                 Project bundledPluginProject = project.project(bundledPluginProjectName)
                 bundledPluginProject.afterEvaluate {
+                    String bundledPluginName = bundledPluginProject.esplugin.name
                     bundle.configure {
                         dependsOn bundledPluginProject.bundlePlugin
                         from(project.zipTree(bundledPluginProject.bundlePlugin.outputs.files.singleFile)) {
                             eachFile { FileCopyDetails details ->
                                 // we want each path to have the plugin name interjected
-                                details.relativePath = new RelativePath(true, bundledPluginProjectName, details.relativePath.toString())
+                                details.relativePath = new RelativePath(true, bundledPluginName, details.relativePath.toString())
                             }
                         }
                     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -22,6 +22,7 @@ import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.plugins.JavaBasePlugin
+import org.gradle.api.plugins.quality.Checkstyle
 
 /**
  * Validation tasks which should be run before committing. These run before tests.
@@ -142,7 +143,7 @@ class PrecommitTasks {
             ]
             toolVersion = 7.5
         }
-        for (String taskName : ['checkstyleMain', 'checkstyleTest']) {
+        for (String taskName : ['checkstyleMain', 'checkstyleJava9', 'checkstyleTest']) {
             Task task = project.tasks.findByName(taskName)
             if (task != null) {
                 project.tasks['check'].dependsOn.remove(task)
@@ -154,6 +155,11 @@ class PrecommitTasks {
                 }
             }
         }
+
+        project.tasks.withType(Checkstyle) {
+            dependsOn(copyCheckstyleConf)
+        }
+
         return checkstyleTask
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -20,10 +20,15 @@ package org.elasticsearch.gradle.test
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.DefaultTask
+import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.execution.TaskExecutionAdapter
 import org.gradle.api.internal.tasks.options.Option
+import org.gradle.api.provider.Property
+import org.gradle.api.provider.Provider
+import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskState
 
@@ -47,7 +52,7 @@ public class RestIntegTestTask extends DefaultTask {
 
     /** Flag indicating whether the rest tests in the rest spec should be run. */
     @Input
-    boolean includePackaged = false
+    Property<Boolean> includePackaged = project.objects.property(Boolean)
 
     public RestIntegTestTask() {
         runner = project.tasks.create("${name}Runner", RandomizedTestingTask.class)
@@ -92,10 +97,9 @@ public class RestIntegTestTask extends DefaultTask {
         }
 
         // copy the rest spec/tests into the test resources
-        RestSpecHack.configureDependencies(project)
-        project.afterEvaluate {
-            runner.dependsOn(RestSpecHack.configureTask(project, includePackaged))
-        }
+        Task copyRestSpec = createCopyRestSpecTask(project, includePackaged)
+        runner.dependsOn(copyRestSpec)
+        
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
@@ -109,6 +113,11 @@ public class RestIntegTestTask extends DefaultTask {
         }
     }
 
+    /** Sets the includePackaged property */
+    public void includePackaged(boolean include) {
+        includePackaged.set(include)
+    }
+
     @Option(
         option = "debug-jvm",
         description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
@@ -184,4 +193,47 @@ public class RestIntegTestTask extends DefaultTask {
         println('=========================================')
 
     }
+
+    /**
+     * Creates a task (if necessary) to copy the rest spec files.
+     *
+     * @param project The project to add the copy task to
+     * @param includePackagedTests true if the packaged tests should be copied, false otherwise
+     */
+    private static Task createCopyRestSpecTask(Project project, Provider<Boolean> includePackagedTests) {
+        project.configurations {
+            restSpec
+        }
+        project.dependencies {
+            restSpec "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
+        }
+        Task copyRestSpec = project.tasks.findByName('copyRestSpec')
+        if (copyRestSpec != null) {
+            return copyRestSpec
+        }
+        Map copyRestSpecProps = [
+                name     : 'copyRestSpec',
+                type     : Copy,
+                dependsOn: [project.configurations.restSpec, 'processTestResources']
+        ]
+        copyRestSpec = project.tasks.create(copyRestSpecProps) {
+            into project.sourceSets.test.output.resourcesDir
+        }
+        project.afterEvaluate {
+            copyRestSpec.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
+                include 'rest-api-spec/api/**'
+                if (includePackagedTests.get()) {
+                    include 'rest-api-spec/test/**'
+                }
+            }
+        }
+        project.idea {
+            module {
+                if (scopes.TEST != null) {
+                    scopes.TEST.plus.add(project.configurations.restSpec)
+                }
+            }
+        }
+        return copyRestSpec
+    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy

@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.doc
+
+import org.elasticsearch.gradle.doc.SnippetsTask.Snippet
+import org.gradle.api.InvalidUserDataException
+
+import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.replaceBlockQuote
+
+class RestTestFromSnippetsTaskTest extends GroovyTestCase {
+    void testInvalidBlockQuote() {
+        String input = "\"foo\": \"\"\"bar\"";
+        String message = shouldFail({ replaceBlockQuote(input) });
+        assertEquals("Invalid block quote starting at 7 in:\n$input", message);
+    }
+
+    void testSimpleBlockQuote() {
+        assertEquals("\"foo\": \"bort baz\"",
+            replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\""));
+    }
+
+    void testMultipleBlockQuotes() {
+        assertEquals("\"foo\": \"bort baz\", \"bar\": \"other\"",
+            replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\", \"bar\": \"\"\"other\"\"\""));
+    }
+
+    void testEscapingInBlockQuote() {
+        assertEquals("\"foo\": \"bort\\\" baz\"",
+            replaceBlockQuote("\"foo\": \"\"\"bort\" baz\"\"\""));
+        assertEquals("\"foo\": \"bort\\n baz\"",
+            replaceBlockQuote("\"foo\": \"\"\"bort\n baz\"\"\""));
+    }
+}