Merge master

Author: liketic

README.textile

@@ -208,8 +208,7 @@ In order to create a distribution, simply run the @gradle assemble@ command in t
 
 The distribution for each project will be created under the @build/distributions@ directory in that project.
 
-See the "TESTING":TESTING.asciidoc file for more information about
-running the Elasticsearch test suite.
+See the "TESTING":TESTING.asciidoc file for more information about running the Elasticsearch test suite.
 
 h3. Upgrading from Elasticsearch 1.x?
 

TESTING.asciidoc

@@ -351,8 +351,8 @@ These are the linux flavors the Vagrantfile currently supports:
 * debian-9 aka stretch, the current debian stable distribution
 * centos-6
 * centos-7
-* fedora-25
 * fedora-26
+* fedora-27
 * oel-6 aka Oracle Enterprise Linux 6
 * oel-7 aka Oracle Enterprise Linux 7
 * sles-12
@@ -428,23 +428,23 @@ sudo -E bats $BATS_TESTS/*.bats
 You can also use Gradle to prepare the test environment and then starts a single VM:
 
 -------------------------------------------------
-gradle vagrantFedora25#up
+gradle vagrantFedora27#up
 -------------------------------------------------
 
 Or any of vagrantCentos6#up, vagrantCentos7#up, vagrantDebian8#up,
-vagrantFedora25#up, vagrantOel6#up, vagrantOel7#up, vagrantOpensuse13#up,
-vagrantSles12#up, vagrantUbuntu1404#up, vagrantUbuntu1604#up.
+vagrantDebian9#up, vagrantFedora26#up, vagrantFedora27#up, vagrantOel6#up, vagrantOel7#up,
+vagrantOpensuse42#up,vagrantSles12#up, vagrantUbuntu1404#up, vagrantUbuntu1604#up.
 
 Once up, you can then connect to the VM using SSH from the elasticsearch directory:
 
 -------------------------------------------------
-vagrant ssh fedora-25
+vagrant ssh fedora-27
 -------------------------------------------------
 
 Or from another directory:
 
 -------------------------------------------------
-VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-25
+VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-27
 -------------------------------------------------
 
 Note: Starting vagrant VM outside of the elasticsearch folder requires to

Vagrantfile

@@ -60,14 +60,14 @@ Vagrant.configure(2) do |config|
     config.vm.box = "elastic/oraclelinux-7-x86_64"
     rpm_common config
   end
-  config.vm.define "fedora-25" do |config|
-    config.vm.box = "elastic/fedora-25-x86_64"
-    dnf_common config
-  end
   config.vm.define "fedora-26" do |config|
     config.vm.box = "elastic/fedora-26-x86_64"
     dnf_common config
   end
+  config.vm.define "fedora-27" do |config|
+    config.vm.box = "elastic/fedora-27-x86_64"
+    dnf_common config
+  end
   config.vm.define "opensuse-42" do |config|
     config.vm.box = "elastic/opensuse-42-x86_64"
     opensuse_common config

build.gradle

@@ -81,6 +81,7 @@ List<Version> versions = []
 // keep track of the previous major version's last minor, so we know where wire compat begins
 int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
 int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
+int prevBugfixIndex = -1 // index in the versions list of the last bugfix release from the prev major
 for (String line : versionLines) {
   /* Note that this skips alphas and betas which is fine because they aren't
    * compatible with anything. */
@@ -108,12 +109,19 @@ for (String line : versionLines) {
         lastPrevMinor = minor
       }
     }
+    if (major == prevMajor) {
+      prevBugfixIndex = versions.size() - 1
+    }
   }
 }
 if (versions.toSorted { it.id } != versions) {
   println "Versions: ${versions}"
   throw new GradleException("Versions.java contains out of order version constants")
 }
+if (prevBugfixIndex != -1) {
+  versions[prevBugfixIndex] = new Version(versions[prevBugfixIndex].major, versions[prevBugfixIndex].minor,
+                                          versions[prevBugfixIndex].bugfix, versions[prevBugfixIndex].suffix, true)
+}
 if (currentVersion.bugfix == 0) {
   // If on a release branch, after the initial release of that branch, the bugfix version will
   // be bumped, and will be != 0. On master and N.x branches, we want to test against the
@@ -223,6 +231,7 @@ subprojects {
     "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
+    "org.elasticsearch:elasticsearch-cli:${version}": ':core:cli',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
@@ -261,6 +270,11 @@ subprojects {
       ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
       ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
     }
+  } else if (indexCompatVersions[-2].snapshot) {
+    /* This is a terrible hack for the bump to 6.0.1 which will be fixed by #27397 */
+    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
+    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
+    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
   }
   project.afterEvaluate {
     configurations.all {

buildSrc/build.gradle

@@ -19,6 +19,8 @@
 
 import java.nio.file.Files
 
+import org.gradle.util.GradleVersion
+
 apply plugin: 'groovy'
 
 group = 'org.elasticsearch.gradle'
@@ -99,9 +101,11 @@ dependencies {
 
 // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
 // Use logging dependency instead
+// Gradle 4.3.1 stopped releasing the logging jars to jcenter, just use the last available one
+GradleVersion logVersion = GradleVersion.current() > GradleVersion.version('4.3') ? GradleVersion.version('4.3') : GradleVersion.current()
 
 dependencies {
-  compileOnly "org.gradle:gradle-logging:${GradleVersion.current().getVersion()}"
+  compileOnly "org.gradle:gradle-logging:${logVersion.getVersion()}"
   compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
 }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -131,9 +131,10 @@ class BuildPlugin implements Plugin<Project> {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }
 
-            final GradleVersion maxGradle = GradleVersion.version('4.2')
-            if (currentGradleVersion >= maxGradle) {
-                throw new GradleException("${maxGradle} or above is not compatible with the elasticsearch build")
+            final GradleVersion gradle42 = GradleVersion.version('4.2')
+            final GradleVersion gradle43 = GradleVersion.version('4.3')
+            if (currentGradleVersion >= gradle42 && currentGradleVersion < gradle43) {
+                throw new GradleException("${currentGradleVersion} is not compatible with the elasticsearch build")
             }
 
             // enforce Java version
@@ -238,7 +239,7 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Return the configuration name used for finding transitive deps of the given dependency. */
     private static String transitiveDepConfigName(String groupId, String artifactId, String version) {
-        return "_transitive_${groupId}:${artifactId}:${version}"
+        return "_transitive_${groupId}_${artifactId}_${version}"
     }
 
     /**

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -64,10 +64,10 @@ class ClusterConfiguration {
 
     /**
      * Configuration of the setting <tt>discovery.zen.minimum_master_nodes</tt> on the nodes.
-     * In case of more than one node, this defaults to (number of nodes / 2) + 1
+     * In case of more than one node, this defaults to the number of nodes
      */
     @Input
-    Closure<Integer> minimumMasterNodes = { getNumNodes() > 1 ? getNumNodes().intdiv(2) + 1 : -1 }
+    Closure<Integer> minimumMasterNodes = { getNumNodes() > 1 ? getNumNodes() : -1 }
 
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -425,7 +425,7 @@ class ClusterFormationTasks {
 
             Project pluginProject = plugin.getValue()
             verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
-            String configurationName = "_plugin_${prefix}_${pluginProject.path}"
+            String configurationName = pluginConfigurationName(prefix, pluginProject)
             Configuration configuration = project.configurations.findByName(configurationName)
             if (configuration == null) {
                 configuration = project.configurations.create(configurationName)
@@ -454,13 +454,21 @@ class ClusterFormationTasks {
         return copyPlugins
     }
 
+    private static String pluginConfigurationName(final String prefix, final Project project) {
+        return "_plugin_${prefix}_${project.path}".replace(':', '_')
+    }
+
+    private static String pluginBwcConfigurationName(final String prefix, final Project project) {
+        return "_plugin_bwc_${prefix}_${project.path}".replace(':', '_')
+    }
+
     /** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */
     static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
         Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins")
         for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
             Project pluginProject = plugin.getValue()
             verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
-            String configurationName = "_plugin_bwc_${prefix}_${pluginProject.path}"
+            String configurationName = pluginBwcConfigurationName(prefix, pluginProject)
             Configuration configuration = project.configurations.findByName(configurationName)
             if (configuration == null) {
                 configuration = project.configurations.create(configurationName)
@@ -499,9 +507,9 @@ class ClusterFormationTasks {
     static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) {
         final FileCollection pluginZip;
         if (node.nodeVersion != VersionProperties.elasticsearch) {
-            pluginZip = project.configurations.getByName("_plugin_bwc_${prefix}_${plugin.path}")
+            pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, plugin))
         } else {
-            pluginZip = project.configurations.getByName("_plugin_${prefix}_${plugin.path}")
+            pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, plugin))
         }
         // delay reading the file location until execution time by wrapping in a closure within a GString
         final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -19,8 +19,8 @@ class VagrantTestPlugin implements Plugin<Project> {
             'centos-7',
             'debian-8',
             'debian-9',
-            'fedora-25',
             'fedora-26',
+            'fedora-27',
             'oel-6',
             'oel-7',
             'opensuse-42',

buildSrc/version.properties

@@ -1,12 +1,12 @@
 # When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.1.0-snapshot-f33ed4ba12a
+lucene            = 7.1.0
 
 # optional dependencies
 spatial4j         = 0.6
 jts               = 1.13
-jackson           = 2.8.6
-snakeyaml         = 1.15
+jackson           = 2.8.10
+snakeyaml         = 1.17
 # when updating log4j, please update also docs/java-api/index.asciidoc
 log4j             = 2.9.1
 slf4j             = 1.6.2

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java

@@ -53,6 +53,7 @@ protected void doExecute(SearchRequest request, ActionListener<SearchResponse> l
                 new SearchHit[0], 0L, 0.0f),
             new InternalAggregations(Collections.emptyList()),
             new Suggest(Collections.emptyList()),
-            new SearchProfileShardResults(Collections.emptyMap()), false, false, 1), "", 1, 1, 0, 0, new ShardSearchFailure[0]));
+            new SearchProfileShardResults(Collections.emptyMap()), false, false, 1), "", 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY,
+                SearchResponse.Clusters.EMPTY));
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.Header;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+/**
+ * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Indices API.
+ *
+ * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices.html">Indices API on elastic.co</a>
+ */
+public final class IndicesClient {
+    private final RestHighLevelClient restHighLevelClient;
+
+    public IndicesClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Deletes an index using the Delete Index API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-delete-index.html">
+     * Delete Index API on elastic.co</a>
+     */
+    public DeleteIndexResponse deleteIndex(DeleteIndexRequest deleteIndexRequest, Header... headers) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(deleteIndexRequest, Request::deleteIndex, DeleteIndexResponse::fromXContent,
+            Collections.emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously deletes an index using the Delete Index API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-delete-index.html">
+     * Delete Index API on elastic.co</a>
+     */
+    public void deleteIndexAsync(DeleteIndexRequest deleteIndexRequest, ActionListener<DeleteIndexResponse> listener, Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(deleteIndexRequest, Request::deleteIndex, DeleteIndexResponse::fromXContent,
+            listener, Collections.emptySet(), headers);
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -29,6 +29,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
@@ -123,6 +124,17 @@ static Request delete(DeleteRequest deleteRequest) {
         return new Request(HttpDelete.METHOD_NAME, endpoint, parameters.getParams(), null);
     }
 
+    static Request deleteIndex(DeleteIndexRequest deleteIndexRequest) {
+        String endpoint = endpoint(deleteIndexRequest.indices(), Strings.EMPTY_ARRAY, "");
+
+        Params parameters = Params.builder();
+        parameters.withTimeout(deleteIndexRequest.timeout());
+        parameters.withMasterTimeout(deleteIndexRequest.masterNodeTimeout());
+        parameters.withIndicesOptions(deleteIndexRequest.indicesOptions());
+
+        return new Request(HttpDelete.METHOD_NAME, endpoint, parameters.getParams(), null);
+    }
+
     static Request info() {
         return new Request(HttpGet.METHOD_NAME, "/", Collections.emptyMap(), null);
     }
@@ -179,23 +191,23 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                         metadata.field("_id", request.id());
                     }
                     if (Strings.hasLength(request.routing())) {
-                        metadata.field("_routing", request.routing());
+                        metadata.field("routing", request.routing());
                     }
                     if (Strings.hasLength(request.parent())) {
-                        metadata.field("_parent", request.parent());
+                        metadata.field("parent", request.parent());
                     }
                     if (request.version() != Versions.MATCH_ANY) {
-                        metadata.field("_version", request.version());
+                        metadata.field("version", request.version());
                     }
 
                     VersionType versionType = request.versionType();
                     if (versionType != VersionType.INTERNAL) {
                         if (versionType == VersionType.EXTERNAL) {
-                            metadata.field("_version_type", "external");
+                            metadata.field("version_type", "external");
                         } else if (versionType == VersionType.EXTERNAL_GTE) {
-                            metadata.field("_version_type", "external_gte");
+                            metadata.field("version_type", "external_gte");
                         } else if (versionType == VersionType.FORCE) {
-                            metadata.field("_version_type", "force");
+                            metadata.field("version_type", "force");
                         }
                     }
 
@@ -207,7 +219,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                     } else if (opType == DocWriteRequest.OpType.UPDATE) {
                         UpdateRequest updateRequest = (UpdateRequest) request;
                         if (updateRequest.retryOnConflict() > 0) {
-                            metadata.field("_retry_on_conflict", updateRequest.retryOnConflict());
+                            metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                         }
                         if (updateRequest.fetchSource() != null) {
                             metadata.field("_source", updateRequest.fetchSource());
@@ -449,6 +461,10 @@ Params withFetchSourceContext(FetchSourceContext fetchSourceContext) {
             return this;
         }
 
+        Params withMasterTimeout(TimeValue masterTimeout) {
+            return putParam("master_timeout", masterTimeout);
+        }
+
         Params withParent(String parent) {
             return putParam("parent", parent);
         }