Merge remote-tracking branch 'es/ccr' into ccr_security

* es/ccr: (37 commits)
  Default to one shard (#30539)
  Unmute IndexUpgradeIT tests
  Forbid expensive query parts in ranking evaluation (#30151)
  Docs: Update HighLevelRestClient migration docs (#30544)
  Clients: Switch to new performRequest (#30543)
  [TEST] Fix typo in MovAvgIT test
  Add missing dependencies on testClasses (#30527)
  [TEST] Mute ML test that needs updating to following ml-cpp changes
  Document woes between auto-expand-replicas and allocation filtering (#30531)
  Moved tokenizers to analysis common module (#30538)
  Adjust copy settings versions
  Mute ShrinkIndexIT suite
  SQL: SYS TABLES ordered according to *DBC specs (#30530)
  Deprecate not copy settings and explicitly disallow (#30404)
  [ML] Improve state persistence log message
  Build: Add mavenPlugin cluster configuration method (#30541)
  Re-enable FlushIT tests
  Bump Gradle heap to 2 GB (#30535)
  SQL: Use request flavored methods in tests (#30345)
  Suppress hdfsFixture if there are spaces in the path (#30302)
  ...

Author: martijnvg

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -744,6 +744,7 @@ class BuildPlugin implements Plugin<Project> {
             additionalTest.testClassesDir = test.testClassesDir
             additionalTest.configure(commonTestConfig(project))
             additionalTest.configure(config)
+            additionalTest.dependsOn(project.tasks.testClasses)
             test.dependsOn(additionalTest)
         });
         return test

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -225,6 +225,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                  * warning every time. */
                 current.println("  - skip:")
                 current.println("      features: ")
+                current.println("        - default_shards")
                 current.println("        - stash_in_key")
                 current.println("        - stash_in_path")
                 current.println("        - stash_path_replace")

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -147,7 +147,7 @@ class ClusterConfiguration {
     // map from destination path, to source file
     Map<String, Object> extraConfigFiles = new HashMap<>()
 
-    LinkedHashMap<String, Project> plugins = new LinkedHashMap<>()
+    LinkedHashMap<String, Object> plugins = new LinkedHashMap<>()
 
     List<Project> modules = new ArrayList<>()
 
@@ -185,6 +185,11 @@ class ClusterConfiguration {
         plugins.put(pluginProject.name, pluginProject)
     }
 
+    @Input
+    void mavenPlugin(String name, String mavenCoords) {
+        plugins.put(name, mavenCoords)
+    }
+
     /** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */
     @Input
     void module(Project moduleProject) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -99,8 +99,8 @@ class ClusterFormationTasks {
             // from mirrors using gradles built-in mechanism etc.
 
             configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion)
-            for (Map.Entry<String, Project> entry : config.plugins.entrySet()) {
-                configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(), bwcPlugins, config.bwcVersion)
+            for (Map.Entry<String, Object> entry : config.plugins.entrySet()) {
+                configureBwcPluginDependency(project, entry.getValue(), bwcPlugins, config.bwcVersion)
             }
             bwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
             bwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
@@ -150,10 +150,15 @@ class ClusterFormationTasks {
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
-    static void configureBwcPluginDependency(String name, Project project, Project pluginProject, Configuration configuration, Version elasticsearchVersion) {
-        verifyProjectHasBuildPlugin(name, elasticsearchVersion, project, pluginProject)
-        final String pluginName = findPluginName(pluginProject)
-        project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
+    static void configureBwcPluginDependency(Project project, Object plugin, Configuration configuration, Version elasticsearchVersion) {
+        if (plugin instanceof Project) {
+            Project pluginProject = (Project)plugin
+            verifyProjectHasBuildPlugin(configuration.name, elasticsearchVersion, project, pluginProject)
+            final String pluginName = findPluginName(pluginProject)
+            project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
+        } else {
+            project.dependencies.add(configuration.name, "${plugin}@zip")
+        }
     }
 
     /**
@@ -210,9 +215,9 @@ class ClusterFormationTasks {
         }
 
         // install plugins
-        for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
-            String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
-            setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue(), prefix)
+        for (String pluginName : node.config.plugins.keySet()) {
+            String actionName = pluginTaskName('install', pluginName, 'Plugin')
+            setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, pluginName, prefix)
         }
 
         // sets up any extra config files that need to be copied over to the ES instance;
@@ -444,31 +449,40 @@ class ClusterFormationTasks {
         Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup)
 
         List<FileCollection> pluginFiles = []
-        for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
+        for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {
 
-            Project pluginProject = plugin.getValue()
-            verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
-            String configurationName = pluginConfigurationName(prefix, pluginProject)
+            String configurationName = pluginConfigurationName(prefix, plugin.key)
             Configuration configuration = project.configurations.findByName(configurationName)
             if (configuration == null) {
                 configuration = project.configurations.create(configurationName)
             }
-            project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip'))
-            setup.dependsOn(pluginProject.tasks.bundlePlugin)
-
-            // also allow rest tests to use the rest spec from the plugin
-            String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec')
-            Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName)
-            for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
-                File restApiDir = new File(resourceDir, 'rest-api-spec/api')
-                if (restApiDir.exists() == false) continue
-                if (copyRestSpec == null) {
-                    copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy)
-                    copyPlugins.dependsOn(copyRestSpec)
-                    copyRestSpec.into(project.sourceSets.test.output.resourcesDir)
+
+            if (plugin.getValue() instanceof Project) {
+                Project pluginProject = plugin.getValue()
+                verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
+
+                project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip'))
+                setup.dependsOn(pluginProject.tasks.bundlePlugin)
+
+                // also allow rest tests to use the rest spec from the plugin
+                String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec')
+                Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName)
+                for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
+                    File restApiDir = new File(resourceDir, 'rest-api-spec/api')
+                    if (restApiDir.exists() == false) continue
+                    if (copyRestSpec == null) {
+                        copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy)
+                        copyPlugins.dependsOn(copyRestSpec)
+                        copyRestSpec.into(project.sourceSets.test.output.resourcesDir)
+                    }
+                    copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
                 }
-                copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
+            } else {
+                project.dependencies.add(configurationName, "${plugin.getValue()}@zip")
             }
+
+
+
             pluginFiles.add(configuration)
         }
 
@@ -477,32 +491,37 @@ class ClusterFormationTasks {
         return copyPlugins
     }
 
-    private static String pluginConfigurationName(final String prefix, final Project project) {
-        return "_plugin_${prefix}_${project.path}".replace(':', '_')
+    private static String pluginConfigurationName(final String prefix, final String name) {
+        return "_plugin_${prefix}_${name}".replace(':', '_')
     }
 
-    private static String pluginBwcConfigurationName(final String prefix, final Project project) {
-        return "_plugin_bwc_${prefix}_${project.path}".replace(':', '_')
+    private static String pluginBwcConfigurationName(final String prefix, final String name) {
+        return "_plugin_bwc_${prefix}_${name}".replace(':', '_')
     }
 
     /** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */
     static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
         Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins")
-        for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
-            Project pluginProject = plugin.getValue()
-            verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
-            String configurationName = pluginBwcConfigurationName(prefix, pluginProject)
+        for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {
+            String configurationName = pluginBwcConfigurationName(prefix, plugin.key)
             Configuration configuration = project.configurations.findByName(configurationName)
             if (configuration == null) {
                 configuration = project.configurations.create(configurationName)
             }
 
-            final String depName = findPluginName(pluginProject)
+            if (plugin.getValue() instanceof Project) {
+                Project pluginProject = plugin.getValue()
+                verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
 
-            Dependency dep = bwcPlugins.dependencies.find {
-                it.name == depName
+                final String depName = findPluginName(pluginProject)
+
+                Dependency dep = bwcPlugins.dependencies.find {
+                    it.name == depName
+                }
+                configuration.dependencies.add(dep)
+            } else {
+                project.dependencies.add(configurationName, "${plugin.getValue()}@zip")
             }
-            configuration.dependencies.add(dep)
         }
 
         Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) {
@@ -527,12 +546,12 @@ class ClusterFormationTasks {
         return installModule
     }
 
-    static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) {
+    static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, String pluginName, String prefix) {
         final FileCollection pluginZip;
         if (node.nodeVersion != VersionProperties.elasticsearch) {
-            pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, plugin))
+            pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, pluginName))
         } else {
-            pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, plugin))
+            pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, pluginName))
         }
         // delay reading the file location until execution time by wrapping in a closure within a GString
         final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"

client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java

@@ -18,27 +18,19 @@
  */
 package org.elasticsearch.client.benchmark.rest;
 
-import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpStatus;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.conn.ConnectionKeepAliveStrategy;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.message.BasicHeader;
-import org.apache.http.nio.entity.NStringEntity;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.benchmark.AbstractBenchmark;
 import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor;
 import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -86,9 +78,10 @@ public boolean bulkIndex(List<String> bulkData) {
                 bulkRequestBody.append(bulkItem);
                 bulkRequestBody.append("\n");
             }
-            HttpEntity entity = new NStringEntity(bulkRequestBody.toString(), ContentType.APPLICATION_JSON);
+            Request request = new Request("POST", "/geonames/type/_noop_bulk");
+            request.setJsonEntity(bulkRequestBody.toString());
             try {
-                Response response = client.performRequest("POST", "/geonames/type/_noop_bulk", Collections.emptyMap(), entity);
+                Response response = client.performRequest(request);
                 return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
             } catch (Exception e) {
                 throw new ElasticsearchException(e);
@@ -107,9 +100,10 @@ private RestSearchRequestExecutor(RestClient client, String indexName) {
 
         @Override
         public boolean search(String source) {
-            HttpEntity searchBody = new NStringEntity(source, StandardCharsets.UTF_8);
+            Request request = new Request("GET", endpoint);
+            request.setJsonEntity(source);
             try {
-                Response response = client.performRequest("GET", endpoint, Collections.emptyMap(), searchBody);
+                Response response = client.performRequest(request);
                 return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
             } catch (IOException e) {
                 throw new ElasticsearchException(e);

client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java

@@ -194,18 +194,16 @@ public void testBulkProcessorWaitOnClose() throws Exception {
     }
 
     public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception {
-
-        String createIndexBody = "{\n" +
+        Request request = new Request("PUT", "/test-ro");
+        request.setJsonEntity("{\n" +
                 "    \"settings\" : {\n" +
                 "        \"index\" : {\n" +
                 "            \"blocks.write\" : true\n" +
                 "        }\n" +
                 "    }\n" +
                 "    \n" +
-                "}";
-
-        NStringEntity entity = new NStringEntity(createIndexBody, ContentType.APPLICATION_JSON);
-        Response response = client().performRequest("PUT", "/test-ro", Collections.emptyMap(), entity);
+                "}");
+        Response response = client().performRequest(request);
         assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
 
         int bulkActions = randomIntBetween(10, 100);

client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java

@@ -19,9 +19,6 @@
 
 package org.elasticsearch.client;
 
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.DocWriteRequest;
@@ -39,6 +36,7 @@
 import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.common.Strings;
@@ -147,11 +145,10 @@ public void testExists() throws IOException {
             GetRequest getRequest = new GetRequest("index", "type", "id");
             assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
         }
-        String document = "{\"field1\":\"value1\",\"field2\":\"value2\"}";
-        StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
-        Response response = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id", Collections.singletonMap("refresh", "wait_for"),
-                stringEntity);
-        assertEquals(201, response.getStatusLine().getStatusCode());
+        IndexRequest index = new IndexRequest("index", "type", "id");
+        index.source("{\"field1\":\"value1\",\"field2\":\"value2\"}", XContentType.JSON);
+        index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        highLevelClient().index(index);
         {
             GetRequest getRequest = new GetRequest("index", "type", "id");
             assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
@@ -175,12 +172,11 @@ public void testGet() throws IOException {
             assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]", exception.getMessage());
             assertEquals("index", exception.getMetadata("es.index").get(0));
         }
-
+        IndexRequest index = new IndexRequest("index", "type", "id");
         String document = "{\"field1\":\"value1\",\"field2\":\"value2\"}";
-        StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
-        Response response = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id", Collections.singletonMap("refresh", "wait_for"),
-                stringEntity);
-        assertEquals(201, response.getStatusLine().getStatusCode());
+        index.source(document, XContentType.JSON);
+        index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        highLevelClient().index(index);
         {
             GetRequest getRequest = new GetRequest("index", "type", "id").version(2);
             ElasticsearchException exception = expectThrows(ElasticsearchException.class,
@@ -271,18 +267,15 @@ public void testMultiGet() throws IOException {
             assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]",
                     response.getResponses()[1].getFailure().getFailure().getMessage());
         }
-
-        String document = "{\"field\":\"value1\"}";
-        StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
-        Response r = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id1", Collections.singletonMap("refresh", "true"),
-                stringEntity);
-        assertEquals(201, r.getStatusLine().getStatusCode());
-
-        document = "{\"field\":\"value2\"}";
-        stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
-        r = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id2", Collections.singletonMap("refresh", "true"), stringEntity);
-        assertEquals(201, r.getStatusLine().getStatusCode());
-
+        BulkRequest bulk = new BulkRequest();
+        bulk.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        IndexRequest index = new IndexRequest("index", "type", "id1");
+        index.source("{\"field\":\"value1\"}", XContentType.JSON);
+        bulk.add(index);
+        index = new IndexRequest("index", "type", "id2");
+        index.source("{\"field\":\"value2\"}", XContentType.JSON);
+        bulk.add(index);
+        highLevelClient().bulk(bulk);
         {
             MultiGetRequest multiGetRequest = new MultiGetRequest();
             multiGetRequest.add("index", "type", "id1");