Merge branch '6.x' into feature-jindex-6x

Author: davidkyle

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -215,6 +215,7 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.inFipsJvm = inFipsJvm
             project.rootProject.ext.gradleJavaVersion = JavaVersion.toVersion(gradleJavaVersion)
             project.rootProject.ext.java9Home = "${-> findJavaHome("9")}"
+            project.rootProject.ext.defaultParallel = findDefaultParallel(project.rootProject)
         }
 
         project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
@@ -774,32 +775,9 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     static void applyCommonTestConfig(Project project) {
-        String defaultParallel = 'auto'
-        // Count physical cores on any Linux distro ( don't count hyper-threading )
-        if (project.file("/proc/cpuinfo").exists()) {
-            Map<String, Integer> socketToCore = [:]
-            String currentID = ""
-            project.file("/proc/cpuinfo").readLines().forEach({ line ->
-                if (line.contains(":")) {
-                    List<String> parts = line.split(":", 2).collect({it.trim()})
-                    String name = parts[0], value = parts[1]
-                    // the ID of the CPU socket
-                    if (name == "physical id") {
-                        currentID = value
-                    }
-                    // Number  of cores not including hyper-threading
-                    if (name == "cpu cores") {
-                        assert currentID.isEmpty() == false
-                        socketToCore[currentID] = Integer.valueOf(value)
-                        currentID = ""
-                    }
-                }
-            })
-            defaultParallel = socketToCore.values().sum().toString();
-        }
         project.tasks.withType(RandomizedTestingTask) {
             jvm "${project.runtimeJavaHome}/bin/java"
-            parallelism System.getProperty('tests.jvms', defaultParallel)
+            parallelism System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel)
             ifNoTests System.getProperty('tests.ifNoTests', 'fail')
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
@@ -849,6 +827,9 @@ class BuildPlugin implements Plugin<Project> {
             // TODO: remove this once ctx isn't added to update script params in 7.0
             systemProperty 'es.scripting.update.ctx_in_params', 'false'
 
+            //TODO: remove this once the cname is prepended to the address by default in 7.0
+            systemProperty 'es.http.cname_in_publish_address', 'true'
+
             // Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
             if (project.inFipsJvm) {
                 systemProperty 'javax.net.ssl.trustStorePassword', 'password'
@@ -905,6 +886,41 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
+    private static String findDefaultParallel(Project project) {
+        if (project.file("/proc/cpuinfo").exists()) {
+            // Count physical cores on any Linux distro ( don't count hyper-threading )
+            Map<String, Integer> socketToCore = [:]
+            String currentID = ""
+            project.file("/proc/cpuinfo").readLines().forEach({ line ->
+                if (line.contains(":")) {
+                    List<String> parts = line.split(":", 2).collect({it.trim()})
+                    String name = parts[0], value = parts[1]
+                    // the ID of the CPU socket
+                    if (name == "physical id") {
+                        currentID = value
+                    }
+                    // Number  of cores not including hyper-threading
+                    if (name == "cpu cores") {
+                        assert currentID.isEmpty() == false
+                        socketToCore[currentID] = Integer.valueOf(value)
+                        currentID = ""
+                    }
+                }
+            })
+            return socketToCore.values().sum().toString();
+        } else if ('Mac OS X'.equals(System.getProperty('os.name'))) {
+            // Ask macOS to count physical CPUs for us
+            ByteArrayOutputStream stdout = new ByteArrayOutputStream()
+            project.exec {
+                executable 'sysctl'
+                args '-n', 'hw.physicalcpu'
+                standardOutput = stdout
+            }
+            return stdout.toString('UTF-8').trim();
+        }
+        return 'auto';
+    }
+
     /** Configures the test task */
     static Task configureTest(Project project) {
         project.tasks.getByName('test') {

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -174,16 +174,7 @@
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CategoryContextMappingTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]GeoContextMappingTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]NoisyChannelSpellCheckerTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]similarity[/\\]SimilarityIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]AbstractSnapshotIntegTestCase.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]DedicatedClusterSnapshotRestoreIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]RepositoriesIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SharedClusterSnapshotRestoreIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotUtilsTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]geo[/\\]RandomShapeGenerator.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchGeoAssertions.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]update[/\\]UpdateIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]validate[/\\]SimpleValidateQueryIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]versioning[/\\]SimpleVersioningIT.java" checks="LineLength" />
+
+ <!-- Temporarily contains extra-long lines as examples for tests to be written, see https://github.com/elastic/elasticsearch/issues/34829 -->
   <suppress files="modules[/\\]lang-painless[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]ContextExampleTests.java" checks="LineLength" />
 </suppressions>

buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs

@@ -16,6 +16,7 @@ eclipse.preferences.version=1
 # org.eclipse.jdt.core.compiler.problem.potentialNullReference=warning
 
 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.formatter.comment.line_length=140
 org.eclipse.jdt.core.formatter.lineSplit=140
 org.eclipse.jdt.core.formatter.tabulation.char=space
 org.eclipse.jdt.core.formatter.tabulation.size=4

buildSrc/version.properties

@@ -15,7 +15,7 @@ slf4j             = 1.6.2
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 4.5.1
 
-netty             = 4.1.30.Final
+netty             = 4.1.31.Final
 joda              = 2.10.1
 
 # test dependencies

client/benchmark/README.md

@@ -1,7 +1,7 @@
 ### Steps to execute the benchmark
 
-1. Build `client-benchmark-noop-api-plugin` with `gradle :client:client-benchmark-noop-api-plugin:assemble`
-2. Install it on the target host with `bin/elasticsearch-plugin install file:///full/path/to/client-benchmark-noop-api-plugin.zip`
+1. Build `client-benchmark-noop-api-plugin` with `./gradlew :client:client-benchmark-noop-api-plugin:assemble`
+2. Install it on the target host with `bin/elasticsearch-plugin install file:///full/path/to/client-benchmark-noop-api-plugin.zip`.
 3. Start Elasticsearch on the target host (ideally *not* on the machine
 that runs the benchmarks)
 4. Run the benchmark with
@@ -49,7 +49,7 @@ The parameters are all in the `'`s and are in order:
 Example invocation:
 
 ```
-gradlew -p client/benchmark run --args ' rest search localhost geonames {"query":{"match_phrase":{"name":"Sankt Georgen"}}} 500,1000,1100,1200'
+./gradlew -p client/benchmark run --args ' rest search localhost geonames {"query":{"match_phrase":{"name":"Sankt Georgen"}}} 500,1000,1100,1200'
 ```
 
 The parameters are in order:

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrClient.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.ccr.PauseFollowRequest;
+import org.elasticsearch.client.ccr.PutAutoFollowPatternRequest;
 import org.elasticsearch.client.ccr.PutFollowRequest;
 import org.elasticsearch.client.ccr.PutFollowResponse;
 import org.elasticsearch.client.ccr.ResumeFollowRequest;
@@ -219,4 +220,46 @@ public void unfollowAsync(UnfollowRequest request,
         );
     }
 
+    /**
+     * Stores an auto follow pattern.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-auto-follow-pattern.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse putAutoFollowPattern(PutAutoFollowPatternRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(
+            request,
+            CcrRequestConverters::putAutoFollowPattern,
+            options,
+            AcknowledgedResponse::fromXContent,
+            Collections.emptySet()
+        );
+    }
+
+    /**
+     * Asynchronously stores an auto follow pattern.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-auto-follow-pattern.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public void putAutoFollowPatternAsync(PutAutoFollowPatternRequest request,
+                                          RequestOptions options,
+                                          ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(
+            request,
+            CcrRequestConverters::putAutoFollowPattern,
+            options,
+            AcknowledgedResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java

@@ -22,6 +22,7 @@
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.ccr.PauseFollowRequest;
+import org.elasticsearch.client.ccr.PutAutoFollowPatternRequest;
 import org.elasticsearch.client.ccr.PutFollowRequest;
 import org.elasticsearch.client.ccr.ResumeFollowRequest;
 import org.elasticsearch.client.ccr.UnfollowRequest;
@@ -69,4 +70,14 @@ static Request unfollow(UnfollowRequest unfollowRequest) {
         return new Request(HttpPost.METHOD_NAME, endpoint);
     }
 
+    static Request putAutoFollowPattern(PutAutoFollowPatternRequest putAutoFollowPatternRequest) throws IOException {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPartAsIs("_ccr", "auto_follow")
+            .addPathPart(putAutoFollowPatternRequest.getName())
+            .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(putAutoFollowPatternRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleRequestConverters.java

@@ -126,10 +126,9 @@ static Request lifecycleManagementStatus(LifecycleManagementStatusRequest lifecy
     }
 
     static Request explainLifecycle(ExplainLifecycleRequest explainLifecycleRequest) {
-        String[] indices = explainLifecycleRequest.indices() == null ? Strings.EMPTY_ARRAY : explainLifecycleRequest.indices();
         Request request = new Request(HttpGet.METHOD_NAME,
             new RequestConverters.EndpointBuilder()
-                .addCommaSeparatedPathParts(indices)
+                .addCommaSeparatedPathParts(explainLifecycleRequest.getIndices())
                 .addPathPartAsIs("_ilm")
                 .addPathPartAsIs("explain")
             .build());

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -60,6 +60,9 @@
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.core.ShardsAcknowledgedResponse;
+import org.elasticsearch.client.indices.FreezeIndexRequest;
+import org.elasticsearch.client.indices.UnfreezeIndexRequest;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -1266,4 +1269,51 @@ public void analyzeAsync(AnalyzeRequest request, RequestOptions options,
         restHighLevelClient.performRequestAsyncAndParseEntity(request, IndicesRequestConverters::analyze, options,
             AnalyzeResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Synchronously calls the _freeze API
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public ShardsAcknowledgedResponse freeze(FreezeIndexRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndicesRequestConverters::freezeIndex, options,
+            ShardsAcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously calls the _freeze API
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void freezeAsync(FreezeIndexRequest request, RequestOptions options, ActionListener<ShardsAcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndicesRequestConverters::freezeIndex, options,
+            ShardsAcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Synchronously calls the _unfreeze API
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public ShardsAcknowledgedResponse unfreeze(UnfreezeIndexRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndicesRequestConverters::unfreezeIndex, options,
+            ShardsAcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously calls the _unfreeze API
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void unfreezeAsync(UnfreezeIndexRequest request, RequestOptions options, ActionListener<ShardsAcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndicesRequestConverters::unfreezeIndex, options,
+            ShardsAcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java

@@ -49,6 +49,8 @@
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.client.indices.FreezeIndexRequest;
+import org.elasticsearch.client.indices.UnfreezeIndexRequest;
 import org.elasticsearch.common.Strings;
 
 import java.io.IOException;
@@ -404,4 +406,26 @@ static Request analyze(AnalyzeRequest request) throws IOException {
         req.setEntity(RequestConverters.createEntity(request, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return req;
     }
+
+    static Request freezeIndex(FreezeIndexRequest freezeIndexRequest) {
+        String endpoint = RequestConverters.endpoint(freezeIndexRequest.getIndices(), "_freeze");
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        parameters.withTimeout(freezeIndexRequest.timeout());
+        parameters.withMasterTimeout(freezeIndexRequest.masterNodeTimeout());
+        parameters.withIndicesOptions(freezeIndexRequest.indicesOptions());
+        parameters.withWaitForActiveShards(freezeIndexRequest.getWaitForActiveShards(), ActiveShardCount.DEFAULT);
+        return request;
+    }
+
+    static Request unfreezeIndex(UnfreezeIndexRequest unfreezeIndexRequest) {
+        String endpoint = RequestConverters.endpoint(unfreezeIndexRequest.getIndices(), "_unfreeze");
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        parameters.withTimeout(unfreezeIndexRequest.timeout());
+        parameters.withMasterTimeout(unfreezeIndexRequest.masterNodeTimeout());
+        parameters.withIndicesOptions(unfreezeIndexRequest.indicesOptions());
+        parameters.withWaitForActiveShards(unfreezeIndexRequest.getWaitForActiveShards(), ActiveShardCount.DEFAULT);
+        return request;
+    }
 }