Merge branch 'master' into replicated-closed-indices

Author: tlrx

Vagrantfile

@@ -346,6 +346,11 @@ def sh_install_deps(config,
       echo "==> Java is not installed"
       return 1
     }
+    cat \<\<JAVA > /etc/profile.d/java_home.sh
+if [ -z "\\\$JAVA_HOME" ]; then
+  export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))
+fi
+JAVA
     ensure tar
     ensure curl
     ensure unzip
@@ -382,6 +387,7 @@ Defaults   env_keep += "BATS_UTILS"
 Defaults   env_keep += "BATS_TESTS"
 Defaults   env_keep += "PACKAGING_ARCHIVES"
 Defaults   env_keep += "PACKAGING_TESTS"
+Defaults   env_keep += "JAVA_HOME"
 SUDOERS_VARS
     chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL

build.gradle

@@ -162,7 +162,6 @@ task verifyVersions {
 
 boolean bwc_tests_enabled = true
 final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
-
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -20,6 +20,7 @@ package org.elasticsearch.gradle.doc
 
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.test.ClusterFormationTasks
 import org.elasticsearch.gradle.test.RestTestPlugin
 import org.gradle.api.Project
 import org.gradle.api.Task
@@ -46,6 +47,7 @@ public class DocsTestPlugin extends RestTestPlugin {
             '\\{version_qualified\\}': VersionProperties.elasticsearch,
             '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
             '\\{build_flavor\\}' : project.integTestCluster.distribution,
+            '\\{build_type\\}' : ClusterFormationTasks.getOs().equals("windows") ? "zip" : "tar",
         ]
         Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)
         listSnippets.group 'Docs'

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -274,7 +274,7 @@ class ClusterFormationTasks {
         }
         setup = configureCheckPreviousTask(taskName(prefix, node, 'checkPrevious'), project, setup, node)
         setup = configureStopTask(taskName(prefix, node, 'stopPrevious'), project, setup, node)
-        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution)
+        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution, config.distribution)
         setup = configureWriteConfigTask(taskName(prefix, node, 'configure'), project, setup, node, writeConfig)
         setup = configureCreateKeystoreTask(taskName(prefix, node, 'createKeystore'), project, setup, node)
         setup = configureAddKeystoreSettingTasks(prefix, project, setup, node)
@@ -343,14 +343,15 @@ class ClusterFormationTasks {
     }
 
     /** Adds a task to extract the elasticsearch distribution */
-    static Task configureExtractTask(String name, Project project, Task setup, NodeInfo node, Configuration configuration) {
+    static Task configureExtractTask(String name, Project project, Task setup, NodeInfo node,
+                                     Configuration configuration, String distribution) {
         List extractDependsOn = [configuration, setup]
         /* configuration.singleFile will be an external artifact if this is being run by a plugin not living in the
           elasticsearch source tree. If this is a plugin built in the elasticsearch source tree or this is a distro in
           the elasticsearch source tree then this should be the version of elasticsearch built by the source tree.
           If it isn't then Bad Things(TM) will happen. */
         Task extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-            if (getOs().equals("windows")) {
+            if (getOs().equals("windows") || distribution.equals("integ-test-zip")) {
                 from {
                     project.zipTree(configuration.singleFile)
                 }
@@ -971,9 +972,9 @@ class ClusterFormationTasks {
     /** Find the current OS */
     static String getOs() {
         String os = "linux"
-        if (Os.FAMILY_WINDOWS) {
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
             os = "windows"
-        } else if (Os.FAMILY_MAC) {
+        } else if (Os.isFamily(Os.FAMILY_MAC)) {
             os = "darwin"
         }
         return os

buildSrc/src/main/resources/forbidden/es-server-signatures.txt

@@ -61,7 +61,7 @@ java.nio.channels.FileChannel#read(java.nio.ByteBuffer, long)
 @defaultMessage Use Lucene.parseLenient instead it strips off minor version
 org.apache.lucene.util.Version#parseLeniently(java.lang.String)
 
-@defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#estimatedTimeInMillisCounter instead
+@defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#relativeTimeInMillis instead
 org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread()
 org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter()
 
@@ -146,4 +146,4 @@ org.apache.logging.log4j.Logger#warn(java.lang.Object, java.lang.Throwable)
 org.apache.logging.log4j.Logger#error(java.lang.Object)
 org.apache.logging.log4j.Logger#error(java.lang.Object, java.lang.Throwable)
 org.apache.logging.log4j.Logger#fatal(java.lang.Object)
-org.apache.logging.log4j.Logger#fatal(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#fatal(java.lang.Object, java.lang.Throwable)

buildSrc/version.properties

@@ -4,6 +4,9 @@ lucene            = 8.0.0-snapshot-83f9835
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
+# note that ingest-geoip has a hard-coded version; if you modify this version,
+# you should also inspect that version to see if it can be advanced along with
+# the com.maxmind.geoip2:geoip2 dependency
 jackson           = 2.8.11
 snakeyaml         = 1.17
 icu4j             = 62.1

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrClient.java

@@ -23,6 +23,8 @@
 import org.elasticsearch.client.ccr.CcrStatsRequest;
 import org.elasticsearch.client.ccr.CcrStatsResponse;
 import org.elasticsearch.client.ccr.DeleteAutoFollowPatternRequest;
+import org.elasticsearch.client.ccr.FollowInfoRequest;
+import org.elasticsearch.client.ccr.FollowInfoResponse;
 import org.elasticsearch.client.ccr.FollowStatsRequest;
 import org.elasticsearch.client.ccr.FollowStatsResponse;
 import org.elasticsearch.client.ccr.GetAutoFollowPatternRequest;
@@ -452,4 +454,46 @@ public void getFollowStatsAsync(FollowStatsRequest request,
         );
     }
 
+    /**
+     * Gets follow info for specific indices.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-get-follow-info.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public FollowInfoResponse getFollowInfo(FollowInfoRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(
+            request,
+            CcrRequestConverters::getFollowInfo,
+            options,
+            FollowInfoResponse::fromXContent,
+            Collections.emptySet()
+        );
+    }
+
+    /**
+     * Asynchronously gets follow info for specific indices.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-get-follow-info.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public void getFollowInfoAsync(FollowInfoRequest request,
+                                   RequestOptions options,
+                                   ActionListener<FollowInfoResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(
+            request,
+            CcrRequestConverters::getFollowInfo,
+            options,
+            FollowInfoResponse::fromXContent,
+            listener,
+            Collections.emptySet()
+        );
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java

@@ -25,6 +25,7 @@
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.ccr.CcrStatsRequest;
 import org.elasticsearch.client.ccr.DeleteAutoFollowPatternRequest;
+import org.elasticsearch.client.ccr.FollowInfoRequest;
 import org.elasticsearch.client.ccr.FollowStatsRequest;
 import org.elasticsearch.client.ccr.GetAutoFollowPatternRequest;
 import org.elasticsearch.client.ccr.PauseFollowRequest;
@@ -119,4 +120,12 @@ static Request getFollowStats(FollowStatsRequest followStatsRequest) {
         return new Request(HttpGet.METHOD_NAME, endpoint);
     }
 
+    static Request getFollowInfo(FollowInfoRequest followInfoRequest) {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPart(followInfoRequest.getFollowerIndex())
+            .addPathPartAsIs("_ccr", "info")
+            .build();
+        return new Request(HttpGet.METHOD_NAME, endpoint);
+    }
+
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/FollowConfig.java

@@ -22,8 +22,10 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -41,6 +43,44 @@ public class FollowConfig {
     static final ParseField MAX_RETRY_DELAY_FIELD = new ParseField("max_retry_delay");
     static final ParseField READ_POLL_TIMEOUT = new ParseField("read_poll_timeout");
 
+    private static final ObjectParser<FollowConfig, Void> PARSER = new ObjectParser<>(
+        "follow_config",
+        true,
+        FollowConfig::new);
+
+    static {
+        PARSER.declareInt(FollowConfig::setMaxReadRequestOperationCount, MAX_READ_REQUEST_OPERATION_COUNT);
+        PARSER.declareInt(FollowConfig::setMaxOutstandingReadRequests, MAX_OUTSTANDING_READ_REQUESTS);
+        PARSER.declareField(
+            FollowConfig::setMaxReadRequestSize,
+            (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_READ_REQUEST_SIZE.getPreferredName()),
+            MAX_READ_REQUEST_SIZE,
+            ObjectParser.ValueType.STRING);
+        PARSER.declareInt(FollowConfig::setMaxWriteRequestOperationCount, MAX_WRITE_REQUEST_OPERATION_COUNT);
+        PARSER.declareField(
+            FollowConfig::setMaxWriteRequestSize,
+            (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_WRITE_REQUEST_SIZE.getPreferredName()),
+            MAX_WRITE_REQUEST_SIZE,
+            ObjectParser.ValueType.STRING);
+        PARSER.declareInt(FollowConfig::setMaxOutstandingWriteRequests, MAX_OUTSTANDING_WRITE_REQUESTS);
+        PARSER.declareInt(FollowConfig::setMaxWriteBufferCount, MAX_WRITE_BUFFER_COUNT);
+        PARSER.declareField(
+            FollowConfig::setMaxWriteBufferSize,
+            (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_WRITE_BUFFER_SIZE.getPreferredName()),
+            MAX_WRITE_BUFFER_SIZE,
+            ObjectParser.ValueType.STRING);
+        PARSER.declareField(FollowConfig::setMaxRetryDelay,
+            (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_RETRY_DELAY_FIELD.getPreferredName()),
+            MAX_RETRY_DELAY_FIELD, ObjectParser.ValueType.STRING);
+        PARSER.declareField(FollowConfig::setReadPollTimeout,
+            (p, c) -> TimeValue.parseTimeValue(p.text(), READ_POLL_TIMEOUT.getPreferredName()),
+            READ_POLL_TIMEOUT, ObjectParser.ValueType.STRING);
+    }
+
+    static FollowConfig fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
     private Integer maxReadRequestOperationCount;
     private Integer maxOutstandingReadRequests;
     private ByteSizeValue maxReadRequestSize;

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/FollowInfoRequest.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ccr;
+
+import org.elasticsearch.client.Validatable;
+
+import java.util.Objects;
+
+public final class FollowInfoRequest implements Validatable {
+
+    private final String followerIndex;
+
+    public FollowInfoRequest(String followerIndex) {
+        this.followerIndex = Objects.requireNonNull(followerIndex);
+    }
+
+    public String getFollowerIndex() {
+        return followerIndex;
+    }
+}