Merge branch 'master' into pr/35195

* master: (24 commits)
  Replicate index settings to followers (elastic#35089)
  Rename RealmConfig.globalSettings() to settings() (elastic#35330)
  [TEST] Cleanup FileUserPasswdStoreTests (elastic#35329)
  Scripting: Add back lookup vars in score script (elastic#34833)
  watcher: Fix integration tests to ensure correct start/stop of Watcher (elastic#35271)
  Remove ALL shard check in CheckShrinkReadyStep (elastic#35346)
  Use soft-deleted docs to resolve strategy for engine operation (elastic#35230)
  [ILM] Check shard and relocation status in AllocationRoutedStep (elastic#35316)
  Ignore date ranges containing 'now' when pre-processing a percolator query (elastic#35160)
  Add a frozen engine implementation (elastic#34357)
  Put a fake allocation id on allocate stale primary command (elastic#34140)
  [CCR] Enforce auto follow pattern name restrictions (elastic#35197)
  [ILM] rolling upgrade tests (elastic#35328)
  [ML] Add Missing data checking class (elastic#35310)
  Apply `ignore_throttled` also to concrete indices (elastic#35335)
  Make version field names more meaningful  (elastic#35334)
  [CCR] Added HLRC support for pause follow API (elastic#35216)
  [Docs] Improve Convert Processor description (elastic#35280)
  [Painless] Removes extraneous compile method (elastic#35323)
  [CCR] Fail with a better error if leader index is red (elastic#35298)
  ...

Author: jasontedor

.ci/java-versions.properties

@@ -8,4 +8,3 @@ ES_BUILD_JAVA=java11
 ES_RUNTIME_JAVA=java8
 GRADLE_TASK=build
 
-GRADLE_EXTRA_ARGS=-Dtests.bwc.refspec=elastic/index-lifecycle-6.x

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -699,7 +699,7 @@ class BuildPlugin implements Plugin<Project> {
                 // this doFirst is added before the info plugin, therefore it will run
                 // after the doFirst added by the info plugin, and we can override attributes
                 jarTask.manifest.attributes(
-                        'X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch.replace("-SNAPSHOT", ""),
+                        'X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch,
                         'X-Compile-Lucene-Version': VersionProperties.lucene,
                         'X-Compile-Elasticsearch-Snapshot': VersionProperties.isElasticsearchSnapshot(),
                         'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.doc
 
+import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.test.RestTestPlugin
 import org.gradle.api.Project
@@ -37,12 +38,12 @@ public class DocsTestPlugin extends RestTestPlugin {
         // Docs are published separately so no need to assemble
         project.tasks.assemble.enabled = false
         Map<String, String> defaultSubstitutions = [
-            /* These match up with the asciidoc syntax for substitutions but
-             * the values may differ. In particular {version} needs to resolve
-             * to the version being built for testing but needs to resolve to
-             * the last released version for docs. */
-            '\\{version\\}':
-                VersionProperties.elasticsearch.replace('-SNAPSHOT', ''),
+                /* These match up with the asciidoc syntax for substitutions but
+                 * the values may differ. In particular {version} needs to resolve
+                 * to the version being built for testing but needs to resolve to
+                 * the last released version for docs. */
+            '\\{version\\}': Version.fromString(VersionProperties.elasticsearch).toString(),
+            '\\{version_qualified\\}': VersionProperties.elasticsearch,
             '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
             '\\{build_flavor\\}' :
                 project.integTestCluster.distribution.startsWith('oss-') ? 'oss' : 'default',

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.plugin
 
+import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Task
@@ -66,17 +67,11 @@ class PluginPropertiesTask extends Copy {
     }
 
     Map<String, String> generateSubstitutions() {
-        def stringSnap = { version ->
-            if (version.endsWith("-SNAPSHOT")) {
-               return version.substring(0, version.length() - 9)
-            }
-            return version
-        }
         return [
             'name': extension.name,
             'description': extension.description,
-            'version': stringSnap(extension.version),
-            'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
+            'version': extension.version,
+            'elasticsearchVersion': Version.fromString(VersionProperties.elasticsearch).toString(),
             'javaVersion': project.targetCompatibility as String,
             'classname': extension.classname,
             'extendedPlugins': extension.extendedPlugins.join(','),

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -278,9 +278,13 @@ class VagrantTestPlugin implements Plugin<Project> {
         }
 
         Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) {
+            String version = project.extensions.esvagrant.upgradeFromVersion
+            if (project.bwcVersions.unreleased.contains(project.extensions.esvagrant.upgradeFromVersion)) {
+                version += "-SNAPSHOT"
+            }
             dependsOn copyPackagingArchives
             file "${archivesDir}/upgrade_from_version"
-            contents project.extensions.esvagrant.upgradeFromVersion.toString()
+            contents version
         }
 
         Task createUpgradeIsOssFile = project.tasks.create('createUpgradeIsOssFile', FileContentsTask) {

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrClient.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.ccr.PauseFollowRequest;
+import org.elasticsearch.client.core.AcknowledgedResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+/**
+ * A wrapper for the {@link RestHighLevelClient} that provides methods for
+ * accessing the Elastic ccr related methods
+ * <p>
+ * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-apis.html">
+ * X-Pack Rollup APIs on elastic.co</a> for more information.
+ */
+public final class CcrClient {
+
+    private final RestHighLevelClient restHighLevelClient;
+
+    CcrClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Instructs a follower index the pause the following of a leader index.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-pause-follow.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse pauseFollow(PauseFollowRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(
+            request,
+            CcrRequestConverters::pauseFollow,
+            options,
+            AcknowledgedResponse::fromXContent,
+            Collections.emptySet()
+        );
+    }
+
+    /**
+     * Asynchronously instruct a follower index the pause the following of a leader index.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-pause-follow.html">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public void pauseFollowAsync(PauseFollowRequest request,
+                                 RequestOptions options,
+                                 ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(
+            request,
+            CcrRequestConverters::pauseFollow,
+            options,
+            AcknowledgedResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.client.ccr.PauseFollowRequest;
+
+final class CcrRequestConverters {
+
+    static Request pauseFollow(PauseFollowRequest pauseFollowRequest) {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPart(pauseFollowRequest.getFollowerIndex())
+            .addPathPartAsIs("_ccr", "pause_follow")
+            .build();
+        return new Request(HttpPost.METHOD_NAME, endpoint);
+    }
+
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -228,6 +228,7 @@ public class RestHighLevelClient implements Closeable {
     private final SecurityClient securityClient = new SecurityClient(this);
     private final IndexLifecycleClient ilmClient = new IndexLifecycleClient(this);
     private final RollupClient rollupClient = new RollupClient(this);
+    private final CcrClient ccrClient = new CcrClient(this);
 
     /**
      * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
@@ -321,6 +322,20 @@ public RollupClient rollup() {
         return rollupClient;
     }
 
+    /**
+     * Provides methods for accessing the Elastic Licensed CCR APIs that
+     * are shipped with the Elastic Stack distribution of Elasticsearch. All of
+     * these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-api.html">
+     * CCR APIs on elastic.co</a> for more information.
+     *
+     * @return the client wrapper for making CCR API calls
+     */
+    public final CcrClient ccr() {
+        return ccrClient;
+    }
+
     /**
      * Provides a {@link TasksClient} which can be used to access the Tasks API.
      *

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PauseFollowRequest.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ccr;
+
+import org.elasticsearch.client.Validatable;
+
+import java.util.Objects;
+
+public final class PauseFollowRequest implements Validatable {
+
+    private final String followerIndex;
+
+    public PauseFollowRequest(String followerIndex) {
+        this.followerIndex = Objects.requireNonNull(followerIndex);
+    }
+
+    public String getFollowerIndex() {
+        return followerIndex;
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/AcknowledgedResponse.java renamed to client/rest-high-level/src/main/java/org/elasticsearch/client/core/AcknowledgedResponse.java

@@ -17,23 +17,27 @@
  * under the License.
  */
 
-package org.elasticsearch.client.rollup;
+package org.elasticsearch.client.core;
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
 import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 
-public abstract class AcknowledgedResponse implements ToXContentObject {
+public class AcknowledgedResponse implements ToXContentObject {
 
     protected static final String PARSE_FIELD_NAME = "acknowledged";
+    private static final ConstructingObjectParser<AcknowledgedResponse, Void> PARSER = AcknowledgedResponse
+        .generateParser("acknowledged_response", AcknowledgedResponse::new, AcknowledgedResponse.PARSE_FIELD_NAME);
+
     private final boolean acknowledged;
 
     public AcknowledgedResponse(final boolean acknowledged) {
@@ -50,6 +54,10 @@ protected static <T> ConstructingObjectParser<T, Void> generateParser(String nam
         return p;
     }
 
+    public static AcknowledgedResponse fromXContent(final XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {

client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/DeleteRollupJobResponse.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.client.rollup;
 
+import org.elasticsearch.client.core.AcknowledgedResponse;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 

client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/PutRollupJobResponse.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.client.rollup;
 
+import org.elasticsearch.client.core.AcknowledgedResponse;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 

client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/StartRollupJobResponse.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.client.rollup;
 
+import org.elasticsearch.client.core.AcknowledgedResponse;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 

client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java

@@ -84,8 +84,7 @@ public void testXPackInfo() throws IOException {
         assertNotNull(ml.description());
         assertTrue(ml.available());
         assertTrue(ml.enabled());
-        assertEquals(mainResponse.getVersion().toString(),
-                ml.nativeCodeInfo().get("version").toString().replace("-SNAPSHOT", ""));
+        assertEquals(mainResponse.getBuild().getQualifiedVersion(), ml.nativeCodeInfo().get("version").toString());
     }
 
     public void testXPackInfoEmptyRequest() throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -785,7 +785,8 @@ public void testApiNamingConventions() throws Exception {
                             apiName.startsWith("graph.") == false &&
                             apiName.startsWith("migration.") == false &&
                             apiName.startsWith("security.") == false &&
-                            apiName.startsWith("index_lifecycle.") == false) {
+                            apiName.startsWith("index_lifecycle.") == false &&
+                            apiName.startsWith("ccr.") == false) {
                             apiNotFound.add(apiName);
                         }
                     }