Merge branch 'master' into zendisco2

Author: DaveCTurner

TESTING.asciidoc

@@ -25,6 +25,46 @@ run it using Gradle:
 ./gradlew run
 -------------------------------------
 
+==== Launching and debugging from an IDE
+
+If you want to run Elasticsearch from your IDE, the `./gradlew run` task
+supports a remote debugging option:
+
+---------------------------------------------------------------------------
+./gradlew run --debug-jvm
+---------------------------------------------------------------------------
+
+==== Distribution
+
+By default a node is started with the zip distribution.
+In order to start with a different distribution use the `-Drun.distribution` argument.
+
+To for example start the open source distribution:
+
+-------------------------------------
+./gradlew run -Drun.distribution=oss-zip
+-------------------------------------
+
+==== License type
+
+By default a node is started with the `basic` license type.
+In order to start with a different license type use the `-Drun.license_type` argument.
+
+In order to start a node with a trial license execute the following command:
+
+-------------------------------------
+./gradlew run -Drun.license_type=trial
+-------------------------------------
+
+This enables security and other paid features and adds a superuser with the username: `elastic-admin` and
+password: `elastic-password`.
+
+==== Other useful arguments
+
+In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
+In order to disable annotations add: `-Dtests.asserts=false`
+In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
+
 === Test case filtering.
 
 - `tests.class` is a class-filtering shell-like glob pattern,
@@ -572,15 +612,6 @@ as its build system. Since the switch to Gradle though, this is no longer possib
 the code currently used to build Elasticsearch does not allow JaCoCo to recognize its tests.
 For more information on this, see the discussion in https://github.com/elastic/elasticsearch/issues/28867[issue #28867].
 
-== Launching and debugging from an IDE
-
-If you want to run Elasticsearch from your IDE, the `./gradlew run` task
-supports a remote debugging option:
-
----------------------------------------------------------------------------
-./gradlew run --debug-jvm
----------------------------------------------------------------------------
-
 == Debugging remotely from an IDE
 
 If you want to run Elasticsearch and be able to remotely attach the process

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -642,7 +642,12 @@ public void testShrink() throws IOException {
 
         ResizeRequest resizeRequest = new ResizeRequest("target", "source");
         resizeRequest.setResizeType(ResizeType.SHRINK);
-        Settings targetSettings = Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build();
+        Settings targetSettings =
+                Settings.builder()
+                        .put("index.number_of_shards", 2)
+                        .put("index.number_of_replicas", 0)
+                        .putNull("index.routing.allocation.require._name")
+                        .build();
         resizeRequest.setTargetIndex(new CreateIndexRequest("target").settings(targetSettings).alias(new Alias("alias")));
         ResizeResponse resizeResponse = highLevelClient().indices().shrink(resizeRequest);
         assertTrue(resizeResponse.isAcknowledged());

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java

@@ -1305,7 +1305,8 @@ public void testShrinkIndex() throws Exception {
         // end::shrink-index-request-waitForActiveShards
         // tag::shrink-index-request-settings
         request.getTargetIndexRequest().settings(Settings.builder()
-                .put("index.number_of_shards", 2)); // <1>
+                .put("index.number_of_shards", 2) // <1>
+                .putNull("index.routing.allocation.require._name")); // <2>
         // end::shrink-index-request-settings
         // tag::shrink-index-request-aliases
         request.getTargetIndexRequest().alias(new Alias("target_alias")); // <1>

distribution/build.gradle

@@ -310,12 +310,14 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 task run(type: RunTask) {
   distribution = System.getProperty('run.distribution', 'zip')
   if (distribution == 'zip') {
-    String licenseType = System.getProperty("license_type", "basic")
+    String licenseType = System.getProperty("run.license_type", "basic")
     if (licenseType == 'trial') {
       setting 'xpack.ml.enabled', 'true'
       setting 'xpack.graph.enabled', 'true'
       setting 'xpack.watcher.enabled', 'true'
       setting 'xpack.license.self_generated.type', 'trial'
+      setupCommand 'setupTestAdmin',
+        'bin/elasticsearch-users', 'useradd', 'elastic-admin', '-p', 'elastic-password', '-r', 'superuser'
     } else if (licenseType != 'basic') {
       throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].")
     }

docs/java-rest/high-level/indices/shrink_index.asciidoc

@@ -45,8 +45,8 @@ returns a response, as an `ActiveShardCount`
 --------------------------------------------------
 include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[shrink-index-request-settings]
 --------------------------------------------------
-<1> The settings to apply to the target index, which include the number of
-shards to create for it
+<1> The number of shards on the target of the shrink index request
+<2> Remove the allocation requirement copied from the source index
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------

docs/reference/indices/shrink-index.asciidoc

@@ -62,7 +62,7 @@ the following request:
 
 [source,js]
 --------------------------------------------------
-POST my_source_index/_shrink/my_target_index?copy_settings=true
+POST my_source_index/_shrink/my_target_index
 {
   "settings": {
     "index.routing.allocation.require._name": null, <1>
@@ -106,7 +106,7 @@ and accepts `settings` and `aliases` parameters for the target index:
 
 [source,js]
 --------------------------------------------------
-POST my_source_index/_shrink/my_target_index?copy_settings=true
+POST my_source_index/_shrink/my_target_index
 {
   "settings": {
     "index.number_of_replicas": 1,
@@ -130,16 +130,6 @@ POST my_source_index/_shrink/my_target_index?copy_settings=true
 
 NOTE: Mappings may not be specified in the `_shrink` request.
 
-NOTE: By default, with the exception of `index.analysis`, `index.similarity`,
-and `index.sort` settings, index settings on the source index are not copied
-during a shrink operation. With the exception of non-copyable settings, settings
-from the source index can be copied to the target index by adding the URL
-parameter `copy_settings=true` to the request. Note that `copy_settings` can not
-be set to `false`. The parameter `copy_settings` will be removed in 8.0.0
-
-deprecated[6.4.0, not copying settings is deprecated, copying settings will be
-the default behavior in 7.x]
-
 [float]
 === Monitoring the shrink process
 

docs/reference/indices/split-index.asciidoc

@@ -123,7 +123,7 @@ the following request:
 
 [source,js]
 --------------------------------------------------
-POST my_source_index/_split/my_target_index?copy_settings=true
+POST my_source_index/_split/my_target_index
 {
   "settings": {
     "index.number_of_shards": 2
@@ -158,7 +158,7 @@ and accepts `settings` and `aliases` parameters for the target index:
 
 [source,js]
 --------------------------------------------------
-POST my_source_index/_split/my_target_index?copy_settings=true
+POST my_source_index/_split/my_target_index
 {
   "settings": {
     "index.number_of_shards": 5 <1>
@@ -177,16 +177,6 @@ POST my_source_index/_split/my_target_index?copy_settings=true
 
 NOTE: Mappings may not be specified in the `_split` request.
 
-NOTE: By default, with the exception of `index.analysis`, `index.similarity`,
-and `index.sort` settings, index settings on the source index are not copied
-during a split operation. With the exception of non-copyable settings, settings
-from the source index can be copied to the target index by adding the URL
-parameter `copy_settings=true` to the request. Note that `copy_settings` can not
-be set to `false`. The parameter `copy_settings` will be removed in 8.0.0
-
-deprecated[6.4.0, not copying settings is deprecated, copying settings will be
-the default behavior in 7.x]
-
 [float]
 === Monitoring the split process
 

docs/reference/migration/migrate_7_0/api.asciidoc

@@ -65,3 +65,13 @@ deprecated in 6.3.0 and now removed in 7.0.0.
 In the past, `fields` could be provided either as a parameter, or as part of the request
 body. Specifying `fields` in the request body as opposed to a parameter was deprecated
 in 6.4.0, and is now unsupported in 7.0.0.
+
+==== `copy_settings` is deprecated on shrink and split APIs
+
+Versions of Elasticsearch prior to 6.4.0 did not copy index settings on shrink
+and split operations. Starting with Elasticsearch 7.0.0, the default behavior
+will be for such settings to be copied on such operations. To enable users in
+6.4.0 to transition in 6.4.0 to the default behavior in 7.0.0, the
+`copy_settings` parameter was added on the REST layer. As this behavior will be
+the only behavior in 8.0.0, this parameter is deprecated in 7.0.0 for removal in
+8.0.0.

docs/reference/setup/install/windows.asciidoc

@@ -79,13 +79,13 @@ part of the installation, with the option to configure a HTTPS proxy through whi
 [[msi-installer-selected-plugins]]
 image::images/msi_installer/msi_installer_selected_plugins.png[]
 
-Upon choosing to install X-Pack plugin, an additional step allows a choice of the type of X-Pack
-license to install, in addition to X-Pack Security configuration and built-in user configuration:
+Upon choosing to install {xpack} plugin, an additional step allows a choice of the type of {xpack}
+license to install, in addition to {security} configuration and built-in user configuration:
 
 [[msi-installer-xpack]]
 image::images/msi_installer/msi_installer_xpack.png[]
 
-NOTE: X-Pack includes a choice of a Trial or Basic license for 30 days. After that, you can obtain one of the
+NOTE: {xpack} includes a choice of a Trial or Basic license for 30 days. After that, you can obtain one of the
 https://www.elastic.co/subscriptions[available subscriptions] or {ref}/security-settings.html[disable Security].
 The Basic license is free and includes the https://www.elastic.co/products/x-pack/monitoring[Monitoring] extension.
 
@@ -286,43 +286,43 @@ as _properties_ within Windows Installer documentation) that can be passed to ms
 
 `XPACKLICENSE`::
 
-  When installing X-Pack plugin, the type of license to install,
+  When installing {xpack} plugin, the type of license to install,
   either `Basic` or `Trial`. Defaults to `Basic`
 
 `XPACKSECURITYENABLED`::
 
-  When installing X-Pack plugin with a `Trial` license, whether X-Pack Security should be enabled.
+  When installing {xpack} plugin with a `Trial` license, whether {security} should be enabled.
   Defaults to `true`
 
 `BOOTSTRAPPASSWORD`::
 
-  When installing X-Pack plugin with a `Trial` license and X-Pack Security enabled, the password to
+  When installing {xpack} plugin with a `Trial` license and {security} enabled, the password to
   used to bootstrap the cluster and persisted as the `bootstrap.password` setting in the keystore.
   Defaults to a randomized value.
 
 `SKIPSETTINGPASSWORDS`::
 
-  When installing X-Pack plugin with a `Trial` license and X-Pack Security enabled, whether the
+  When installing {xpack} plugin with a `Trial` license and {security} enabled, whether the
   installation should skip setting up the built-in users `elastic`, `kibana` and `logstash_system`.
   Defaults to `false`
 
 `ELASTICUSERPASSWORD`::
 
-  When installing X-Pack plugin with a `Trial` license and X-Pack Security enabled, the password
+  When installing {xpack} plugin with a `Trial` license and {security} enabled, the password
   to use for the built-in user `elastic`. Defaults to `""`
 
 `KIBANAUSERPASSWORD`::
 
-  When installing X-Pack plugin with a `Trial` license and X-Pack Security enabled, the password
+  When installing {xpack} plugin with a `Trial` license and {security} enabled, the password
   to use for the built-in user `kibana`. Defaults to `""`
 
 `LOGSTASHSYSTEMUSERPASSWORD`::
 
-  When installing X-Pack plugin with a `Trial` license and X-Pack Security enabled, the password
+  When installing {xpack} plugin with a `Trial` license and {security} enabled, the password
   to use for the built-in user `logstash_system`. Defaults to `""`
 
 To pass a value, simply append the property name and value using the format `<PROPERTYNAME>="<VALUE>"` to
-the installation command. For example, to use a different installation directory to the default one and to install https://www.elastic.co/products/x-pack[X-Pack]:
+the installation command. For example, to use a different installation directory to the default one and to install https://www.elastic.co/products/x-pack[{xpack}]:
 
 ["source","sh",subs="attributes,callouts"]
 --------------------------------------------

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml

@@ -2,7 +2,7 @@
 #
 setup:
   - skip:
-        version: " - 6.4.0"
+        version: " - 6.3.99"
         reason:  "moving_fn added in 6.4.0"
   - do:
       indices.create:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml

@@ -1,8 +1,8 @@
 ---
 "Shrink index via API":
   - skip:
-      version: " - 6.3.99"
-      reason: expects warnings that pre-6.4.0 will not send
+      version: " - 6.9.99"
+      reason: expects warnings that pre-7.0.0 will not send
       features: "warnings"
   # creates an index with one document solely allocated on the master node
   # and shrinks it into a new index with a single shard
@@ -67,8 +67,6 @@
         body:
           settings:
             index.number_of_replicas: 0
-      warnings:
-        - "resize operations without copying settings is deprecated; set parameter [copy_settings] to [true] for future default behavior"
 
   - do:
       cluster.health:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/20_source_mapping.yml

@@ -1,8 +1,8 @@
 ---
 "Shrink index ignores target template mapping":
     - skip:
-        version: " - 6.3.99"
-        reason: expects warnings that pre-6.4.0 will not send
+        version: " - 6.9.99"
+        reason: expects warnings that pre-7.0.0 will not send
         features: "warnings"
 
     - do:
@@ -71,8 +71,6 @@
           body:
             settings:
               index.number_of_replicas: 0
-        warnings:
-          - "resize operations without copying settings is deprecated; set parameter [copy_settings] to [true] for future default behavior"
 
     - do:
         cluster.health:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml

@@ -1,8 +1,8 @@
 ---
 "Copy settings during shrink index":
   - skip:
-      version: " - 6.3.99"
-      reason: expects warnings that pre-6.4.0 will not send
+      version: " - 6.9.99"
+      reason: expects warnings that pre-7.0.0 will not send
       features: "warnings"
 
   - do:
@@ -48,6 +48,8 @@
           settings:
             index.number_of_replicas: 0
             index.merge.scheduler.max_thread_count: 2
+      warnings:
+        - "parameter [copy_settings] is deprecated and will be removed in 8.0.0"
 
   - do:
       cluster.health:
@@ -63,33 +65,31 @@
   - match: { copy-settings-target.settings.index.blocks.write: "true" }
   - match: { copy-settings-target.settings.index.routing.allocation.include._id: $master }
 
-  # now we do a actual shrink and do not copy settings (by default)
+  # now we do a actual shrink and copy settings (by default)
   - do:
       indices.shrink:
         index: "source"
-        target: "no-copy-settings-target"
+        target: "default-copy-settings-target"
         wait_for_active_shards: 1
         master_timeout: 10s
         body:
           settings:
             index.number_of_replicas: 0
             index.merge.scheduler.max_thread_count: 2
-      warnings:
-        - "resize operations without copying settings is deprecated; set parameter [copy_settings] to [true] for future default behavior"
 
   - do:
       cluster.health:
         wait_for_status: green
 
   - do:
       indices.get_settings:
-        index: "no-copy-settings-target"
+        index: "default-copy-settings-target"
 
-  # only the request setting should be copied
-  - is_false: no-copy-settings-target.settings.index.merge.scheduler.max_merge_count
-  - match: { no-copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" }
-  - is_false: no-copy-settings-target.settings.index.blocks.write
-  - is_false: no-copy-settings-target.settings.index.routing.allocation.include._id
+  # settings should be copied
+  - match: { default-copy-settings-target.settings.index.merge.scheduler.max_merge_count: "4" }
+  - match: { default-copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" }
+  - match: { default-copy-settings-target.settings.index.blocks.write: "true" }
+  - match: { default-copy-settings-target.settings.index.routing.allocation.include._id: $master }
 
   # now we do a actual shrink and try to set no copy settings
   - do: