Merge remote-tracking branch 'elastic/master' into store-snapshot-version-in-index-N

Author: original-brownbear

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -423,7 +423,10 @@ public Cancellable flushAsync(FlushRequest flushRequest, RequestOptions options,
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @return the response
      * @throws IOException in case there is a problem sending the request or parsing back the response
+     * @deprecated synced flush is deprecated and will be removed in 8.0.
+     * Use {@link #flush(FlushRequest, RequestOptions)} instead.
      */
+    @Deprecated
     public SyncedFlushResponse flushSynced(SyncedFlushRequest syncedFlushRequest, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(syncedFlushRequest, IndicesRequestConverters::flushSynced, options,
                 SyncedFlushResponse::fromXContent, emptySet());
@@ -437,7 +440,10 @@ public SyncedFlushResponse flushSynced(SyncedFlushRequest syncedFlushRequest, Re
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      * @return cancellable that may be used to cancel the request
+     * @deprecated synced flush is deprecated and will be removed in 8.0.
+     * Use {@link #flushAsync(FlushRequest, RequestOptions, ActionListener)} instead.
      */
+    @Deprecated
     public Cancellable flushSyncedAsync(SyncedFlushRequest syncedFlushRequest, RequestOptions options,
                                         ActionListener<SyncedFlushResponse> listener) {
         return restHighLevelClient.performRequestAsyncAndParseEntity(syncedFlushRequest, IndicesRequestConverters::flushSynced, options,

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -97,6 +97,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.indices.flush.SyncedFlushService;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -768,7 +769,8 @@ public void testSyncedFlush() throws IOException {
             createIndex(index, settings);
             SyncedFlushRequest syncedFlushRequest = new SyncedFlushRequest(index);
             SyncedFlushResponse flushResponse =
-                    execute(syncedFlushRequest, highLevelClient().indices()::flushSynced, highLevelClient().indices()::flushSyncedAsync);
+                    execute(syncedFlushRequest, highLevelClient().indices()::flushSynced, highLevelClient().indices()::flushSyncedAsync,
+                        expectWarnings(SyncedFlushService.SYNCED_FLUSH_DEPRECATION_MESSAGE));
             assertThat(flushResponse.totalShards(), equalTo(1));
             assertThat(flushResponse.successfulShards(), equalTo(1));
             assertThat(flushResponse.failedShards(), equalTo(0));
@@ -783,7 +785,8 @@ public void testSyncedFlush() throws IOException {
                     execute(
                         syncedFlushRequest,
                         highLevelClient().indices()::flushSynced,
-                        highLevelClient().indices()::flushSyncedAsync
+                        highLevelClient().indices()::flushSyncedAsync,
+                        expectWarnings(SyncedFlushService.SYNCED_FLUSH_DEPRECATION_MESSAGE)
                     )
             );
             assertEquals(RestStatus.NOT_FOUND, exception.status());

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -839,7 +839,8 @@ public void testApiNamingConventions() throws Exception {
         // looking like it doesn't have a valid implementatation when it does.
         apiUnsupported.remove("indices.get_template");
 
-
+        // Synced flush is deprecated
+        apiUnsupported.remove("indices.flush_synced");
 
         for (Map.Entry<String, Set<Method>> entry : methods.entrySet()) {
             String apiName = entry.getKey();

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java

@@ -1018,7 +1018,9 @@ public void testSyncedFlushIndex() throws Exception {
             // end::flush-synced-request-indicesOptions
 
             // tag::flush-synced-execute
-            SyncedFlushResponse flushSyncedResponse = client.indices().flushSynced(request, RequestOptions.DEFAULT);
+            SyncedFlushResponse flushSyncedResponse = client.indices().flushSynced(request, expectWarnings(
+                "Synced flush is deprecated and will be removed in 8.0. Use flush at _/flush or /{index}/_flush instead."
+            ));
             // end::flush-synced-execute
 
             // tag::flush-synced-response
@@ -1062,7 +1064,9 @@ public void onFailure(Exception e) {
             listener = new LatchedActionListener<>(listener, latch);
 
             // tag::flush-synced-execute-async
-            client.indices().flushSyncedAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            client.indices().flushSyncedAsync(request, expectWarnings(
+                "Synced flush is deprecated and will be removed in 8.0. Use flush at _/flush or /{index}/_flush instead."
+            ), listener); // <1>
             // end::flush-synced-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java

@@ -694,8 +694,8 @@ public void testGetRoles() throws Exception {
 
             List<Role> roles = response.getRoles();
             assertNotNull(response);
-            // 28 system roles plus the three we created
-            assertThat(roles.size(), equalTo(28 + 3));
+            // 29 system roles plus the three we created
+            assertThat(roles.size(), equalTo(29 + 3));
         }
 
         {

docs/reference/ccr/requirements.asciidoc

@@ -41,58 +41,3 @@ to a follower the following process will fail due to incomplete history on the l
 The default value is `12h`.
 
 For more information about index settings, see {ref}/index-modules.html[Index modules].
-
-
-[[ccr-overview-beats]]
-==== Setting soft deletes on indices created by APM Server or Beats
-
-If you want to replicate indices created by APM Server or Beats, and are
-allowing APM Server or Beats to manage index templates, you need to configure
-soft deletes on the underlying index templates. To configure soft deletes on the
-underlying index templates, incorporate the following changes to the relevant
-APM Server or Beats configuration file.
-
-["source","yaml"]
-----------------------------------------------------------------------
-setup.template.overwrite: true
-setup.template.settings:
-  index.soft_deletes.retention.operations: 1024
-----------------------------------------------------------------------
-
-For additional information on controlling the index templates managed by APM
-Server or Beats, see the relevant documentation on loading the Elasticsearch
-index template.
-
-
-[[ccr-overview-logstash]]
-==== Setting soft deletes on indices created by Logstash
-
-If you want to replicate indices created by Logstash, and are using Logstash to
-manage index templates, you need to configure soft deletes on a custom Logstash
-index template. To configure soft deletes on the underlying index template,
-incorporate the following change to a custom Logstash template.
-
-["source","js"]
-----------------------------------------------------------------------
-{
-  "settings" : {
-    "index.soft_deletes.retention.operations" : 1024
-  }
-}
-----------------------------------------------------------------------
-// NOTCONSOLE
-
-Additionally, you will need to configure the Elasticsearch output plugin to use
-this custom template.
-
-["source","ruby"]
-----------------------------------------------------------------------
-output {
-  elasticsearch {
-    template => "/path/to/custom/logstash/template.json"
-  }
-}
-----------------------------------------------------------------------
-
-For additional information on controlling the index templates managed by
-Logstash, see the relevant documentation on the Elasticsearch output plugin.

docs/reference/commands/index.asciidoc

@@ -10,6 +10,7 @@ tasks from the command line:
 * <<certgen>>
 * <<certutil>>
 * <<elasticsearch-croneval>>
+* <<elasticsearch-keystore>>
 * <<node-tool>>
 * <<saml-metadata>>
 * <<setup-passwords>>
@@ -22,6 +23,7 @@ tasks from the command line:
 include::certgen.asciidoc[]
 include::certutil.asciidoc[]
 include::croneval.asciidoc[]
+include::keystore.asciidoc[]
 include::node-tool.asciidoc[]
 include::saml-metadata.asciidoc[]
 include::setup-passwords.asciidoc[]

docs/reference/commands/keystore.asciidoc

@@ -0,0 +1,149 @@
+[[elasticsearch-keystore]]
+== elasticsearch-keystore
+
+The `elasticsearch-keystore` command manages <<secure-settings,secure settings>>
+in the {es} keystore.
+
+[discrete]
+[[elasticsearch-keystore-synopsis]]
+=== Synopsis
+
+[source,shell]
+--------------------------------------------------
+bin/elasticsearch-keystore
+([add <setting>] [--stdin] |
+[add-file <setting> <path>] | [create] |
+[list] | [remove <setting>] | [upgrade])
+[-h, --help] ([-s, --silent] | [-v, --verbose])
+--------------------------------------------------
+
+[discrete]
+[[elasticsearch-keystore-description]]
+=== Description
+
+IMPORTANT: This command should be run as the user that will run {es}.
+
+Currently, all secure settings are node-specific settings that must have the
+same value on every node. Therefore you must run this command on every node.
+
+Modifications to the keystore do not take effect until you restart {es}.
+
+Only some settings are designed to be read from the keystore. However, there
+is no validation to block unsupported settings from the keystore and they can
+cause {es} to fail to start. To see whether a setting is supported in the
+keystore, see the setting reference.
+
+[discrete]
+[[elasticsearch-keystore-parameters]]
+=== Parameters
+
+`add <setting>`:: Adds settings to the keystore. By default, you are prompted
+for the value of the setting.
+
+`add-file <setting> <path>`:: Adds a file to the keystore.
+
+`create`:: Creates the keystore.
+
+`-h, --help`:: Returns all of the command parameters.
+
+`list`:: Lists the settings in the keystore.
+
+`remove <setting>:: Removes a setting from the keystore.
+
+`-s, --silent`:: Shows minimal output.
+
+`--stdin`:: When used with the `add` parameter, you can pass the setting value
+through standard input (stdin). See <<add-string-to-keystore>>.
+
+`upgrade`:: Upgrades the internal format of the keystore.
+
+`-v, --verbose`:: Shows verbose output.
+
+[discrete]
+[[elasticsearch-keystore-examples]]
+=== Examples
+
+[discrete]
+[[creating-keystore]]
+==== Create the keystore
+
+To create the `elasticsearch.keystore`, use the `create` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore create
+----------------------------------------------------------------
+
+A `elasticsearch.keystore` file is created alongside the `elasticsearch.yml`
+file.
+
+[discrete]
+[[list-settings]]
+==== List settings in the keystore
+
+To list the settings in the keystore, use the `list` command.
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore list
+----------------------------------------------------------------
+
+[discrete]
+[[add-string-to-keystore]]
+==== Add settings to the keystore
+
+Sensitive string settings, like authentication credentials for Cloud plugins,
+can be added with the `add` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore add the.setting.name.to.set
+----------------------------------------------------------------
+
+You are prompted to enter the value of the setting. To pass the value
+through standard input (stdin), use the `--stdin` flag:
+
+[source,sh]
+----------------------------------------------------------------
+cat /file/containing/setting/value | bin/elasticsearch-keystore add --stdin the.setting.name.to.set
+----------------------------------------------------------------
+
+[discrete]
+[[add-file-to-keystore]]
+==== Add files to the keystore
+
+You can add sensitive files, like authentication key files for Cloud plugins,
+using the `add-file` command. Be sure to include your file path as an argument
+after the setting name.
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore add-file the.setting.name.to.set /path/example-file.json
+----------------------------------------------------------------
+
+[discrete]
+[[remove-settings]]
+==== Remove settings from the keystore
+
+To remove a setting from the keystore, use the `remove` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore remove the.setting.name.to.remove
+----------------------------------------------------------------
+
+[discrete]
+[[keystore-upgrade]]
+==== Upgrade the keystore
+
+Occasionally, the internal format of the keystore changes. When {es} is
+installed from a package manager, an upgrade of the on-disk keystore to the new
+format is done during package upgrade. In other cases, {es} performs the upgrade
+during node startup. This requires that {es} has write permissions to the
+directory that contains the keystore. Alternatively, you can manually perform
+such an upgrade by using the `upgrade` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore upgrade
+----------------------------------------------------------------

docs/reference/ilm/policy-definitions.asciidoc

@@ -55,7 +55,7 @@ PUT _ilm/policy/my_policy
 }
 --------------------------------------------------
 
-The Above example configures a policy that moves the index into the warm
+The above example configures a policy that moves the index into the warm
 phase after one day. Until then, the index is in a waiting state. After
 moving into the warm phase, it will wait until 30 days have elapsed before
 moving to the delete phase and deleting the index.
@@ -76,10 +76,14 @@ check occurs.
 === Phase Execution
 
 The current phase definition, of an index's policy being executed, is stored
-in the index's metadata. The phase and its actions are compiled into a series
-of discrete steps that are executed sequentially. Since some {ilm-init} actions
-are more complex and involve multiple operations against an index, each of these
-operations are done in isolation in a unit called a "step". The
+in the index's metadata. This phase definition is cached to prevent changes to
+the policy from putting the index in a state where it cannot proceed from its
+current step. When the policy is updated we check to see if this phase
+definition can be safely updated, and if so, update the cached definition in
+indices using the updated policy. The phase and its actions are compiled into a
+series of discrete steps that are executed sequentially. Since some {ilm-init}
+actions are more complex and involve multiple operations against an index, each
+of these operations are done in isolation in a unit called a "step". The
 <<ilm-explain-lifecycle,Explain Lifecycle API>> exposes this information to us
 to see which step our index is either to execute next, or is currently
 executing.

docs/reference/indices/synced-flush.asciidoc

@@ -4,6 +4,10 @@
 <titleabbrev>Synced flush</titleabbrev>
 ++++
 
+deprecated::[7.6, synced-flush is deprecated and will be removed in 8.0.
+Use <<indices-flush,flush>> instead. A <<indices-flush,flush>> has the
+same effect as a synced flush on Elasticsearch 7.6 or later]
+
 Performs a synced flush on one or more indices.
 
 [source,console]

docs/reference/monitoring/configuring-filebeat.asciidoc

@@ -117,7 +117,7 @@ If {security-features} are enabled, you must provide a valid user ID and
 password so that {filebeat} can connect to {kib}: 
 
 .. Create a user on the monitoring cluster that has the 
-<<built-in-roles,`kibana_user` built-in role>> or equivalent
+<<built-in-roles,`kibana_admin` built-in role>> or equivalent
 privileges.
 
 .. Add the `username` and `password` settings to the {es} output information in 

docs/reference/rest-api/common-parms.asciidoc

@@ -764,8 +764,9 @@ end::source-transforms[]
 
 tag::source-index-transforms[]
 The _source indices_ for the {transform}. It can be a single index, an index
-pattern (for example, `"myindex*"`), or an array of indices (for example,
-`["index1", "index2"]`).
+pattern (for example, `"myindex*"`), an array of indices (for example,
+`["index1", "index2"]`), or an array of index patterns (for example,
+`["myindex1-*", "myindex2-*"]`.
 end::source-index-transforms[]
 
 tag::source-query-transforms[]