Introduce formal role for remote cluster client (#54138)

This commit introduce a formal role for identifying nodes that are
capable of making connections to remote clusters.

Relates #53924

Author: jasontedor

distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml

@@ -7,7 +7,7 @@
   - match:
       $body: |
         /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role           master          name
-        ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
+        ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmrt]{1,6}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
@@ -16,7 +16,7 @@
   - match:
       $body: |
         /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role      \s+  master   \s+   name  \n
-           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
+           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmrt]{1,6}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:

docs/reference/migration/migrate_7_7.asciidoc

@@ -36,6 +36,22 @@ pre-7.7 default behaviour by adjusting your <<logging>>.
 [[breaking_77_settings_changes]]
 === Settings changes
 
+[discrete]
+[[deprecate-listener-thread-pool]]
+==== `thread_pool.listener.size` and `thread_pool.listener.queue_size` have been deprecated
+The listener thread pool is no longer used internally by Elasticsearch.
+Therefore, these settings have been deprecated. You can safely remove these
+settings from the configuration of your nodes.
+
+[discrete]
+[[deprecate-cluster-remote-connect]]
+==== `cluster.remote.connect` is deprecated in favor of `node.remote_cluster_client`
+Previously the setting `cluster.remote.connect` was used to configure whether or
+not the local node is capable of acting as a remote cluster client in
+cross-cluster search and cross-cluster replication. This setting is deprecated
+in favor of `node.remote_cluster_client` serves the same purpose and identifies
+the local node as having the `remote_cluster_client` role.
+
 [discrete]
 [[deprecate-missing-realm-order]]
 ==== Authentication realm `order` will be a required config in version 8.0.0.
@@ -72,14 +88,6 @@ for `gt` and `lte` boundaries, the same queries on `date_range` fields didn't
 do this. The behavior is now the same for both field types like documented in
 <<range-query-date-math-rounding>>.
 
-[float]
-[[deprecate-listener-thread-pool]]
-==== `thread_pool.listener.size` and `thread_pool.listener.queue_size` have been deprecated
-
-The listener thread pool is no longer used internally by Elasticsearch.
-Therefore, these settings have been deprecated. You can safely remove these
-settings from the configuration of your nodes.
-
 [discrete]
 [[breaking_77_highlighters_changes]]
 === Highlighters changes

docs/reference/ml/ml-shared.asciidoc

@@ -797,8 +797,8 @@ An array of index names. Wildcards are supported. For example:
 `["it_ops_metrics", "server*"]`.
 +
 --
-NOTE: If any indices are in remote clusters then `cluster.remote.connect` must
-not be set to `false` on any {ml} nodes.
+NOTE: If any indices are in remote clusters then `node.remote_cluster_client`
+must not be set to `false` on any {ml} nodes.
 
 --
 end::indices[]

docs/reference/modules/node.asciidoc

@@ -1,8 +1,8 @@
 [[modules-node]]
 == Node
 
-Any time that you start an instance of Elasticsearch, you are starting a _node_. 
-A collection of connected nodes is called a <<modules-cluster,cluster>>. If you 
+Any time that you start an instance of Elasticsearch, you are starting a _node_.
+A collection of connected nodes is called a <<modules-cluster,cluster>>. If you
 are running a single node of {es}, then you have a cluster of one node.
 
 Every node in the cluster can handle <<modules-http,HTTP>> and
@@ -11,7 +11,7 @@ exclusively for communication between nodes; the HTTP layer is used by REST
 clients.
 
 All nodes know about all the other nodes in the cluster and can forward client
-requests to the appropriate node. 
+requests to the appropriate node.
 
 By default, a node is all of the following types: master-eligible, data, ingest,
 and machine learning (if available).
@@ -95,22 +95,22 @@ be elected to become the master node by the <<modules-discovery,master election
 process>>.
 
 IMPORTANT: Master nodes must have access to the `data/` directory (just like
-`data` nodes) as this is where the cluster state is persisted between node 
+`data` nodes) as this is where the cluster state is persisted between node
 restarts.
 
 [float]
 [[dedicated-master-node]]
 ==== Dedicated master-eligible node
 
-It is important for the health of the cluster that the elected master node has 
-the resources it needs to fulfill its responsibilities. If the elected master 
-node is overloaded with other tasks then the cluster may not operate well. In 
-particular, indexing and searching your data can be very resource-intensive, so 
-in large or high-throughput clusters it is a good idea to avoid using the 
-master-eligible nodes for tasks such as indexing and searching. You can do this 
-by configuring three of your nodes to be dedicated master-eligible nodes. 
+It is important for the health of the cluster that the elected master node has
+the resources it needs to fulfill its responsibilities. If the elected master
+node is overloaded with other tasks then the cluster may not operate well. In
+particular, indexing and searching your data can be very resource-intensive, so
+in large or high-throughput clusters it is a good idea to avoid using the
+master-eligible nodes for tasks such as indexing and searching. You can do this
+by configuring three of your nodes to be dedicated master-eligible nodes.
 Dedicated master-eligible nodes only have the `master` role, allowing them to
-focus on managing the cluster. While master nodes can also behave as 
+focus on managing the cluster. While master nodes can also behave as
 <<coordinating-node,coordinating nodes>> and route search and indexing requests
 from clients to data nodes, it is better _not_ to use dedicated master nodes for
 this purpose.
@@ -127,7 +127,7 @@ node.ml: false <5>
 xpack.ml.enabled: true <6>
 node.transform: false <7>
 xpack.transform.enabled: true <8>
-cluster.remote.connect: false <9>
+node.remote_client_client: false <9>
 -------------------
 <1> The `node.master` role is enabled by default.
 <2> The `node.voting_only` role is disabled by default.
@@ -146,7 +146,7 @@ To create a dedicated master-eligible node in the {oss-dist}, set:
 node.master: true <1>
 node.data: false <2>
 node.ingest: false <3>
-cluster.remote.connect: false <4>
+node.remote_cluster_client: false <4>
 -------------------
 <1> The `node.master` role is enabled by default.
 <2> Disable the `node.data` role (enabled by default).
@@ -210,7 +210,7 @@ node.ml: false <5>
 xpack.ml.enabled: true <6>
 node.transform: false <7>
 xpack.transform.enabled: true <8>
-cluster.remote.connect: false <9>
+node.remote_cluster_client: false <9>
 -------------------
 <1> The `node.master` role is enabled by default.
 <2> Enable the `node.voting_only` role (disabled by default).
@@ -243,7 +243,7 @@ node.data: true <3>
 node.ingest: false <4>
 node.ml: false <5>
 node.transform: false <6>
-cluster.remote.connect: false <7>
+node.remote_cluster_client: false <7>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
@@ -259,7 +259,7 @@ To create a dedicated data node in the {oss-dist}, set:
 node.master: false <1>
 node.data: true <2>
 node.ingest: false <3>
-cluster.remote.connect: false <4>
+node.remote_cluster_client: false <4>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.data` role is enabled by default.
@@ -285,7 +285,7 @@ node.data: false <3>
 node.ingest: true <4>
 node.ml: false <5>
 node.transform: false <6>
-cluster.remote.connect: false <7>
+node.remote_cluster_client: false <7>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
@@ -302,7 +302,7 @@ To create a dedicated ingest node in the {oss-dist}, set:
 node.master: false <1>
 node.data: false <2>
 node.ingest: true <3>
-cluster.remote.connect: false <4>
+node.remote_cluster_client: false <4>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> Disable the `node.data` role (enabled by default).
@@ -340,7 +340,7 @@ node.data: false <3>
 node.ingest: false <4>
 node.ml: false <5>
 node.transform: false <6>
-cluster.remote.connect: false <7>
+node.remote_cluster_client: false <7>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
@@ -357,7 +357,7 @@ To create a dedicated coordinating node in the {oss-dist}, set:
 node.master: false <1>
 node.data: false <2>
 node.ingest: false <3>
-cluster.remote.connect: false <4>
+node.remote_cluster_client: false <4>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> Disable the `node.data` role (enabled by default).
@@ -390,7 +390,7 @@ node.ml: true <5>
 xpack.ml.enabled: true <6>
 node.transform: false <7>
 xpack.transform.enabled: true <8>
-cluster.remote.connect: false <9>
+node.remote_cluster_client: false <9>
 -------------------
 <1> Disable the `node.master` role (enabled by default).
 <2> The `node.voting_only` role is disabled by default.
@@ -425,7 +425,7 @@ node.ingest: false <4>
 node.ml: false <5>
 node.transform: true <6>
 xpack.transform.enabled: true <7>
-cluster.remote.connect: false <8>
+node.remote_cluster_client: false <8>
 -------------------
 <1> Disable the `node.master` role.
 <2> Disable the `node.voting_only`.

docs/reference/modules/remote-clusters.asciidoc

@@ -254,12 +254,12 @@ and <<remote-cluster-proxy-settings,proxy mode settings>> are described below.
   The time to wait for remote connections to be established when the node
   starts. The default is `30s`.
 
-`cluster.remote.connect`::
+`node.remote_cluster_client`::
 
   By default, any node in the cluster can act as a cross-cluster client and
-  connect to remote clusters. The `cluster.remote.connect` setting can be set to
-  `false` (defaults to `true`) to prevent certain nodes from connecting to
-  remote clusters. Remote cluster requests must be sent to a node that is
+  connect to remote clusters. The `node.remote_cluster_client` setting can be
+  set to `false` (defaults to `true`) to prevent certain nodes from connecting
+  to remote clusters. Remote cluster requests must be sent to a node that is
   allowed to act as a cross-cluster client.
 
 `cluster.remote.<cluster_alias>.skip_unavailable`::

qa/multi-cluster-search/build.gradle

@@ -35,7 +35,7 @@ task 'remote-cluster'(type: RestIntegTestTask) {
 
 testClusters.'remote-cluster' {
   numberOfNodes = 2
-  setting 'cluster.remote.connect', 'false'
+  setting 'node.remote_cluster_client', 'false'
 }
 
 task mixedClusterTest(type: RestIntegTestTask) {
@@ -50,7 +50,7 @@ testClusters.mixedClusterTest {
   setting 'cluster.remote.my_remote_cluster.seeds',
     { "\"${testClusters.'remote-cluster'.getAllTransportPortURI().get(0)}\"" }
   setting 'cluster.remote.connections_per_cluster', '1'
-  setting 'cluster.remote.connect', 'true'
+  setting 'node.remote_cluster_client', 'true'
 }
 
 

rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml

@@ -7,7 +7,7 @@
   - match:
       $body: |
                /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role           master          name
-               ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
+               ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[dilmrt]{1,6}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
@@ -16,7 +16,7 @@
   - match:
       $body: |
                /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role      \s+  master   \s+   name  \n
-                  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmt]{1,5}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
+                  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilmrt]{1,6}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java

@@ -73,6 +73,10 @@ public static boolean isIngestNode(Settings settings) {
         return Node.NODE_INGEST_SETTING.get(settings);
     }
 
+    public static boolean isRemoteClusterClient(final Settings settings) {
+        return Node.NODE_REMOTE_CLUSTER_CLIENT.get(settings);
+    }
+
     private final String nodeName;
     private final String nodeId;
     private final String ephemeralId;
@@ -282,7 +286,7 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             // an old node will only understand legacy roles since pluggable roles is a new concept
             final List<DiscoveryNodeRole> rolesToWrite =
-                    roles.stream().filter(DiscoveryNodeRole.BUILT_IN_ROLES::contains).collect(Collectors.toList());
+                    roles.stream().filter(DiscoveryNodeRole.LEGACY_ROLES::contains).collect(Collectors.toList());
             out.writeVInt(rolesToWrite.size());
             for (final DiscoveryNodeRole role : rolesToWrite) {
                 if (role == DiscoveryNodeRole.MASTER_ROLE) {
@@ -357,6 +361,15 @@ public boolean isIngestNode() {
         return roles.contains(DiscoveryNodeRole.INGEST_ROLE);
     }
 
+    /**
+     * Returns whether or not the node can be a remote cluster client.
+     *
+     * @return true if the node can be a remote cluster client, false otherwise
+     */
+    public boolean isRemoteClusterClient() {
+        return roles.contains(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE);
+    }
+
     /**
      * Returns a set of all the roles that the node fulfills.
      * If the node doesn't have any specific role, the set is returned empty, which means that the node is a coordinating only node.

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java

@@ -137,11 +137,23 @@ protected Setting<Boolean> roleSetting() {
 
     };
 
+    public static final DiscoveryNodeRole REMOTE_CLUSTER_CLIENT_ROLE = new DiscoveryNodeRole("remote_cluster_client", "r") {
+
+        @Override
+        protected Setting<Boolean> roleSetting() {
+            return Node.NODE_REMOTE_CLUSTER_CLIENT;
+        }
+
+    };
+
     /**
      * The built-in node roles.
      */
     public static Set<DiscoveryNodeRole> BUILT_IN_ROLES =
-            Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DATA_ROLE, INGEST_ROLE, MASTER_ROLE)));
+            Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DATA_ROLE, INGEST_ROLE, MASTER_ROLE, REMOTE_CLUSTER_CLIENT_ROLE)));
+
+    static Set<DiscoveryNodeRole> LEGACY_ROLES =
+        Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DATA_ROLE, INGEST_ROLE, MASTER_ROLE)));
 
     /**
      * Represents an unknown role. This can occur if a newer version adds a role that an older version does not know about, or a newer

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -462,6 +462,7 @@ public void apply(Settings value, Settings current, Settings previous) {
             Node.NODE_DATA_SETTING,
             Node.NODE_MASTER_SETTING,
             Node.NODE_INGEST_SETTING,
+            Node.NODE_REMOTE_CLUSTER_CLIENT,
             Node.NODE_ATTRIBUTES,
             Node.NODE_LOCAL_STORAGE_SETTING,
             AutoCreateIndex.AUTO_CREATE_INDEX_SETTING,

server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java

@@ -28,7 +28,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -41,7 +40,7 @@ public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue>, ToXC
     /**
      * We have to lazy initialize the deprecation logger as otherwise a static logger here would be constructed before logging is configured
      * leading to a runtime failure (see {@link LogConfigurator#checkErrorListener()} ). The premature construction would come from any
-     * {@link Setting} object constructed in, for example, settings in {@link org.elasticsearch.common.network.NetworkService}.
+     * {@link ByteSizeValue} object constructed in, for example, settings in {@link org.elasticsearch.common.network.NetworkService}.
      */
     static class DeprecationLoggerHolder {
         static DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(ByteSizeValue.class));

server/src/main/java/org/elasticsearch/node/Node.java

@@ -158,6 +158,7 @@
 import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportService;
@@ -203,6 +204,8 @@ public class Node implements Closeable {
         Setting.boolSetting("node.master", true, Property.NodeScope);
     public static final Setting<Boolean> NODE_INGEST_SETTING =
         Setting.boolSetting("node.ingest", true, Property.NodeScope);
+    public static final Setting<Boolean> NODE_REMOTE_CLUSTER_CLIENT =
+        Setting.boolSetting("node.remote_cluster_client", RemoteClusterService.ENABLE_REMOTE_CLUSTERS, Property.NodeScope);
 
     /**
     * controls whether the node is allowed to persist things like metadata to disk