Run Node deprecation checks locally (#38065)

At times, we need to check for usage of deprecated settings in settings
which should not be returned by the NodeInfo API.  This commit changes
the deprecation info API to run all node checks locally so that these
settings can be checked without exposing them via any externally
accessible API.

Author: gwbrown

client/rest-high-level/build.gradle

@@ -105,7 +105,7 @@ integTestCluster {
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.authc.token.enabled', 'true'
   // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API
-  setting 'xpack.ssl.certificate_authorities', 'testnode.crt'
+  setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
   setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'
   setting 'indices.lifecycle.poll_interval', '1000ms'
   keystoreSetting 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java

@@ -11,7 +11,6 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
@@ -35,7 +34,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -63,6 +61,23 @@ public static <T> List<DeprecationIssue> filterChecks(List<T> checks, Function<T
         return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList());
     }
 
+    private static List<DeprecationIssue> mergeNodeIssues(NodesDeprecationCheckResponse response) {
+        Map<DeprecationIssue, List<String>> issueListMap = new HashMap<>();
+        for (NodesDeprecationCheckAction.NodeResponse resp : response.getNodes()) {
+            for (DeprecationIssue issue : resp.getDeprecationIssues()) {
+                issueListMap.computeIfAbsent(issue, (key) -> new ArrayList<>()).add(resp.getNode().getName());
+            }
+        }
+
+        return issueListMap.entrySet().stream()
+            .map(entry -> {
+                DeprecationIssue issue = entry.getKey();
+                String details = issue.getDetails() != null ? issue.getDetails() + " " : "";
+                return new DeprecationIssue(issue.getLevel(), issue.getMessage(), issue.getUrl(),
+                    details + "(nodes impacted: " + entry.getValue() + ")");
+            }).collect(Collectors.toList());
+    }
+
     @Override
     public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
         return new RequestBuilder(client, this);
@@ -167,32 +182,29 @@ public int hashCode() {
          * this function will run through all the checks and build out the final list of issues that exist in the
          * cluster.
          *
-         * @param nodesInfo The list of {@link NodeInfo} metadata objects for retrieving node-level information
-         * @param nodesStats The list of {@link NodeStats} metadata objects for retrieving node-level information
          * @param state The cluster state
          * @param indexNameExpressionResolver Used to resolve indices into their concrete names
          * @param indices The list of index expressions to evaluate using `indexNameExpressionResolver`
          * @param indicesOptions The options to use when resolving and filtering which indices to check
          * @param datafeeds The ml datafeed configurations
-         * @param clusterSettingsChecks The list of cluster-level checks
-         * @param nodeSettingsChecks The list of node-level checks
+         * @param nodeDeprecationResponse The response containing the deprecation issues found on each node
          * @param indexSettingsChecks The list of index-level checks that will be run across all specified
          *                            concrete indices
+         * @param clusterSettingsChecks The list of cluster-level checks
          * @param mlSettingsCheck The list of ml checks
          * @return The list of deprecation issues found in the cluster
          */
-        public static DeprecationInfoAction.Response from(List<NodeInfo> nodesInfo, List<NodeStats> nodesStats, ClusterState state,
-                                                  IndexNameExpressionResolver indexNameExpressionResolver,
-                                                  String[] indices, IndicesOptions indicesOptions,
-                                                  List<DatafeedConfig> datafeeds,
-                                                  List<Function<ClusterState,DeprecationIssue>>clusterSettingsChecks,
-                                                  List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> nodeSettingsChecks,
-                                                  List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks,
-                                                  List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsCheck) {
+        public static DeprecationInfoAction.Response from(ClusterState state,
+                                                          IndexNameExpressionResolver indexNameExpressionResolver,
+                                                          String[] indices, IndicesOptions indicesOptions,
+                                                          List<DatafeedConfig> datafeeds,
+                                                          NodesDeprecationCheckResponse nodeDeprecationResponse,
+                                                          List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks,
+                                                          List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks,
+                                                          List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsCheck) {
             List<DeprecationIssue> clusterSettingsIssues = filterChecks(clusterSettingsChecks,
                 (c) -> c.apply(state));
-            List<DeprecationIssue> nodeSettingsIssues = filterChecks(nodeSettingsChecks,
-                (c) -> c.apply(nodesInfo, nodesStats));
+            List<DeprecationIssue> nodeSettingsIssues = mergeNodeIssues(nodeDeprecationResponse);
             List<DeprecationIssue> mlSettingsIssues = new ArrayList<>();
             for (DatafeedConfig config : datafeeds) {
                 mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config)));

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.deprecation;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.support.nodes.BaseNodeRequest;
+import org.elasticsearch.action.support.nodes.BaseNodeResponse;
+import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Runs deprecation checks on each node. Deprecation checks are performed locally so that filtered settings
+ * can be accessed in the deprecation checks.
+ */
+public class NodesDeprecationCheckAction extends Action<NodesDeprecationCheckRequest, NodesDeprecationCheckResponse,
+                                                       NodesDeprecationCheckAction.RequestBuilder> {
+    public static final NodesDeprecationCheckAction INSTANCE = new NodesDeprecationCheckAction();
+    public static final String NAME = "cluster:admin/xpack/deprecation/nodes/info";
+
+    private NodesDeprecationCheckAction() {
+        super(NAME);
+    }
+
+    @Override
+    public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new RequestBuilder(client, INSTANCE, new NodesDeprecationCheckRequest());
+    }
+
+    @Override
+    public NodesDeprecationCheckResponse newResponse() {
+        return new NodesDeprecationCheckResponse();
+    }
+
+    public static class NodeRequest extends BaseNodeRequest {
+
+        NodesDeprecationCheckRequest request;
+
+        public NodeRequest() {}
+        public NodeRequest(String nodeId, NodesDeprecationCheckRequest request) {
+            super(nodeId);
+            this.request = request;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            request = new NodesDeprecationCheckRequest();
+            request.readFrom(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            request.writeTo(out);
+        }
+    }
+
+    public static class NodeResponse extends BaseNodeResponse {
+        private List<DeprecationIssue> deprecationIssues;
+
+        public NodeResponse() {
+            super();
+        }
+
+        public NodeResponse(DiscoveryNode node, List<DeprecationIssue> deprecationIssues) {
+            super(node);
+            this.deprecationIssues = deprecationIssues;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            deprecationIssues = in.readList(DeprecationIssue::new);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeList(this.deprecationIssues);
+        }
+
+        public static NodeResponse readNodeResponse(StreamInput in) throws IOException {
+            NodeResponse nodeResponse = new NodeResponse();
+            nodeResponse.readFrom(in);
+            return nodeResponse;
+        }
+
+        public List<DeprecationIssue> getDeprecationIssues() {
+            return deprecationIssues;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            NodeResponse that = (NodeResponse) o;
+            return Objects.equals(getDeprecationIssues(), that.getDeprecationIssues())
+                && Objects.equals(getNode(), that.getNode());
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(getNode(), getDeprecationIssues());
+        }
+    }
+
+    public static class RequestBuilder extends NodesOperationRequestBuilder<NodesDeprecationCheckRequest,
+        NodesDeprecationCheckResponse, RequestBuilder> {
+
+        protected RequestBuilder(ElasticsearchClient client,
+                                 Action<NodesDeprecationCheckRequest, NodesDeprecationCheckResponse, RequestBuilder> action,
+                                 NodesDeprecationCheckRequest request) {
+            super(client, action, request);
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.deprecation;
+
+import org.elasticsearch.action.support.nodes.BaseNodesRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+public class NodesDeprecationCheckRequest extends BaseNodesRequest<NodesDeprecationCheckRequest> {
+    public NodesDeprecationCheckRequest() {}
+
+    public NodesDeprecationCheckRequest(String... nodesIds) {
+        super(nodesIds);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash((Object[]) this.nodesIds());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        NodesDeprecationCheckRequest that = (NodesDeprecationCheckRequest) obj;
+        return Arrays.equals(this.nodesIds(), that.nodesIds());
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.deprecation;
+
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.support.nodes.BaseNodesResponse;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class NodesDeprecationCheckResponse extends BaseNodesResponse<NodesDeprecationCheckAction.NodeResponse> {
+
+    public NodesDeprecationCheckResponse() {}
+
+    public NodesDeprecationCheckResponse(ClusterName clusterName,
+                                         List<NodesDeprecationCheckAction.NodeResponse> nodes,
+                                         List<FailedNodeException> failures) {
+        super(clusterName, nodes, failures);
+    }
+
+    @Override
+    protected List<NodesDeprecationCheckAction.NodeResponse> readNodesFrom(StreamInput in) throws IOException {
+        return in.readList(NodesDeprecationCheckAction.NodeResponse::readNodeResponse);
+    }
+
+    @Override
+    protected void writeNodesTo(StreamOutput out, List<NodesDeprecationCheckAction.NodeResponse> nodes) throws IOException {
+        out.writeStreamableList(nodes);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NodesDeprecationCheckResponse that = (NodesDeprecationCheckResponse) o;
+        return Objects.equals(getClusterName(), that.getClusterName())
+            && Objects.equals(getNodes(), that.getNodes())
+            && Objects.equals(failures(), that.failures());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getClusterName(), getNodes(), failures());
+    }
+}