Skip to content

Commit be78816

Browse files
droberts195droberts195
authored
[ML] Datafeed deprecation checks (#38026)
Deprecation checks for the ML datafeed query and aggregations.
1 parent 81c443c commit be78816

File tree

8 files changed

+239
-20
lines changed

8 files changed

+239
-20
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*/
66
package org.elasticsearch.xpack.core.deprecation;
77

8+
import org.elasticsearch.Version;
89
import org.elasticsearch.action.Action;
910
import org.elasticsearch.action.ActionRequestValidationException;
1011
import org.elasticsearch.action.ActionResponse;
@@ -23,9 +24,12 @@
2324
import org.elasticsearch.common.io.stream.StreamOutput;
2425
import org.elasticsearch.common.xcontent.ToXContentObject;
2526
import org.elasticsearch.common.xcontent.XContentBuilder;
27+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
2628

2729
import java.io.IOException;
30+
import java.util.ArrayList;
2831
import java.util.Arrays;
32+
import java.util.Collections;
2933
import java.util.HashMap;
3034
import java.util.List;
3135
import java.util.Map;
@@ -67,16 +71,19 @@ public static class Response extends ActionResponse implements ToXContentObject
6771
private List<DeprecationIssue> clusterSettingsIssues;
6872
private List<DeprecationIssue> nodeSettingsIssues;
6973
private Map<String, List<DeprecationIssue>> indexSettingsIssues;
74+
private List<DeprecationIssue> mlSettingsIssues;
7075

7176
public Response() {
7277
}
7378

7479
public Response(List<DeprecationIssue> clusterSettingsIssues,
7580
List<DeprecationIssue> nodeSettingsIssues,
76-
Map<String, List<DeprecationIssue>> indexSettingsIssues) {
81+
Map<String, List<DeprecationIssue>> indexSettingsIssues,
82+
List<DeprecationIssue> mlSettingsIssues) {
7783
this.clusterSettingsIssues = clusterSettingsIssues;
7884
this.nodeSettingsIssues = nodeSettingsIssues;
7985
this.indexSettingsIssues = indexSettingsIssues;
86+
this.mlSettingsIssues = mlSettingsIssues;
8087
}
8188

8289
public List<DeprecationIssue> getClusterSettingsIssues() {
@@ -91,12 +98,21 @@ public Map<String, List<DeprecationIssue>> getIndexSettingsIssues() {
9198
return indexSettingsIssues;
9299
}
93100

101+
public List<DeprecationIssue> getMlSettingsIssues() {
102+
return mlSettingsIssues;
103+
}
104+
94105
@Override
95106
public void readFrom(StreamInput in) throws IOException {
96107
super.readFrom(in);
97108
clusterSettingsIssues = in.readList(DeprecationIssue::new);
98109
nodeSettingsIssues = in.readList(DeprecationIssue::new);
99110
indexSettingsIssues = in.readMapOfLists(StreamInput::readString, DeprecationIssue::new);
111+
if (in.getVersion().onOrAfter(Version.V_6_7_0)) {
112+
mlSettingsIssues = in.readList(DeprecationIssue::new);
113+
} else {
114+
mlSettingsIssues = Collections.emptyList();
115+
}
100116
}
101117

102118
@Override
@@ -105,6 +121,9 @@ public void writeTo(StreamOutput out) throws IOException {
105121
out.writeList(clusterSettingsIssues);
106122
out.writeList(nodeSettingsIssues);
107123
out.writeMapOfLists(indexSettingsIssues, StreamOutput::writeString, (o, v) -> v.writeTo(o));
124+
if (out.getVersion().onOrAfter(Version.V_6_7_0)) {
125+
out.writeList(mlSettingsIssues);
126+
}
108127
}
109128

110129
@Override
@@ -114,23 +133,24 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
114133
.array("node_settings", nodeSettingsIssues.toArray())
115134
.field("index_settings")
116135
.map(indexSettingsIssues)
136+
.array("ml_settings", mlSettingsIssues.toArray())
117137
.endObject();
118138
}
119139

120-
121140
@Override
122141
public boolean equals(Object o) {
123142
if (this == o) return true;
124143
if (o == null || getClass() != o.getClass()) return false;
125144
Response response = (Response) o;
126145
return Objects.equals(clusterSettingsIssues, response.clusterSettingsIssues) &&
127146
Objects.equals(nodeSettingsIssues, response.nodeSettingsIssues) &&
128-
Objects.equals(indexSettingsIssues, response.indexSettingsIssues);
147+
Objects.equals(indexSettingsIssues, response.indexSettingsIssues) &&
148+
Objects.equals(mlSettingsIssues, response.mlSettingsIssues);
129149
}
130150

131151
@Override
132152
public int hashCode() {
133-
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues);
153+
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues);
134154
}
135155

136156
/**
@@ -145,22 +165,30 @@ public int hashCode() {
145165
* @param indexNameExpressionResolver Used to resolve indices into their concrete names
146166
* @param indices The list of index expressions to evaluate using `indexNameExpressionResolver`
147167
* @param indicesOptions The options to use when resolving and filtering which indices to check
168+
* @param datafeeds The ml datafeed configurations
148169
* @param clusterSettingsChecks The list of cluster-level checks
149170
* @param nodeSettingsChecks The list of node-level checks
150171
* @param indexSettingsChecks The list of index-level checks that will be run across all specified
151172
* concrete indices
173+
* @param mlSettingsCheck The list of ml checks
152174
* @return The list of deprecation issues found in the cluster
153175
*/
154176
public static DeprecationInfoAction.Response from(List<NodeInfo> nodesInfo, List<NodeStats> nodesStats, ClusterState state,
155-
IndexNameExpressionResolver indexNameExpressionResolver,
156-
String[] indices, IndicesOptions indicesOptions,
157-
List<Function<ClusterState,DeprecationIssue>>clusterSettingsChecks,
158-
List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> nodeSettingsChecks,
159-
List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks) {
177+
IndexNameExpressionResolver indexNameExpressionResolver,
178+
String[] indices, IndicesOptions indicesOptions,
179+
List<DatafeedConfig> datafeeds,
180+
List<Function<ClusterState,DeprecationIssue>>clusterSettingsChecks,
181+
List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> nodeSettingsChecks,
182+
List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks,
183+
List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsCheck) {
160184
List<DeprecationIssue> clusterSettingsIssues = filterChecks(clusterSettingsChecks,
161185
(c) -> c.apply(state));
162186
List<DeprecationIssue> nodeSettingsIssues = filterChecks(nodeSettingsChecks,
163187
(c) -> c.apply(nodesInfo, nodesStats));
188+
List<DeprecationIssue> mlSettingsIssues = new ArrayList<>();
189+
for (DatafeedConfig config : datafeeds) {
190+
mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config)));
191+
}
164192

165193
String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, indices);
166194

@@ -174,7 +202,7 @@ public static DeprecationInfoAction.Response from(List<NodeInfo> nodesInfo, List
174202
}
175203
}
176204

177-
return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues);
205+
return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues);
178206
}
179207
}
180208

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -665,7 +665,7 @@ public void setParsedQuery(QueryBuilder query) {
665665
}
666666
}
667667

668-
void setQuery(Map<String, Object> query) {
668+
public void setQuery(Map<String, Object> query) {
669669
this.query = ExceptionsHelper.requireNonNull(query, QUERY.getPreferredName());
670670
}
671671

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import org.elasticsearch.common.xcontent.XContentBuilder;
2323
import org.elasticsearch.common.xcontent.XContentFactory;
2424
import org.elasticsearch.test.AbstractStreamableTestCase;
25+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
26+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigTests;
2527

2628
import java.io.IOException;
2729
import java.util.Arrays;
@@ -45,13 +47,15 @@ protected DeprecationInfoAction.Response createTestInstance() {
4547
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
4648
List<DeprecationIssue> nodeIssues = Stream.generate(DeprecationIssueTests::createTestInstance)
4749
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
50+
List<DeprecationIssue> mlIssues = Stream.generate(DeprecationIssueTests::createTestInstance)
51+
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
4852
Map<String, List<DeprecationIssue>> indexIssues = new HashMap<>();
4953
for (int i = 0; i < randomIntBetween(0, 10); i++) {
5054
List<DeprecationIssue> perIndexIssues = Stream.generate(DeprecationIssueTests::createTestInstance)
5155
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
5256
indexIssues.put(randomAlphaOfLength(10), perIndexIssues);
5357
}
54-
return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues);
58+
return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues, mlIssues);
5559
}
5660

5761
@Override
@@ -80,12 +84,14 @@ public void testFrom() throws IOException {
8084
List<NodeStats> nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null,
8185
null, null, null, null, null, null, null, null,
8286
null, null, null, null));
87+
List<DatafeedConfig> datafeeds = Collections.singletonList(DatafeedConfigTests.createRandomizedDatafeedConfig("foo"));
8388
IndexNameExpressionResolver resolver = new IndexNameExpressionResolver();
8489
IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false,
8590
true, true);
8691
boolean clusterIssueFound = randomBoolean();
8792
boolean nodeIssueFound = randomBoolean();
8893
boolean indexIssueFound = randomBoolean();
94+
boolean mlIssueFound = randomBoolean();
8995
DeprecationIssue foundIssue = DeprecationIssueTests.createTestInstance();
9096
List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks =
9197
Collections.unmodifiableList(Arrays.asList(
@@ -100,10 +106,14 @@ public void testFrom() throws IOException {
100106
Collections.unmodifiableList(Arrays.asList(
101107
(idx) -> indexIssueFound ? foundIssue : null
102108
));
109+
List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsChecks =
110+
Collections.unmodifiableList(Arrays.asList(
111+
(idx) -> mlIssueFound ? foundIssue : null
112+
));
103113

104114
DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(nodeInfos, nodeStats, state,
105-
resolver, Strings.EMPTY_ARRAY, indicesOptions,
106-
clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks);
115+
resolver, Strings.EMPTY_ARRAY, indicesOptions, datafeeds,
116+
clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks, mlSettingsChecks);
107117

108118
if (clusterIssueFound) {
109119
assertThat(response.getClusterSettingsIssues(), equalTo(Collections.singletonList(foundIssue)));
@@ -123,5 +133,11 @@ public void testFrom() throws IOException {
123133
} else {
124134
assertTrue(response.getIndexSettingsIssues().isEmpty());
125135
}
136+
137+
if (mlIssueFound) {
138+
assertThat(response.getMlSettingsIssues(), equalTo(Collections.singletonList(foundIssue)));
139+
} else {
140+
assertTrue(response.getMlSettingsIssues().isEmpty());
141+
}
126142
}
127143
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.cluster.metadata.IndexMetaData;
1212
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
1313
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
14+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
1415

1516
import java.util.Arrays;
1617
import java.util.Collections;
@@ -41,6 +42,12 @@ private DeprecationChecks() {
4142
Collections.unmodifiableList(Arrays.asList(
4243
IndexDeprecationChecks::oldIndicesCheck));
4344

45+
static List<Function<DatafeedConfig, DeprecationIssue>> ML_SETTINGS_CHECKS =
46+
Collections.unmodifiableList(Arrays.asList(
47+
MlDeprecationChecks::checkDataFeedAggregations,
48+
MlDeprecationChecks::checkDataFeedQuery
49+
));
50+
4451
/**
4552
* helper utility function to reduce repeat of running a specific {@link List} of checks.
4653
*

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.deprecation;
7+
8+
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
9+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
10+
11+
import java.util.List;
12+
13+
/**
14+
* Check the {@link DatafeedConfig} query and aggregations for deprecated usages.
15+
*/
16+
final class MlDeprecationChecks {
17+
18+
private MlDeprecationChecks() {
19+
}
20+
21+
static DeprecationIssue checkDataFeedQuery(DatafeedConfig datafeedConfig) {
22+
List<String> deprecations = datafeedConfig.getQueryDeprecations();
23+
if (deprecations.isEmpty()) {
24+
return null;
25+
} else {
26+
return new DeprecationIssue(DeprecationIssue.Level.WARNING,
27+
"Datafeed [" + datafeedConfig.getId() + "] uses deprecated query options",
28+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#breaking_70_search_changes",
29+
deprecations.toString());
30+
}
31+
}
32+
33+
static DeprecationIssue checkDataFeedAggregations(DatafeedConfig datafeedConfig) {
34+
List<String> deprecations = datafeedConfig.getAggDeprecations();
35+
if (deprecations.isEmpty()) {
36+
return null;
37+
} else {
38+
return new DeprecationIssue(DeprecationIssue.Level.WARNING,
39+
"Datafeed [" + datafeedConfig.getId() + "] uses deprecated aggregation options",
40+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" +
41+
"#breaking_70_aggregations_changes", deprecations.toString());
42+
}
43+
}
44+
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,24 +19,32 @@
1919
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2020
import org.elasticsearch.cluster.service.ClusterService;
2121
import org.elasticsearch.common.inject.Inject;
22+
import org.elasticsearch.common.settings.Settings;
2223
import org.elasticsearch.common.util.concurrent.ThreadContext;
2324
import org.elasticsearch.license.LicenseUtils;
2425
import org.elasticsearch.license.XPackLicenseState;
2526
import org.elasticsearch.threadpool.ThreadPool;
2627
import org.elasticsearch.transport.TransportService;
2728
import org.elasticsearch.xpack.core.ClientHelper;
2829
import org.elasticsearch.xpack.core.XPackField;
30+
import org.elasticsearch.xpack.core.XPackSettings;
2931
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
32+
import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
33+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
34+
35+
import java.util.Collections;
36+
import java.util.List;
3037

3138
public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction<DeprecationInfoAction.Request,
3239
DeprecationInfoAction.Response> {
3340

3441
private final XPackLicenseState licenseState;
3542
private final NodeClient client;
3643
private final IndexNameExpressionResolver indexNameExpressionResolver;
44+
private final Settings settings;
3745

3846
@Inject
39-
public TransportDeprecationInfoAction(TransportService transportService, ClusterService clusterService,
47+
public TransportDeprecationInfoAction(Settings settings, TransportService transportService, ClusterService clusterService,
4048
ThreadPool threadPool, ActionFilters actionFilters,
4149
IndexNameExpressionResolver indexNameExpressionResolver,
4250
XPackLicenseState licenseState, NodeClient client) {
@@ -45,6 +53,7 @@ public TransportDeprecationInfoAction(TransportService transportService, Cluster
4553
this.licenseState = licenseState;
4654
this.client = client;
4755
this.indexNameExpressionResolver = indexNameExpressionResolver;
56+
this.settings = settings;
4857
}
4958

5059
@Override
@@ -83,16 +92,37 @@ protected final void masterOperation(final DeprecationInfoAction.Request request
8392
if (nodesStatsResponse.hasFailures()) {
8493
throw nodesStatsResponse.failures().get(0);
8594
}
86-
listener.onResponse(DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(),
87-
nodesStatsResponse.getNodes(), state, indexNameExpressionResolver,
88-
request.indices(), request.indicesOptions(),
89-
DeprecationChecks.CLUSTER_SETTINGS_CHECKS, DeprecationChecks.NODE_SETTINGS_CHECKS,
90-
DeprecationChecks.INDEX_SETTINGS_CHECKS));
95+
96+
getDatafeedConfigs(ActionListener.wrap(
97+
datafeeds -> {
98+
listener.onResponse(
99+
DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(),
100+
nodesStatsResponse.getNodes(), state, indexNameExpressionResolver,
101+
request.indices(), request.indicesOptions(), datafeeds,
102+
DeprecationChecks.CLUSTER_SETTINGS_CHECKS,
103+
DeprecationChecks.NODE_SETTINGS_CHECKS,
104+
DeprecationChecks.INDEX_SETTINGS_CHECKS,
105+
DeprecationChecks.ML_SETTINGS_CHECKS));
106+
},
107+
listener::onFailure
108+
));
91109
}, listener::onFailure),
92110
client.admin().cluster()::nodesStats);
93111
}, listener::onFailure), client.admin().cluster()::nodesInfo);
94112
} else {
95113
listener.onFailure(LicenseUtils.newComplianceException(XPackField.DEPRECATION));
96114
}
97115
}
116+
117+
private void getDatafeedConfigs(ActionListener<List<DatafeedConfig>> listener) {
118+
if (XPackSettings.MACHINE_LEARNING_ENABLED.get(settings) == false) {
119+
listener.onResponse(Collections.emptyList());
120+
} else {
121+
ClientHelper.executeAsyncWithOrigin(client, ClientHelper.DEPRECATION_ORIGIN, GetDatafeedsAction.INSTANCE,
122+
new GetDatafeedsAction.Request(GetDatafeedsAction.ALL), ActionListener.wrap(
123+
datafeedsResponse -> listener.onResponse(datafeedsResponse.getResponse().results()),
124+
listener::onFailure
125+
));
126+
}
127+
}
98128
}

0 commit comments

Comments
 (0)