Skip to content

Commit 58a76c2

Browse files
davidkyledroberts195
davidkyle
authored and
droberts195
committed
[ML] Datafeed deprecation checks (#37932)
Deprecation checks for the ML datafeed query and aggregations.
1 parent 313fef6 commit 58a76c2

File tree

8 files changed

+237
-21
lines changed

8 files changed

+237
-21
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*/
66
package org.elasticsearch.xpack.core.deprecation;
77

8+
import org.elasticsearch.Version;
89
import org.elasticsearch.action.Action;
910
import org.elasticsearch.action.ActionRequestValidationException;
1011
import org.elasticsearch.action.ActionResponse;
@@ -23,9 +24,12 @@
2324
import org.elasticsearch.common.io.stream.StreamOutput;
2425
import org.elasticsearch.common.xcontent.ToXContentObject;
2526
import org.elasticsearch.common.xcontent.XContentBuilder;
27+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
2628

2729
import java.io.IOException;
30+
import java.util.ArrayList;
2831
import java.util.Arrays;
32+
import java.util.Collections;
2933
import java.util.HashMap;
3034
import java.util.List;
3135
import java.util.Map;
@@ -73,16 +77,19 @@ public static class Response extends ActionResponse implements ToXContentObject
7377
private List<DeprecationIssue> clusterSettingsIssues;
7478
private List<DeprecationIssue> nodeSettingsIssues;
7579
private Map<String, List<DeprecationIssue>> indexSettingsIssues;
80+
private List<DeprecationIssue> mlSettingsIssues;
7681

7782
public Response() {
7883
}
7984

8085
public Response(List<DeprecationIssue> clusterSettingsIssues,
8186
List<DeprecationIssue> nodeSettingsIssues,
82-
Map<String, List<DeprecationIssue>> indexSettingsIssues) {
87+
Map<String, List<DeprecationIssue>> indexSettingsIssues,
88+
List<DeprecationIssue> mlSettingsIssues) {
8389
this.clusterSettingsIssues = clusterSettingsIssues;
8490
this.nodeSettingsIssues = nodeSettingsIssues;
8591
this.indexSettingsIssues = indexSettingsIssues;
92+
this.mlSettingsIssues = mlSettingsIssues;
8693
}
8794

8895
public List<DeprecationIssue> getClusterSettingsIssues() {
@@ -97,12 +104,22 @@ public Map<String, List<DeprecationIssue>> getIndexSettingsIssues() {
97104
return indexSettingsIssues;
98105
}
99106

107+
public List<DeprecationIssue> getMlSettingsIssues() {
108+
return mlSettingsIssues;
109+
}
110+
100111
@Override
101112
public void readFrom(StreamInput in) throws IOException {
102113
super.readFrom(in);
103114
clusterSettingsIssues = in.readList(DeprecationIssue::new);
104115
nodeSettingsIssues = in.readList(DeprecationIssue::new);
105116
indexSettingsIssues = in.readMapOfLists(StreamInput::readString, DeprecationIssue::new);
117+
if ((in.getVersion().onOrAfter(Version.V_5_6_5) && in.getVersion().before(Version.V_6_0_0))
118+
|| in.getVersion().onOrAfter(Version.V_6_7_0)) {
119+
mlSettingsIssues = in.readList(DeprecationIssue::new);
120+
} else {
121+
mlSettingsIssues = Collections.emptyList();
122+
}
106123
}
107124

108125
@Override
@@ -111,6 +128,10 @@ public void writeTo(StreamOutput out) throws IOException {
111128
out.writeList(clusterSettingsIssues);
112129
out.writeList(nodeSettingsIssues);
113130
out.writeMapOfLists(indexSettingsIssues, StreamOutput::writeString, (o, v) -> v.writeTo(o));
131+
if ((out.getVersion().onOrAfter(Version.V_5_6_5) && out.getVersion().before(Version.V_6_0_0))
132+
|| out.getVersion().onOrAfter(Version.V_6_7_0)) {
133+
out.writeList(mlSettingsIssues);
134+
}
114135
}
115136

116137
@Override
@@ -119,24 +140,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
119140
.array("cluster_settings", clusterSettingsIssues.toArray())
120141
.array("node_settings", nodeSettingsIssues.toArray())
121142
.field("index_settings")
122-
.map(indexSettingsIssues)
143+
.map(indexSettingsIssues)
144+
.array("ml_settings", mlSettingsIssues.toArray())
123145
.endObject();
124146
}
125147

126-
127148
@Override
128149
public boolean equals(Object o) {
129150
if (this == o) return true;
130151
if (o == null || getClass() != o.getClass()) return false;
131152
Response response = (Response) o;
132153
return Objects.equals(clusterSettingsIssues, response.clusterSettingsIssues) &&
133154
Objects.equals(nodeSettingsIssues, response.nodeSettingsIssues) &&
134-
Objects.equals(indexSettingsIssues, response.indexSettingsIssues);
155+
Objects.equals(indexSettingsIssues, response.indexSettingsIssues) &&
156+
Objects.equals(mlSettingsIssues, response.mlSettingsIssues);
135157
}
136158

137159
@Override
138160
public int hashCode() {
139-
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues);
161+
return Objects.hash(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues);
140162
}
141163

142164
/**
@@ -151,22 +173,30 @@ public int hashCode() {
151173
* @param indexNameExpressionResolver Used to resolve indices into their concrete names
152174
* @param indices The list of index expressions to evaluate using `indexNameExpressionResolver`
153175
* @param indicesOptions The options to use when resolving and filtering which indices to check
176+
* @param datafeeds The ml datafeed configurations
154177
* @param clusterSettingsChecks The list of cluster-level checks
155178
* @param nodeSettingsChecks The list of node-level checks
156179
* @param indexSettingsChecks The list of index-level checks that will be run across all specified
157180
* concrete indices
181+
* @param mlSettingsCheck The list of ml checks
158182
* @return The list of deprecation issues found in the cluster
159183
*/
160184
public static DeprecationInfoAction.Response from(List<NodeInfo> nodesInfo, List<NodeStats> nodesStats, ClusterState state,
161-
IndexNameExpressionResolver indexNameExpressionResolver,
162-
String[] indices, IndicesOptions indicesOptions,
163-
List<Function<ClusterState,DeprecationIssue>>clusterSettingsChecks,
164-
List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> nodeSettingsChecks,
165-
List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks) {
185+
IndexNameExpressionResolver indexNameExpressionResolver,
186+
String[] indices, IndicesOptions indicesOptions,
187+
List<DatafeedConfig> datafeeds,
188+
List<Function<ClusterState,DeprecationIssue>>clusterSettingsChecks,
189+
List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> nodeSettingsChecks,
190+
List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks,
191+
List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsCheck) {
166192
List<DeprecationIssue> clusterSettingsIssues = filterChecks(clusterSettingsChecks,
167193
(c) -> c.apply(state));
168194
List<DeprecationIssue> nodeSettingsIssues = filterChecks(nodeSettingsChecks,
169195
(c) -> c.apply(nodesInfo, nodesStats));
196+
List<DeprecationIssue> mlSettingsIssues = new ArrayList<>();
197+
for (DatafeedConfig config : datafeeds) {
198+
mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config)));
199+
}
170200

171201
String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, indices);
172202

@@ -180,7 +210,7 @@ public static DeprecationInfoAction.Response from(List<NodeInfo> nodesInfo, List
180210
}
181211
}
182212

183-
return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues);
213+
return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, mlSettingsIssues);
184214
}
185215
}
186216

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ public void setParsedQuery(QueryBuilder query) {
692692
}
693693
}
694694

695-
void setQuery(Map<String, Object> query) {
695+
public void setQuery(Map<String, Object> query) {
696696
this.query = ExceptionsHelper.requireNonNull(query, QUERY.getPreferredName());
697697
}
698698

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import org.elasticsearch.common.xcontent.XContentBuilder;
2323
import org.elasticsearch.common.xcontent.XContentFactory;
2424
import org.elasticsearch.test.AbstractStreamableTestCase;
25+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
26+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigTests;
2527

2628
import java.io.IOException;
2729
import java.util.Arrays;
@@ -45,13 +47,15 @@ protected DeprecationInfoAction.Response createTestInstance() {
4547
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
4648
List<DeprecationIssue> nodeIssues = Stream.generate(DeprecationIssueTests::createTestInstance)
4749
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
50+
List<DeprecationIssue> mlIssues = Stream.generate(DeprecationIssueTests::createTestInstance)
51+
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
4852
Map<String, List<DeprecationIssue>> indexIssues = new HashMap<>();
4953
for (int i = 0; i < randomIntBetween(0, 10); i++) {
5054
List<DeprecationIssue> perIndexIssues = Stream.generate(DeprecationIssueTests::createTestInstance)
5155
.limit(randomIntBetween(0, 10)).collect(Collectors.toList());
5256
indexIssues.put(randomAlphaOfLength(10), perIndexIssues);
5357
}
54-
return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues);
58+
return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues, mlIssues);
5559
}
5660

5761
@Override
@@ -80,12 +84,14 @@ public void testFrom() throws IOException {
8084
List<NodeStats> nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null,
8185
null, null, null, null, null, null, null, null,
8286
null, null, null, null));
87+
List<DatafeedConfig> datafeeds = Collections.singletonList(DatafeedConfigTests.createRandomizedDatafeedConfig("foo"));
8388
IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(Settings.EMPTY);
8489
IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false,
8590
true, true);
8691
boolean clusterIssueFound = randomBoolean();
8792
boolean nodeIssueFound = randomBoolean();
8893
boolean indexIssueFound = randomBoolean();
94+
boolean mlIssueFound = randomBoolean();
8995
DeprecationIssue foundIssue = DeprecationIssueTests.createTestInstance();
9096
List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks =
9197
Collections.unmodifiableList(Arrays.asList(
@@ -100,10 +106,14 @@ public void testFrom() throws IOException {
100106
Collections.unmodifiableList(Arrays.asList(
101107
(idx) -> indexIssueFound ? foundIssue : null
102108
));
109+
List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsChecks =
110+
Collections.unmodifiableList(Arrays.asList(
111+
(idx) -> mlIssueFound ? foundIssue : null
112+
));
103113

104114
DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(nodeInfos, nodeStats, state,
105-
resolver, Strings.EMPTY_ARRAY, indicesOptions,
106-
clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks);
115+
resolver, Strings.EMPTY_ARRAY, indicesOptions, datafeeds,
116+
clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks, mlSettingsChecks);
107117

108118
if (clusterIssueFound) {
109119
assertThat(response.getClusterSettingsIssues(), equalTo(Collections.singletonList(foundIssue)));
@@ -123,5 +133,11 @@ public void testFrom() throws IOException {
123133
} else {
124134
assertTrue(response.getIndexSettingsIssues().isEmpty());
125135
}
136+
137+
if (mlIssueFound) {
138+
assertThat(response.getMlSettingsIssues(), equalTo(Collections.singletonList(foundIssue)));
139+
} else {
140+
assertTrue(response.getMlSettingsIssues().isEmpty());
141+
}
126142
}
127143
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.cluster.metadata.IndexMetaData;
1212
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
1313
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
14+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
1415

1516
import java.util.Arrays;
1617
import java.util.Collections;
@@ -64,6 +65,12 @@ private DeprecationChecks() {
6465
IndexDeprecationChecks::shardOnStartupCheck,
6566
IndexDeprecationChecks::classicSimilarityMappingCheck,
6667
IndexDeprecationChecks::classicSimilaritySettingsCheck
68+
));
69+
70+
static List<Function<DatafeedConfig, DeprecationIssue>> ML_SETTINGS_CHECKS =
71+
Collections.unmodifiableList(Arrays.asList(
72+
MlDeprecationChecks::checkDataFeedAggregations,
73+
MlDeprecationChecks::checkDataFeedQuery
6774
));
6875

6976
/**

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecks.java

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.deprecation;
7+
8+
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
9+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
10+
11+
import java.util.List;
12+
13+
/**
14+
* Check the {@link DatafeedConfig} query and aggregations for deprecated usages.
15+
*/
16+
final class MlDeprecationChecks {
17+
18+
private MlDeprecationChecks() {
19+
}
20+
21+
static DeprecationIssue checkDataFeedQuery(DatafeedConfig datafeedConfig) {
22+
List<String> deprecations = datafeedConfig.getQueryDeprecations();
23+
if (deprecations.isEmpty()) {
24+
return null;
25+
} else {
26+
return new DeprecationIssue(DeprecationIssue.Level.WARNING,
27+
"Datafeed [" + datafeedConfig.getId() + "] uses deprecated query options",
28+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#breaking_70_search_changes",
29+
deprecations.toString());
30+
}
31+
}
32+
33+
static DeprecationIssue checkDataFeedAggregations(DatafeedConfig datafeedConfig) {
34+
List<String> deprecations = datafeedConfig.getAggDeprecations();
35+
if (deprecations.isEmpty()) {
36+
return null;
37+
} else {
38+
return new DeprecationIssue(DeprecationIssue.Level.WARNING,
39+
"Datafeed [" + datafeedConfig.getId() + "] uses deprecated aggregation options",
40+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" +
41+
"#breaking_70_aggregations_changes", deprecations.toString());
42+
}
43+
}
44+
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,20 +26,28 @@
2626
import org.elasticsearch.threadpool.ThreadPool;
2727
import org.elasticsearch.transport.TransportService;
2828
import org.elasticsearch.xpack.core.XPackField;
29+
import org.elasticsearch.xpack.core.XPackSettings;
2930
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
31+
import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
32+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
33+
34+
import java.util.Collections;
35+
import java.util.List;
3036

3137
import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN;
3238
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
3339
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS;
3440
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS;
3541
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.NODE_SETTINGS_CHECKS;
42+
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.ML_SETTINGS_CHECKS;
3643

3744
public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction<DeprecationInfoAction.Request,
3845
DeprecationInfoAction.Response> {
3946

4047
private final XPackLicenseState licenseState;
4148
private final NodeClient client;
4249
private final IndexNameExpressionResolver indexNameExpressionResolver;
50+
private final Settings settings;
4351

4452
@Inject
4553
public TransportDeprecationInfoAction(Settings settings, TransportService transportService,
@@ -52,6 +60,7 @@ public TransportDeprecationInfoAction(Settings settings, TransportService transp
5260
this.licenseState = licenseState;
5361
this.client = client;
5462
this.indexNameExpressionResolver = indexNameExpressionResolver;
63+
this.settings = settings;
5564
}
5665

5766
@Override
@@ -90,17 +99,34 @@ protected final void masterOperation(final DeprecationInfoAction.Request request
9099
if (nodesStatsResponse.hasFailures()) {
91100
throw nodesStatsResponse.failures().get(0);
92101
}
93-
listener.onResponse(
94-
DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(),
95-
nodesStatsResponse.getNodes(), state, indexNameExpressionResolver,
96-
request.indices(), request.indicesOptions(),
97-
CLUSTER_SETTINGS_CHECKS, NODE_SETTINGS_CHECKS,
98-
INDEX_SETTINGS_CHECKS));
102+
getDatafeedConfigs(ActionListener.wrap(
103+
datafeeds -> {
104+
listener.onResponse(
105+
DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(),
106+
nodesStatsResponse.getNodes(), state, indexNameExpressionResolver,
107+
request.indices(), request.indicesOptions(), datafeeds,
108+
CLUSTER_SETTINGS_CHECKS, NODE_SETTINGS_CHECKS,
109+
INDEX_SETTINGS_CHECKS, ML_SETTINGS_CHECKS));
110+
},
111+
listener::onFailure
112+
));
99113
}, listener::onFailure),
100114
client.admin().cluster()::nodesStats);
101115
}, listener::onFailure), client.admin().cluster()::nodesInfo);
102116
} else {
103117
listener.onFailure(LicenseUtils.newComplianceException(XPackField.DEPRECATION));
104118
}
105119
}
120+
121+
private void getDatafeedConfigs(ActionListener<List<DatafeedConfig>> listener) {
122+
if (XPackSettings.MACHINE_LEARNING_ENABLED.get(settings) == false) {
123+
listener.onResponse(Collections.emptyList());
124+
} else {
125+
executeAsyncWithOrigin(client, DEPRECATION_ORIGIN, GetDatafeedsAction.INSTANCE,
126+
new GetDatafeedsAction.Request(GetDatafeedsAction.ALL), ActionListener.wrap(
127+
datafeedsResponse -> listener.onResponse(datafeedsResponse.getResponse().results()),
128+
listener::onFailure
129+
));
130+
}
131+
}
106132
}

0 commit comments

Comments
 (0)