Skip to content

Commit 61f0c47

Browse files
author
Hendrik Muhs
authored
[Transform] Enhance transform role checks (#70139)
improve robustness and ux in case of a missing transform node: - warn if cluster lacks a transform node in all API's (except DELETE) - report waiting state in stats if transform waits for assignment - cancel p-task on stop transform even if config has been deleted relates #69518
1 parent ff50da5 commit 61f0c47

File tree

16 files changed

+391
-79
lines changed

16 files changed

+391
-79
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformStats.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public boolean equals(Object other) {
117117

118118
public enum State {
119119

120-
STARTED, INDEXING, ABORTING, STOPPING, STOPPED, FAILED;
120+
STARTED, INDEXING, ABORTING, STOPPING, STOPPED, FAILED, WAITING;
121121

122122
public static State fromString(String name) {
123123
return valueOf(name.trim().toUpperCase(Locale.ROOT));

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java

+2
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ public class TransformMessages {
3030
public static final String UNKNOWN_TRANSFORM_STATS = "Statistics for transform [{0}] could not be found";
3131

3232
public static final String REST_DEPRECATED_ENDPOINT = "[_data_frame/transforms/] is deprecated, use [_transform/] in the future.";
33+
public static final String REST_WARN_NO_TRANSFORM_NODES =
34+
"Transform requires the transform node role for at least 1 node, found no transform nodes";
3335

3436
public static final String CANNOT_STOP_FAILED_TRANSFORM = "Unable to stop transform [{0}] as it is in a failed state with reason [{1}]."
3537
+ " Use force stop to stop the transform.";

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformStats.java

+9-2
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
150150
public void writeTo(StreamOutput out) throws IOException {
151151
if (out.getVersion().onOrAfter(Version.V_7_4_0)) {
152152
out.writeString(id);
153-
out.writeEnum(state);
153+
// 7.13 introduced the waiting state, in older version report the state as started
154+
if (out.getVersion().before(Version.V_8_0_0) && state.equals(State.WAITING)) { // TODO: V_7_13_0
155+
out.writeEnum(State.STARTED);
156+
} else {
157+
out.writeEnum(state);
158+
}
154159
out.writeOptionalString(reason);
155160
if (node != null) {
156161
out.writeBoolean(true);
@@ -247,7 +252,8 @@ public enum State implements Writeable {
247252
ABORTING,
248253
STOPPING,
249254
STOPPED,
250-
FAILED;
255+
FAILED,
256+
WAITING;
251257

252258
public static State fromString(String name) {
253259
return valueOf(name.trim().toUpperCase(Locale.ROOT));
@@ -299,6 +305,7 @@ public String value() {
299305
return name().toLowerCase(Locale.ROOT);
300306
}
301307

308+
// only used when speaking to nodes < 7.4 (can be removed for 8.0)
302309
public Tuple<TransformTaskState, IndexerState> toComponents() {
303310

304311
switch (this) {

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java

+30
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import java.util.function.Predicate;
1919

2020
import static org.elasticsearch.xpack.core.transform.transforms.TransformStats.State.STARTED;
21+
import static org.elasticsearch.xpack.core.transform.transforms.TransformStats.State.WAITING;
2122
import static org.hamcrest.Matchers.equalTo;
2223

2324
public class TransformStatsTests extends AbstractSerializingTestCase<TransformStats> {
@@ -120,4 +121,33 @@ public void testBwcWith76() throws IOException {
120121
}
121122
}
122123
}
124+
125+
public void testBwcWith712() throws IOException {
126+
for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
127+
TransformStats stats = new TransformStats(
128+
"bwc-id",
129+
WAITING,
130+
randomBoolean() ? null : randomAlphaOfLength(100),
131+
randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(),
132+
new TransformIndexerStats(1, 2, 3, 0, 5, 6, 7, 0, 0, 10, 11, 0, 13, 14, 15.0, 16.0, 17.0),
133+
new TransformCheckpointingInfo(
134+
new TransformCheckpointStats(0, null, null, 10, 100),
135+
new TransformCheckpointStats(0, null, null, 100, 1000),
136+
// changesLastDetectedAt aren't serialized back
137+
100,
138+
null,
139+
null
140+
)
141+
);
142+
try (BytesStreamOutput output = new BytesStreamOutput()) {
143+
output.setVersion(Version.V_7_12_0);
144+
stats.writeTo(output);
145+
try (StreamInput in = output.bytes().streamInput()) {
146+
in.setVersion(Version.V_8_0_0); // TODO: V_7_13_0
147+
TransformStats statsFromOld = new TransformStats(in);
148+
assertThat(statsFromOld.getState(), equalTo(STARTED));
149+
}
150+
}
151+
}
152+
}
123153
}

‎x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/TransformSingleNodeTestCase.java

+7
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@
1313
import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
1414
import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
1515
import org.elasticsearch.common.CheckedConsumer;
16+
import org.elasticsearch.common.settings.Settings;
1617
import org.elasticsearch.index.reindex.ReindexPlugin;
18+
import org.elasticsearch.node.NodeRoleSettings;
1719
import org.elasticsearch.plugins.Plugin;
1820
import org.elasticsearch.tasks.TaskInfo;
1921
import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -32,6 +34,11 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
3234
return pluginList(LocalStateTransform.class, ReindexPlugin.class);
3335
}
3436

37+
@Override
38+
protected Settings nodeSettings() {
39+
return Settings.builder().put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "master, data, ingest, transform").build();
40+
}
41+
3542
protected <T> void assertAsync(Consumer<ActionListener<T>> function, T expected, CheckedConsumer<T, ? extends Exception> onAnswer,
3643
Consumer<Exception> onException) throws InterruptedException {
3744

‎x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.transform.integration;
9+
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.node.NodeRoleSettings;
12+
import org.elasticsearch.xpack.core.transform.action.GetTransformAction;
13+
import org.elasticsearch.xpack.core.transform.action.GetTransformStatsAction;
14+
import org.elasticsearch.xpack.transform.TransformSingleNodeTestCase;
15+
16+
public class TransformNoTransformNodeIT extends TransformSingleNodeTestCase {
17+
@Override
18+
protected Settings nodeSettings() {
19+
return Settings.builder().put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "master, data, ingest").build();
20+
}
21+
22+
public void testWarningForStats() {
23+
GetTransformStatsAction.Request getTransformStatsRequest = new GetTransformStatsAction.Request("_all");
24+
GetTransformStatsAction.Response getTransformStatsResponse = client().execute(
25+
GetTransformStatsAction.INSTANCE,
26+
getTransformStatsRequest
27+
).actionGet();
28+
29+
assertEquals(0, getTransformStatsResponse.getTransformsStats().size());
30+
31+
assertWarnings("Transform requires the transform node role for at least 1 node, found no transform nodes");
32+
}
33+
34+
public void testWarningForGet() {
35+
GetTransformAction.Request getTransformRequest = new GetTransformAction.Request("_all");
36+
GetTransformAction.Response getTransformResponse = client().execute(GetTransformAction.INSTANCE, getTransformRequest).actionGet();
37+
assertEquals(0, getTransformResponse.getTransformConfigurations().size());
38+
39+
assertWarnings("Transform requires the transform node role for at least 1 node, found no transform nodes");
40+
}
41+
}

‎x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformNodes.java

+90-3
Original file line numberDiff line numberDiff line change
@@ -8,28 +8,38 @@
88
package org.elasticsearch.xpack.transform.action;
99

1010
import org.elasticsearch.cluster.ClusterState;
11+
import org.elasticsearch.common.Strings;
12+
import org.elasticsearch.common.logging.HeaderWarning;
13+
import org.elasticsearch.common.regex.Regex;
1114
import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
15+
import org.elasticsearch.persistent.PersistentTasksCustomMetadata.Assignment;
16+
import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask;
1217
import org.elasticsearch.xpack.core.transform.TransformField;
18+
import org.elasticsearch.xpack.core.transform.TransformMessages;
19+
import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
20+
import org.elasticsearch.xpack.transform.Transform;
1321

1422
import java.util.Collection;
23+
import java.util.Collections;
1524
import java.util.HashSet;
1625
import java.util.List;
1726
import java.util.Set;
27+
import java.util.function.Predicate;
1828
import java.util.stream.Collectors;
29+
import java.util.stream.StreamSupport;
1930

2031
public final class TransformNodes {
2132

2233
private TransformNodes() {}
2334

2435
/**
25-
* Get the list of nodes transforms are executing on
36+
* Get node assignments for a given list of transforms.
2637
*
2738
* @param transformIds The transforms.
2839
* @param clusterState State
29-
* @return The executor nodes
40+
* @return The {@link TransformNodeAssignments} for the given transforms.
3041
*/
3142
public static TransformNodeAssignments transformTaskNodes(List<String> transformIds, ClusterState clusterState) {
32-
3343
Set<String> executorNodes = new HashSet<>();
3444
Set<String> assigned = new HashSet<>();
3545
Set<String> waitingForAssignment = new HashSet<>();
@@ -60,4 +70,81 @@ public static TransformNodeAssignments transformTaskNodes(List<String> transform
6070

6171
return new TransformNodeAssignments(executorNodes, assigned, waitingForAssignment, stopped);
6272
}
73+
74+
/**
75+
* Get node assignments for a given transform pattern.
76+
*
77+
* Note: This only returns p-task assignments, stopped transforms are not reported. P-Tasks can be running or waiting for a node.
78+
*
79+
* @param transformId The transform or a wildcard pattern, including '_all' to match against transform tasks.
80+
* @param clusterState State
81+
* @return The {@link TransformNodeAssignments} for the given pattern.
82+
*/
83+
public static TransformNodeAssignments findPersistentTasks(String transformId, ClusterState clusterState) {
84+
Set<String> executorNodes = new HashSet<>();
85+
Set<String> assigned = new HashSet<>();
86+
Set<String> waitingForAssignment = new HashSet<>();
87+
88+
PersistentTasksCustomMetadata tasksMetadata = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(clusterState);
89+
90+
if (tasksMetadata != null) {
91+
Predicate<PersistentTask<?>> taskMatcher = Strings.isAllOrWildcard(new String[] { transformId }) ? t -> true : t -> {
92+
TransformTaskParams transformParams = (TransformTaskParams) t.getParams();
93+
return Regex.simpleMatch(transformId, transformParams.getId());
94+
};
95+
96+
for (PersistentTasksCustomMetadata.PersistentTask<?> task : tasksMetadata.findTasks(TransformField.TASK_NAME, taskMatcher)) {
97+
if (task.isAssigned()) {
98+
executorNodes.add(task.getExecutorNode());
99+
assigned.add(task.getId());
100+
} else {
101+
waitingForAssignment.add(task.getId());
102+
}
103+
}
104+
}
105+
return new TransformNodeAssignments(executorNodes, assigned, waitingForAssignment, Collections.emptySet());
106+
}
107+
108+
/**
109+
* Get the assignment of a specific transform.
110+
*
111+
* @param transformId the transform id
112+
* @param clusterState state
113+
* @return {@link Assignment} of task
114+
*/
115+
public static Assignment getAssignment(String transformId, ClusterState clusterState) {
116+
PersistentTasksCustomMetadata tasksMetadata = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata(clusterState);
117+
PersistentTask<?> task = tasksMetadata.getTask(transformId);
118+
119+
if (task != null) {
120+
return task.getAssignment();
121+
}
122+
123+
return PersistentTasksCustomMetadata.INITIAL_ASSIGNMENT;
124+
}
125+
126+
/**
127+
* Get the number of transform nodes in the cluster
128+
*
129+
* @param clusterState state
130+
* @return number of transform nodes
131+
*/
132+
public static long getNumberOfTransformNodes(ClusterState clusterState) {
133+
return StreamSupport.stream(clusterState.getNodes().spliterator(), false)
134+
.filter(node -> node.getRoles().contains(Transform.TRANSFORM_ROLE))
135+
.count();
136+
}
137+
138+
/**
139+
* Check if cluster has at least 1 transform nodes and add a header warning if not.
140+
* To be used by transport actions only.
141+
*
142+
* @param clusterState state
143+
*/
144+
public static void warnIfNoTransformNodes(ClusterState clusterState) {
145+
long transformNodes = getNumberOfTransformNodes(clusterState);
146+
if (transformNodes == 0) {
147+
HeaderWarning.addWarning(TransformMessages.REST_WARN_NO_TRANSFORM_NODES);
148+
}
149+
}
63150
}

‎x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformAction.java

+24-8
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import org.elasticsearch.action.ActionListener;
1212
import org.elasticsearch.action.support.ActionFilters;
1313
import org.elasticsearch.client.Client;
14+
import org.elasticsearch.cluster.ClusterState;
15+
import org.elasticsearch.cluster.service.ClusterService;
1416
import org.elasticsearch.common.ParseField;
1517
import org.elasticsearch.common.inject.Inject;
1618
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -34,23 +36,37 @@
3436
import static org.elasticsearch.xpack.core.transform.TransformField.INDEX_DOC_TYPE;
3537

3638

37-
public class TransportGetTransformAction extends AbstractTransportGetResourcesAction<TransformConfig,
38-
Request,
39-
Response> {
39+
public class TransportGetTransformAction extends AbstractTransportGetResourcesAction<TransformConfig, Request, Response> {
40+
41+
private final ClusterService clusterService;
4042

4143
@Inject
42-
public TransportGetTransformAction(TransportService transportService, ActionFilters actionFilters, Client client,
43-
NamedXContentRegistry xContentRegistry) {
44-
this(GetTransformAction.NAME, transportService, actionFilters, client, xContentRegistry);
44+
public TransportGetTransformAction(
45+
TransportService transportService,
46+
ActionFilters actionFilters,
47+
ClusterService clusterService,
48+
Client client,
49+
NamedXContentRegistry xContentRegistry
50+
) {
51+
this(GetTransformAction.NAME, transportService, actionFilters, clusterService, client, xContentRegistry);
4552
}
4653

47-
protected TransportGetTransformAction(String name, TransportService transportService, ActionFilters actionFilters, Client client,
48-
NamedXContentRegistry xContentRegistry) {
54+
protected TransportGetTransformAction(
55+
String name,
56+
TransportService transportService,
57+
ActionFilters actionFilters,
58+
ClusterService clusterService,
59+
Client client,
60+
NamedXContentRegistry xContentRegistry
61+
) {
4962
super(name, transportService, actionFilters, Request::new, client, xContentRegistry);
63+
this.clusterService = clusterService;
5064
}
5165

5266
@Override
5367
protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
68+
final ClusterState state = clusterService.state();
69+
TransformNodes.warnIfNoTransformNodes(state);
5470
searchResources(request, ActionListener.wrap(
5571
r -> listener.onResponse(new Response(r.results(), r.count())),
5672
listener::onFailure

0 commit comments

Comments
 (0)