Skip to content

Commit 90c4fa8

Browse files
ywelschywelsch
committed
Only allow x-pack metadata if all nodes are ready (#30743)
Enables a rolling restart from the OSS distribution to the x-pack based distribution by preventing x-pack code from installing custom metadata into the cluster state until all nodes are capable of deserializing this metadata.
1 parent 4bf388f commit 90c4fa8

File tree

16 files changed

+270
-25
lines changed

16 files changed

+270
-25
lines changed

‎server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ default void onNodeStarted() {
7070
* Returns a map of {@link ClusterState.Custom} supplier that should be invoked to initialize the initial clusterstate.
7171
* This allows custom clusterstate extensions to be always present and prevents invariants where clusterstates are published
7272
* but customs are not initialized.
73+
*
74+
* TODO: Remove this whole concept of InitialClusterStateCustomSupplier, it's not used anymore
7375
*/
7476
default Map<String, Supplier<ClusterState.Custom>> getInitialClusterStateCustomSupplier() { return Collections.emptyMap(); }
7577
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ protected PutLicenseResponse newResponse(boolean acknowledged) {
223223

224224
@Override
225225
public ClusterState execute(ClusterState currentState) throws Exception {
226+
XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
226227
MetaData currentMetadata = currentState.metaData();
227228
LicensesMetaData licensesMetaData = currentMetadata.custom(LicensesMetaData.TYPE);
228229
Version trialVersion = null;
@@ -341,7 +342,7 @@ protected void doStart() throws ElasticsearchException {
341342
if (clusterService.lifecycleState() == Lifecycle.State.STARTED) {
342343
final ClusterState clusterState = clusterService.state();
343344
if (clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK) == false &&
344-
clusterState.nodes().getMasterNode() != null) {
345+
clusterState.nodes().getMasterNode() != null && XPackPlugin.isReadyForXPackCustomMetadata(clusterState)) {
345346
final LicensesMetaData currentMetaData = clusterState.metaData().custom(LicensesMetaData.TYPE);
346347
boolean noLicense = currentMetaData == null || currentMetaData.getLicense() == null;
347348
if (clusterState.getNodes().isLocalNodeElectedMaster() &&
@@ -374,6 +375,12 @@ public void clusterChanged(ClusterChangedEvent event) {
374375
final ClusterState previousClusterState = event.previousState();
375376
final ClusterState currentClusterState = event.state();
376377
if (!currentClusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
378+
if (XPackPlugin.isReadyForXPackCustomMetadata(currentClusterState) == false) {
379+
logger.debug("cannot add license to cluster as the following nodes might not understand the license metadata: {}",
380+
() -> XPackPlugin.nodesNotReadyForXPackCustomMetadata(currentClusterState));
381+
return;
382+
}
383+
377384
final LicensesMetaData prevLicensesMetaData = previousClusterState.getMetaData().custom(LicensesMetaData.TYPE);
378385
final LicensesMetaData currentLicensesMetaData = currentClusterState.getMetaData().custom(LicensesMetaData.TYPE);
379386
if (logger.isDebugEnabled()) {

‎x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartBasicClusterTask.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.cluster.ClusterStateUpdateTask;
1414
import org.elasticsearch.cluster.metadata.MetaData;
1515
import org.elasticsearch.common.Nullable;
16+
import org.elasticsearch.xpack.core.XPackPlugin;
1617

1718
import java.time.Clock;
1819
import java.util.Collections;
@@ -59,6 +60,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
5960

6061
@Override
6162
public ClusterState execute(ClusterState currentState) throws Exception {
63+
XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
6264
LicensesMetaData licensesMetaData = currentState.metaData().custom(LicensesMetaData.TYPE);
6365
License currentLicense = LicensesMetaData.extractLicense(licensesMetaData);
6466
if (currentLicense == null || currentLicense.type().equals("basic") == false) {

‎x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.cluster.ClusterStateUpdateTask;
1414
import org.elasticsearch.cluster.metadata.MetaData;
1515
import org.elasticsearch.common.Nullable;
16+
import org.elasticsearch.xpack.core.XPackPlugin;
1617

1718
import java.time.Clock;
1819
import java.util.Collections;
@@ -64,6 +65,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
6465

6566
@Override
6667
public ClusterState execute(ClusterState currentState) throws Exception {
68+
XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
6769
LicensesMetaData currentLicensesMetaData = currentState.metaData().custom(LicensesMetaData.TYPE);
6870

6971
if (request.isAcknowledged() == false) {

‎x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartupSelfGeneratedLicenseTask.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.common.Nullable;
1717
import org.elasticsearch.common.logging.Loggers;
1818
import org.elasticsearch.common.settings.Settings;
19+
import org.elasticsearch.xpack.core.XPackPlugin;
1920

2021
import java.time.Clock;
2122
import java.util.UUID;
@@ -49,6 +50,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
4950

5051
@Override
5152
public ClusterState execute(ClusterState currentState) throws Exception {
53+
XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
5254
final MetaData metaData = currentState.metaData();
5355
final LicensesMetaData currentLicensesMetaData = metaData.custom(LicensesMetaData.TYPE);
5456
// do not generate a license if any license is present

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,20 @@
99
import org.apache.lucene.util.SetOnce;
1010
import org.bouncycastle.operator.OperatorCreationException;
1111
import org.elasticsearch.SpecialPermission;
12+
import org.elasticsearch.Version;
1213
import org.elasticsearch.action.ActionRequest;
1314
import org.elasticsearch.action.ActionResponse;
1415
import org.elasticsearch.action.GenericAction;
1516
import org.elasticsearch.action.support.ActionFilter;
1617
import org.elasticsearch.client.Client;
1718
import org.elasticsearch.client.transport.TransportClient;
19+
import org.elasticsearch.cluster.ClusterState;
1820
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
21+
import org.elasticsearch.cluster.metadata.MetaData;
22+
import org.elasticsearch.cluster.node.DiscoveryNode;
1923
import org.elasticsearch.cluster.node.DiscoveryNodes;
2024
import org.elasticsearch.cluster.service.ClusterService;
25+
import org.elasticsearch.common.Booleans;
2126
import org.elasticsearch.common.inject.Binder;
2227
import org.elasticsearch.common.inject.Module;
2328
import org.elasticsearch.common.inject.multibindings.Multibinder;
@@ -33,6 +38,7 @@
3338
import org.elasticsearch.env.Environment;
3439
import org.elasticsearch.env.NodeEnvironment;
3540
import org.elasticsearch.license.LicenseService;
41+
import org.elasticsearch.license.LicensesMetaData;
3642
import org.elasticsearch.license.Licensing;
3743
import org.elasticsearch.license.XPackLicenseState;
3844
import org.elasticsearch.plugins.ExtensiblePlugin;
@@ -46,10 +52,13 @@
4652
import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
4753
import org.elasticsearch.xpack.core.action.XPackInfoAction;
4854
import org.elasticsearch.xpack.core.action.XPackUsageAction;
55+
import org.elasticsearch.xpack.core.ml.MLMetadataField;
4956
import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction;
5057
import org.elasticsearch.xpack.core.rest.action.RestXPackUsageAction;
58+
import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
5159
import org.elasticsearch.xpack.core.ssl.SSLConfigurationReloader;
5260
import org.elasticsearch.xpack.core.ssl.SSLService;
61+
import org.elasticsearch.xpack.core.watcher.WatcherMetaData;
5362

5463
import javax.security.auth.DestroyFailedException;
5564

@@ -62,14 +71,19 @@
6271
import java.time.Clock;
6372
import java.util.ArrayList;
6473
import java.util.Collection;
74+
import java.util.Collections;
6575
import java.util.List;
6676
import java.util.function.Supplier;
77+
import java.util.stream.Collectors;
78+
import java.util.stream.StreamSupport;
6779

6880
public class XPackPlugin extends XPackClientPlugin implements ScriptPlugin, ExtensiblePlugin {
6981

7082
private static Logger logger = ESLoggerFactory.getLogger(XPackPlugin.class);
7183
private static DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
7284

85+
public static final String XPACK_INSTALLED_NODE_ATTR = "xpack.installed";
86+
7387
// TODO: clean up this library to not ask for write access to all system properties!
7488
static {
7589
// invoke this clinit in unbound with permissions to access all system properties
@@ -138,6 +152,75 @@ protected Clock getClock() {
138152
public static LicenseService getSharedLicenseService() { return licenseService.get(); }
139153
public static XPackLicenseState getSharedLicenseState() { return licenseState.get(); }
140154

155+
/**
156+
* Checks if the cluster state allows this node to add x-pack metadata to the cluster state,
157+
* and throws an exception otherwise.
158+
* This check should be called before installing any x-pack metadata to the cluster state,
159+
* to ensure that the other nodes that are part of the cluster will be able to deserialize
160+
* that metadata. Note that if the cluster state already contains x-pack metadata, this
161+
* check assumes that the nodes are already ready to receive additional x-pack metadata.
162+
* Having this check properly in place everywhere allows to install x-pack into a cluster
163+
* using a rolling restart.
164+
*/
165+
public static void checkReadyForXPackCustomMetadata(ClusterState clusterState) {
166+
if (alreadyContainsXPackCustomMetadata(clusterState)) {
167+
return;
168+
}
169+
List<DiscoveryNode> notReadyNodes = nodesNotReadyForXPackCustomMetadata(clusterState);
170+
if (notReadyNodes.isEmpty() == false) {
171+
throw new IllegalStateException("The following nodes are not ready yet for enabling x-pack custom metadata: " + notReadyNodes);
172+
}
173+
}
174+
175+
/**
176+
* Checks if the cluster state allows this node to add x-pack metadata to the cluster state.
177+
* See {@link #checkReadyForXPackCustomMetadata} for more details.
178+
*/
179+
public static boolean isReadyForXPackCustomMetadata(ClusterState clusterState) {
180+
return alreadyContainsXPackCustomMetadata(clusterState) || nodesNotReadyForXPackCustomMetadata(clusterState).isEmpty();
181+
}
182+
183+
/**
184+
* Returns the list of nodes that won't allow this node from adding x-pack metadata to the cluster state.
185+
* See {@link #checkReadyForXPackCustomMetadata} for more details.
186+
*/
187+
public static List<DiscoveryNode> nodesNotReadyForXPackCustomMetadata(ClusterState clusterState) {
188+
// check that all nodes would be capable of deserializing newly added x-pack metadata
189+
final List<DiscoveryNode> notReadyNodes = StreamSupport.stream(clusterState.nodes().spliterator(), false).filter(node -> {
190+
final String xpackInstalledAttr = node.getAttributes().getOrDefault(XPACK_INSTALLED_NODE_ATTR, "false");
191+
192+
// The node attribute XPACK_INSTALLED_NODE_ATTR was only introduced in 6.3.0, so when
193+
// we have an older node in this mixed-version cluster without any x-pack metadata,
194+
// we want to prevent x-pack from adding custom metadata
195+
return node.getVersion().before(Version.V_6_3_0) || Booleans.parseBoolean(xpackInstalledAttr) == false;
196+
}).collect(Collectors.toList());
197+
198+
return notReadyNodes;
199+
}
200+
201+
private static boolean alreadyContainsXPackCustomMetadata(ClusterState clusterState) {
202+
final MetaData metaData = clusterState.metaData();
203+
return metaData.custom(LicensesMetaData.TYPE) != null ||
204+
metaData.custom(MLMetadataField.TYPE) != null ||
205+
metaData.custom(WatcherMetaData.TYPE) != null ||
206+
clusterState.custom(TokenMetaData.TYPE) != null;
207+
}
208+
209+
@Override
210+
public Settings additionalSettings() {
211+
final String xpackInstalledNodeAttrSetting = "node.attr." + XPACK_INSTALLED_NODE_ATTR;
212+
213+
if (settings.get(xpackInstalledNodeAttrSetting) != null) {
214+
throw new IllegalArgumentException("Directly setting [" + xpackInstalledNodeAttrSetting + "] is not permitted");
215+
}
216+
217+
if (transportClientMode) {
218+
return super.additionalSettings();
219+
} else {
220+
return Settings.builder().put(super.additionalSettings()).put(xpackInstalledNodeAttrSetting, "true").build();
221+
}
222+
}
223+
141224
@Override
142225
public Collection<Module> createGuiceModules() {
143226
ArrayList<Module> modules = new ArrayList<>();

‎x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicenseServiceTestCase.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,16 @@
1818
import org.elasticsearch.env.Environment;
1919
import org.elasticsearch.test.ESTestCase;
2020
import org.elasticsearch.watcher.ResourceWatcherService;
21+
import org.elasticsearch.xpack.core.XPackPlugin;
2122
import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
2223
import org.junit.After;
2324
import org.junit.Before;
2425

2526
import java.nio.file.Path;
27+
import java.util.Arrays;
2628

27-
import static java.util.Collections.emptyMap;
2829
import static java.util.Collections.emptySet;
30+
import static java.util.Collections.singletonMap;
2931
import static org.mockito.Mockito.mock;
3032
import static org.mockito.Mockito.when;
3133

@@ -66,6 +68,7 @@ protected void setInitialState(License license, XPackLicenseState licenseState,
6668
when(state.metaData()).thenReturn(metaData);
6769
final DiscoveryNode mockNode = getLocalNode();
6870
when(discoveryNodes.getMasterNode()).thenReturn(mockNode);
71+
when(discoveryNodes.spliterator()).thenReturn(Arrays.asList(mockNode).spliterator());
6972
when(discoveryNodes.isLocalNodeElectedMaster()).thenReturn(false);
7073
when(state.nodes()).thenReturn(discoveryNodes);
7174
when(state.getNodes()).thenReturn(discoveryNodes); // it is really ridiculous we have nodes() and getNodes()...
@@ -76,7 +79,8 @@ protected void setInitialState(License license, XPackLicenseState licenseState,
7679
}
7780

7881
protected DiscoveryNode getLocalNode() {
79-
return new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
82+
return new DiscoveryNode("b", buildNewFakeTransportAddress(), singletonMap(XPackPlugin.XPACK_INSTALLED_NODE_ATTR, "true"),
83+
emptySet(), Version.CURRENT);
8084
}
8185

8286
@After

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core;
7+
8+
import org.elasticsearch.Version;
9+
import org.elasticsearch.client.Client;
10+
import org.elasticsearch.cluster.ClusterName;
11+
import org.elasticsearch.cluster.ClusterState;
12+
import org.elasticsearch.cluster.node.DiscoveryNode;
13+
import org.elasticsearch.cluster.node.DiscoveryNodes;
14+
import org.elasticsearch.common.settings.Settings;
15+
import org.elasticsearch.license.XPackLicenseState;
16+
import org.elasticsearch.test.ESTestCase;
17+
import org.elasticsearch.test.VersionUtils;
18+
import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
19+
import org.elasticsearch.xpack.core.ssl.SSLService;
20+
21+
import java.util.Collections;
22+
import java.util.Map;
23+
24+
import static org.hamcrest.Matchers.containsString;
25+
26+
public class XPackPluginTests extends ESTestCase {
27+
28+
public void testXPackInstalledAttrClash() throws Exception {
29+
Settings.Builder builder = Settings.builder();
30+
builder.put("node.attr." + XPackPlugin.XPACK_INSTALLED_NODE_ATTR, randomBoolean());
31+
if (randomBoolean()) {
32+
builder.put(Client.CLIENT_TYPE_SETTING_S.getKey(), "transport");
33+
}
34+
XPackPlugin xpackPlugin = createXPackPlugin(builder.put("path.home", createTempDir()).build());
35+
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, xpackPlugin::additionalSettings);
36+
assertThat(e.getMessage(),
37+
containsString("Directly setting [node.attr." + XPackPlugin.XPACK_INSTALLED_NODE_ATTR + "] is not permitted"));
38+
}
39+
40+
public void testXPackInstalledAttrExists() throws Exception {
41+
XPackPlugin xpackPlugin = createXPackPlugin(Settings.builder().put("path.home", createTempDir()).build());
42+
assertEquals("true", xpackPlugin.additionalSettings().get("node.attr." + XPackPlugin.XPACK_INSTALLED_NODE_ATTR));
43+
}
44+
45+
public void testNodesNotReadyForXPackCustomMetadata() {
46+
boolean compatible;
47+
boolean nodesCompatible = true;
48+
DiscoveryNodes.Builder discoveryNodes = DiscoveryNodes.builder();
49+
50+
for (int i = 0; i < randomInt(3); i++) {
51+
final Version version = VersionUtils.randomVersion(random());
52+
final Map<String, String> attributes;
53+
if (randomBoolean() && version.onOrAfter(Version.V_6_3_0)) {
54+
attributes = Collections.singletonMap(XPackPlugin.XPACK_INSTALLED_NODE_ATTR, "true");
55+
} else {
56+
nodesCompatible = false;
57+
attributes = Collections.emptyMap();
58+
}
59+
60+
discoveryNodes.add(new DiscoveryNode("node_" + i, buildNewFakeTransportAddress(), attributes, Collections.emptySet(),
61+
Version.CURRENT));
62+
}
63+
ClusterState.Builder clusterStateBuilder = ClusterState.builder(ClusterName.DEFAULT);
64+
65+
if (randomBoolean()) {
66+
clusterStateBuilder.putCustom(TokenMetaData.TYPE, new TokenMetaData(Collections.emptyList(), new byte[0]));
67+
compatible = true;
68+
} else {
69+
compatible = nodesCompatible;
70+
}
71+
72+
ClusterState clusterState = clusterStateBuilder.nodes(discoveryNodes.build()).build();
73+
74+
assertEquals(XPackPlugin.nodesNotReadyForXPackCustomMetadata(clusterState).isEmpty(), nodesCompatible);
75+
assertEquals(XPackPlugin.isReadyForXPackCustomMetadata(clusterState), compatible);
76+
77+
if (compatible == false) {
78+
IllegalStateException e = expectThrows(IllegalStateException.class,
79+
() -> XPackPlugin.checkReadyForXPackCustomMetadata(clusterState));
80+
assertThat(e.getMessage(), containsString("The following nodes are not ready yet for enabling x-pack custom metadata:"));
81+
}
82+
}
83+
84+
private XPackPlugin createXPackPlugin(Settings settings) throws Exception {
85+
return new XPackPlugin(settings, null){
86+
87+
@Override
88+
protected void setSslService(SSLService sslService) {
89+
// disable
90+
}
91+
92+
@Override
93+
protected void setLicenseState(XPackLicenseState licenseState) {
94+
// disable
95+
}
96+
};
97+
}
98+
99+
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.common.settings.Settings;
2222
import org.elasticsearch.threadpool.ThreadPool;
2323
import org.elasticsearch.transport.TransportService;
24+
import org.elasticsearch.xpack.core.XPackPlugin;
2425
import org.elasticsearch.xpack.core.ml.MLMetadataField;
2526
import org.elasticsearch.xpack.core.ml.MlMetadata;
2627
import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction;
@@ -120,6 +121,7 @@ protected DeleteDatafeedAction.Response newResponse(boolean acknowledged) {
120121

121122
@Override
122123
public ClusterState execute(ClusterState currentState) {
124+
XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
123125
MlMetadata currentMetadata = MlMetadata.getMlMetadata(currentState);
124126
PersistentTasksCustomMetaData persistentTasks =
125127
currentState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.common.settings.Settings;
2020
import org.elasticsearch.threadpool.ThreadPool;
2121
import org.elasticsearch.transport.TransportService;
22+
import org.elasticsearch.xpack.core.XPackPlugin;
2223
import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
2324
import org.elasticsearch.xpack.core.ml.MLMetadataField;
2425
import org.elasticsearch.xpack.core.ml.MlMetadata;
@@ -57,6 +58,7 @@ protected void masterOperation(FinalizeJobExecutionAction.Request request, Clust
5758
clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() {
5859
@Override
5960
public ClusterState execute(ClusterState currentState) {
61+
XPackPlugin.checkReadyForXPackCustomMetadata(currentState);
6062
MlMetadata mlMetadata = MlMetadata.getMlMetadata(currentState);
6163
MlMetadata.Builder mlMetadataBuilder = new MlMetadata.Builder(mlMetadata);
6264
Date finishedTime = new Date();

0 commit comments

Comments
 (0)