Skip to content

Commit d94b81e

Browse files
ywelschywelsch
authored
Remove custom metadata tool (#50813)
Adds a command-line tool to remove broken custom metadata from the cluster state. Relates to #48701
1 parent 6b20a2c commit d94b81e

File tree

4 files changed

+280
-3
lines changed

4 files changed

+280
-3
lines changed

‎docs/reference/commands/node-tool.asciidoc

+52-3
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,20 @@ bin/elasticsearch-node repurpose|unsafe-bootstrap|detach-cluster|override-versio
2020
[float]
2121
=== Description
2222

23-
This tool has five modes:
23+
This tool has a number of modes:
2424

2525
* `elasticsearch-node repurpose` can be used to delete unwanted data from a
2626
node if it used to be a <<data-node,data node>> or a
2727
<<master-node,master-eligible node>> but has been repurposed not to have one
2828
or other of these roles.
2929

3030
* `elasticsearch-node remove-settings` can be used to remove persistent settings
31-
from the cluster state in case where it contains incompatible settings that
32-
prevent the cluster from forming.
31+
from the cluster state in case where it contains incompatible settings that
32+
prevent the cluster from forming.
33+
34+
* `elasticsearch-node remove-customs` can be used to remove custom metadata
35+
from the cluster state in case where it contains broken metadata that
36+
prevents the cluster state from being loaded.
3337

3438
* `elasticsearch-node unsafe-bootstrap` can be used to perform _unsafe cluster
3539
bootstrapping_. It forces one of the nodes to form a brand-new cluster on
@@ -100,6 +104,24 @@ The intended use is:
100104
* Repeat for all other master-eligible nodes
101105
* Start the nodes
102106

107+
[float]
108+
==== Removing custom metadata from the cluster state
109+
110+
There may be situations where a node contains custom metadata, typically
111+
provided by plugins, that prevent the node from starting up and loading
112+
the cluster from disk.
113+
114+
The `elasticsearch-node remove-customs` tool allows you to forcefully remove
115+
the problematic custom metadata. The tool takes a list of custom metadata names
116+
as parameters that should be removed, and also supports wildcard patterns.
117+
118+
The intended use is:
119+
120+
* Stop the node
121+
* Run `elasticsearch-node remove-customs name-of-custom-to-remove` on the node
122+
* Repeat for all other master-eligible nodes
123+
* Start the nodes
124+
103125
[float]
104126
==== Recovering data after a disaster
105127

@@ -407,6 +429,33 @@ You can also use wildcards to remove multiple settings, for example using
407429
node$ ./bin/elasticsearch-node remove-settings xpack.monitoring.*
408430
----
409431

432+
[float]
433+
==== Removing custom metadata from the cluster state
434+
435+
If the on-disk cluster state contains custom metadata that prevents the node
436+
from starting up and loading the cluster state, you can run the following
437+
commands to remove this custom metadata.
438+
439+
[source,txt]
440+
----
441+
node$ ./bin/elasticsearch-node remove-customs snapshot_lifecycle
442+
443+
WARNING: Elasticsearch MUST be stopped before running this tool.
444+
445+
The following customs will be removed:
446+
snapshot_lifecycle
447+
448+
You should only run this tool if you have broken custom metadata in the
449+
cluster state that prevents the cluster state from being loaded.
450+
This tool can cause data loss and its use should be your last resort.
451+
452+
Do you want to proceed?
453+
454+
Confirm [y/N] y
455+
456+
Customs were successfully removed from the cluster state
457+
----
458+
410459
[float]
411460
==== Unsafe cluster bootstrapping
412461

‎server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public NodeToolCli() {
4242
subcommands.put("detach-cluster", new DetachClusterCommand());
4343
subcommands.put("override-version", new OverrideNodeVersionCommand());
4444
subcommands.put("remove-settings", new RemoveSettingsCommand());
45+
subcommands.put("remove-customs", new RemoveCustomsCommand());
4546
}
4647

4748
public static void main(String[] args) throws Exception {

‎server/src/main/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommand.java

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.cluster.coordination;
20+
21+
import com.carrotsearch.hppc.cursors.ObjectCursor;
22+
import joptsimple.OptionSet;
23+
import joptsimple.OptionSpec;
24+
import org.elasticsearch.cli.ExitCodes;
25+
import org.elasticsearch.cli.Terminal;
26+
import org.elasticsearch.cli.UserException;
27+
import org.elasticsearch.cluster.ClusterState;
28+
import org.elasticsearch.cluster.metadata.MetaData;
29+
import org.elasticsearch.common.collect.Tuple;
30+
import org.elasticsearch.common.regex.Regex;
31+
import org.elasticsearch.env.Environment;
32+
import org.elasticsearch.gateway.PersistedClusterStateService;
33+
34+
import java.io.IOException;
35+
import java.nio.file.Path;
36+
import java.util.List;
37+
38+
public class RemoveCustomsCommand extends ElasticsearchNodeCommand {
39+
40+
static final String CUSTOMS_REMOVED_MSG = "Customs were successfully removed from the cluster state";
41+
static final String CONFIRMATION_MSG =
42+
DELIMITER +
43+
"\n" +
44+
"You should only run this tool if you have broken custom metadata in the\n" +
45+
"cluster state that prevents the cluster state from being loaded.\n" +
46+
"This tool can cause data loss and its use should be your last resort.\n" +
47+
"\n" +
48+
"Do you want to proceed?\n";
49+
50+
private final OptionSpec<String> arguments;
51+
52+
public RemoveCustomsCommand() {
53+
super("Removes custom metadata from the cluster state");
54+
arguments = parser.nonOptions("custom metadata names");
55+
}
56+
57+
@Override
58+
protected void processNodePaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env)
59+
throws IOException, UserException {
60+
final List<String> customsToRemove = arguments.values(options);
61+
if (customsToRemove.isEmpty()) {
62+
throw new UserException(ExitCodes.USAGE, "Must supply at least one custom metadata name to remove");
63+
}
64+
65+
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(dataPaths);
66+
67+
terminal.println(Terminal.Verbosity.VERBOSE, "Loading cluster state");
68+
final Tuple<Long, ClusterState> termAndClusterState = loadTermAndClusterState(persistedClusterStateService, env);
69+
final ClusterState oldClusterState = termAndClusterState.v2();
70+
terminal.println(Terminal.Verbosity.VERBOSE, "custom metadata names: " + oldClusterState.metaData().customs().keys());
71+
final MetaData.Builder metaDataBuilder = MetaData.builder(oldClusterState.metaData());
72+
for (String customToRemove : customsToRemove) {
73+
boolean matched = false;
74+
for (ObjectCursor<String> customKeyCur : oldClusterState.metaData().customs().keys()) {
75+
final String customKey = customKeyCur.value;
76+
if (Regex.simpleMatch(customToRemove, customKey)) {
77+
metaDataBuilder.removeCustom(customKey);
78+
if (matched == false) {
79+
terminal.println("The following customs will be removed:");
80+
}
81+
matched = true;
82+
terminal.println(customKey);
83+
}
84+
}
85+
if (matched == false) {
86+
throw new UserException(ExitCodes.USAGE,
87+
"No custom metadata matching [" + customToRemove + "] were found on this node");
88+
}
89+
}
90+
final ClusterState newClusterState = ClusterState.builder(oldClusterState).metaData(metaDataBuilder.build()).build();
91+
terminal.println(Terminal.Verbosity.VERBOSE,
92+
"[old cluster state = " + oldClusterState + ", new cluster state = " + newClusterState + "]");
93+
94+
confirm(terminal, CONFIRMATION_MSG);
95+
96+
try (PersistedClusterStateService.Writer writer = persistedClusterStateService.createWriter()) {
97+
writer.writeFullStateAndCommit(termAndClusterState.v1(), newClusterState);
98+
}
99+
100+
terminal.println(CUSTOMS_REMOVED_MSG);
101+
}
102+
}

‎server/src/test/java/org/elasticsearch/cluster/coordination/RemoveCustomsCommandIT.java

+125
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.cluster.coordination;
20+
21+
import joptsimple.OptionSet;
22+
import org.elasticsearch.ElasticsearchException;
23+
import org.elasticsearch.cli.MockTerminal;
24+
import org.elasticsearch.cli.UserException;
25+
import org.elasticsearch.common.settings.Settings;
26+
import org.elasticsearch.env.Environment;
27+
import org.elasticsearch.env.TestEnvironment;
28+
import org.elasticsearch.test.ESIntegTestCase;
29+
30+
import static org.hamcrest.Matchers.containsString;
31+
32+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
33+
public class RemoveCustomsCommandIT extends ESIntegTestCase {
34+
35+
public void testRemoveCustomsAbortedByUser() throws Exception {
36+
internalCluster().setBootstrapMasterNodeIndex(0);
37+
String node = internalCluster().startNode();
38+
Settings dataPathSettings = internalCluster().dataPathSettings(node);
39+
ensureStableCluster(1);
40+
internalCluster().stopRandomDataNode();
41+
42+
Environment environment = TestEnvironment.newEnvironment(
43+
Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build());
44+
expectThrows(() -> removeCustoms(environment, true, new String[]{ "index-graveyard" }),
45+
ElasticsearchNodeCommand.ABORTED_BY_USER_MSG);
46+
}
47+
48+
public void testRemoveCustomsSuccessful() throws Exception {
49+
internalCluster().setBootstrapMasterNodeIndex(0);
50+
String node = internalCluster().startNode();
51+
createIndex("test");
52+
client().admin().indices().prepareDelete("test").get();
53+
assertEquals(1, client().admin().cluster().prepareState().get().getState().metaData().indexGraveyard().getTombstones().size());
54+
Settings dataPathSettings = internalCluster().dataPathSettings(node);
55+
ensureStableCluster(1);
56+
internalCluster().stopRandomDataNode();
57+
58+
Environment environment = TestEnvironment.newEnvironment(
59+
Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build());
60+
MockTerminal terminal = removeCustoms(environment, false,
61+
randomBoolean() ?
62+
new String[]{ "index-graveyard" } :
63+
new String[]{ "index-*" }
64+
);
65+
assertThat(terminal.getOutput(), containsString(RemoveCustomsCommand.CUSTOMS_REMOVED_MSG));
66+
assertThat(terminal.getOutput(), containsString("The following customs will be removed:"));
67+
assertThat(terminal.getOutput(), containsString("index-graveyard"));
68+
69+
internalCluster().startNode(dataPathSettings);
70+
assertEquals(0, client().admin().cluster().prepareState().get().getState().metaData().indexGraveyard().getTombstones().size());
71+
}
72+
73+
public void testCustomDoesNotMatch() throws Exception {
74+
internalCluster().setBootstrapMasterNodeIndex(0);
75+
String node = internalCluster().startNode();
76+
createIndex("test");
77+
client().admin().indices().prepareDelete("test").get();
78+
assertEquals(1, client().admin().cluster().prepareState().get().getState().metaData().indexGraveyard().getTombstones().size());
79+
Settings dataPathSettings = internalCluster().dataPathSettings(node);
80+
ensureStableCluster(1);
81+
internalCluster().stopRandomDataNode();
82+
83+
Environment environment = TestEnvironment.newEnvironment(
84+
Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build());
85+
UserException ex = expectThrows(UserException.class, () -> removeCustoms(environment, false,
86+
new String[]{ "index-greveyard-with-typos" }));
87+
assertThat(ex.getMessage(), containsString("No custom metadata matching [index-greveyard-with-typos] were " +
88+
"found on this node"));
89+
}
90+
91+
private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, boolean abort, String... args)
92+
throws Exception {
93+
final MockTerminal terminal = new MockTerminal();
94+
final OptionSet options = command.getParser().parse(args);
95+
final String input;
96+
97+
if (abort) {
98+
input = randomValueOtherThanMany(c -> c.equalsIgnoreCase("y"), () -> randomAlphaOfLength(1));
99+
} else {
100+
input = randomBoolean() ? "y" : "Y";
101+
}
102+
103+
terminal.addTextInput(input);
104+
105+
try {
106+
command.execute(terminal, options, environment);
107+
} finally {
108+
assertThat(terminal.getOutput(), containsString(ElasticsearchNodeCommand.STOP_WARNING_MSG));
109+
}
110+
111+
return terminal;
112+
}
113+
114+
private MockTerminal removeCustoms(Environment environment, boolean abort, String... args) throws Exception {
115+
final MockTerminal terminal = executeCommand(new RemoveCustomsCommand(), environment, abort, args);
116+
assertThat(terminal.getOutput(), containsString(RemoveCustomsCommand.CONFIRMATION_MSG));
117+
assertThat(terminal.getOutput(), containsString(RemoveCustomsCommand.CUSTOMS_REMOVED_MSG));
118+
return terminal;
119+
}
120+
121+
private void expectThrows(ThrowingRunnable runnable, String message) {
122+
ElasticsearchException ex = expectThrows(ElasticsearchException.class, runnable);
123+
assertThat(ex.getMessage(), containsString(message));
124+
}
125+
}

0 commit comments

Comments
 (0)