Skip to content

Commit 1bba782

Browse files
ywelschywelsch
committed
Make elasticsearch-node tools custom metadata-aware (#48390)
The elasticsearch-node tools allow manipulating the on-disk cluster state. The tool is currently unaware of plugins and will therefore drop custom metadata from the cluster state once the state is written out again (as it skips over the custom metadata that it can't read). This commit preserves unknown customs when editing on-disk metadata through the elasticsearch-node command-line tools.
1 parent dab1b8e commit 1bba782

File tree

14 files changed

+156
-57
lines changed

14 files changed

+156
-57
lines changed

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -418,26 +418,26 @@ public synchronized void start() {
418418

419419
if (plugins.isEmpty() == false) {
420420
logToProcessStdout("Installing " + plugins.size() + " plugins");
421-
plugins.forEach(plugin -> runElaticsearchBinScript(
421+
plugins.forEach(plugin -> runElasticsearchBinScript(
422422
"elasticsearch-plugin",
423423
"install", "--batch", plugin.toString())
424424
);
425425
}
426426

427427
if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) {
428428
LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion());
429-
runElaticsearchBinScript(
429+
runElasticsearchBinScript(
430430
"elasticsearch-plugin",
431431
"install", "--batch", "x-pack"
432432
);
433433
}
434434

435435
if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
436436
logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
437-
runElaticsearchBinScript("elasticsearch-keystore", "create");
437+
runElasticsearchBinScript("elasticsearch-keystore", "create");
438438

439439
keystoreSettings.forEach((key, value) ->
440-
runElaticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
440+
runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
441441
);
442442

443443
for (Map.Entry<String, File> entry : keystoreFiles.entrySet()) {
@@ -446,7 +446,7 @@ public synchronized void start() {
446446
if (file.exists() == false) {
447447
throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
448448
}
449-
runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
449+
runElasticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
450450
}
451451
}
452452

@@ -463,7 +463,7 @@ public synchronized void start() {
463463
if (credentials.isEmpty() == false) {
464464
logToProcessStdout("Setting up " + credentials.size() + " users");
465465

466-
credentials.forEach(paramMap -> runElaticsearchBinScript(
466+
credentials.forEach(paramMap -> runElasticsearchBinScript(
467467
getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
468468
paramMap.entrySet().stream()
469469
.flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
@@ -592,7 +592,7 @@ public void user(Map<String, String> userSpec) {
592592
credentials.add(cred);
593593
}
594594

595-
private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) {
595+
private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
596596
if (
597597
Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false &&
598598
Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false
@@ -632,8 +632,8 @@ private void runElaticsearchBinScriptWithInput(String input, String tool, String
632632
}
633633
}
634634

635-
private void runElaticsearchBinScript(String tool, String... args) {
636-
runElaticsearchBinScriptWithInput("", tool, args);
635+
private void runElasticsearchBinScript(String tool, String... args) {
636+
runElasticsearchBinScriptWithInput("", tool, args);
637637
}
638638

639639
private Map<String, String> getESEnvironment() {

‎libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -819,7 +819,7 @@ private void unknownValue(Object value, boolean ensureNoSelfReferences) throws I
819819
} else if (value instanceof Map) {
820820
@SuppressWarnings("unchecked")
821821
final Map<String, ?> valueMap = (Map<String, ?>) value;
822-
map(valueMap, ensureNoSelfReferences);
822+
map(valueMap, ensureNoSelfReferences, true);
823823
} else if (value instanceof Iterable) {
824824
value((Iterable<?>) value, ensureNoSelfReferences);
825825
} else if (value instanceof Object[]) {
@@ -867,10 +867,15 @@ public XContentBuilder field(String name, Map<String, Object> values) throws IOE
867867
}
868868

869869
public XContentBuilder map(Map<String, ?> values) throws IOException {
870-
return map(values, true);
870+
return map(values, true, true);
871871
}
872872

873-
private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences) throws IOException {
873+
/** writes a map without the start object and end object headers */
874+
public XContentBuilder mapContents(Map<String, ?> values) throws IOException {
875+
return map(values, true, false);
876+
}
877+
878+
private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences, boolean writeStartAndEndHeaders) throws IOException {
874879
if (values == null) {
875880
return nullValue();
876881
}
@@ -881,13 +886,17 @@ private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReference
881886
ensureNoSelfReferences(values);
882887
}
883888

884-
startObject();
889+
if (writeStartAndEndHeaders) {
890+
startObject();
891+
}
885892
for (Map.Entry<String, ?> value : values.entrySet()) {
886893
field(value.getKey());
887894
// pass ensureNoSelfReferences=false as we already performed the check at a higher level
888895
unknownValue(value.getValue(), false);
889896
}
890-
endObject();
897+
if (writeStartAndEndHeaders) {
898+
endObject();
899+
}
891900
return this;
892901
}
893902

‎server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,19 @@ protected void execute(Terminal terminal, OptionSet options) throws Exception {
8888

8989
/** Create an {@link Environment} for the command to use. Overrideable for tests. */
9090
protected Environment createEnv(final Map<String, String> settings) throws UserException {
91+
return createEnv(Settings.EMPTY, settings);
92+
}
93+
94+
/** Create an {@link Environment} for the command to use. Overrideable for tests. */
95+
protected final Environment createEnv(final Settings baseSettings, final Map<String, String> settings) throws UserException {
9196
final String esPathConf = System.getProperty("es.path.conf");
9297
if (esPathConf == null) {
9398
throw new UserException(ExitCodes.CONFIG, "the system property [es.path.conf] must be set");
9499
}
95-
return InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, settings,
96-
getConfigPath(esPathConf),
97-
// HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
98-
() -> System.getenv("HOSTNAME"));
100+
return InternalSettingsPreparer.prepareEnvironment(baseSettings, settings,
101+
getConfigPath(esPathConf),
102+
// HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
103+
() -> System.getenv("HOSTNAME"));
99104
}
100105

101106
@SuppressForbidden(reason = "need path to construct environment")

‎server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.elasticsearch.ElasticsearchException;
2828
import org.elasticsearch.cli.EnvironmentAwareCommand;
2929
import org.elasticsearch.cli.Terminal;
30-
import org.elasticsearch.cluster.ClusterModule;
3130
import org.elasticsearch.cluster.metadata.Manifest;
3231
import org.elasticsearch.cluster.metadata.MetaData;
3332
import org.elasticsearch.common.collect.Tuple;
@@ -43,7 +42,6 @@
4342

4443
public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
4544
private static final Logger logger = LogManager.getLogger(ElasticsearchNodeCommand.class);
46-
protected final NamedXContentRegistry namedXContentRegistry;
4745
protected static final String DELIMITER = "------------------------------------------------------------------------\n";
4846

4947
static final String STOP_WARNING_MSG =
@@ -65,7 +63,6 @@ public ElasticsearchNodeCommand(String description) {
6563
super(description);
6664
nodeOrdinalOption = parser.accepts("ordinal", "Optional node ordinal, 0 if not specified")
6765
.withRequiredArg().ofType(Integer.class);
68-
namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
6966
}
7067

7168
protected void processNodePathsWithLock(Terminal terminal, OptionSet options, Environment env) throws IOException {
@@ -88,7 +85,7 @@ protected void processNodePathsWithLock(Terminal terminal, OptionSet options, En
8885

8986
protected Tuple<Manifest, MetaData> loadMetaData(Terminal terminal, Path[] dataPaths) throws IOException {
9087
terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file");
91-
final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
88+
final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
9289

9390
if (manifest == null) {
9491
throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG);
@@ -97,8 +94,8 @@ protected Tuple<Manifest, MetaData> loadMetaData(Terminal terminal, Path[] dataP
9794
throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG);
9895
}
9996
terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file");
100-
final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(),
101-
dataPaths);
97+
final MetaData metaData = MetaData.FORMAT_PRESERVE_CUSTOMS.loadGeneration(
98+
logger, NamedXContentRegistry.EMPTY, manifest.getGlobalGeneration(), dataPaths);
10299
if (metaData == null) {
103100
throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]");
104101
}

‎server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.common.collect.Tuple;
2929
import org.elasticsearch.common.settings.Setting;
3030
import org.elasticsearch.common.settings.Settings;
31+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3132
import org.elasticsearch.env.Environment;
3233
import org.elasticsearch.env.NodeMetaData;
3334
import org.elasticsearch.node.Node;
@@ -84,7 +85,7 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) {
8485

8586
protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
8687
terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata");
87-
final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
88+
final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
8889
if (nodeMetaData == null) {
8990
throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG);
9091
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
import org.elasticsearch.common.settings.Setting;
4646
import org.elasticsearch.common.settings.Setting.Property;
4747
import org.elasticsearch.common.settings.Settings;
48-
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
4948
import org.elasticsearch.common.xcontent.ToXContent;
5049
import org.elasticsearch.common.xcontent.ToXContentFragment;
5150
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -1513,8 +1512,6 @@ public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOEx
15131512

15141513
@Override
15151514
public IndexMetaData fromXContent(XContentParser parser) throws IOException {
1516-
assert parser.getXContentRegistry() != NamedXContentRegistry.EMPTY
1517-
: "loading index metadata requires a working named xcontent registry";
15181515
return Builder.fromXContent(parser);
15191516
}
15201517
};

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java

Lines changed: 67 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -813,7 +813,7 @@ public static Diff<MetaData> readDiffFrom(StreamInput in) throws IOException {
813813
}
814814

815815
public static MetaData fromXContent(XContentParser parser) throws IOException {
816-
return Builder.fromXContent(parser);
816+
return Builder.fromXContent(parser, false);
817817
}
818818

819819
@Override
@@ -1353,7 +1353,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
13531353
builder.endObject();
13541354
}
13551355

1356-
public static MetaData fromXContent(XContentParser parser) throws IOException {
1356+
public static MetaData fromXContent(XContentParser parser, boolean preserveUnknownCustoms) throws IOException {
13571357
Builder builder = new Builder();
13581358

13591359
// we might get here after the meta-data element, or on a fresh parser
@@ -1403,8 +1403,13 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
14031403
Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
14041404
builder.putCustom(custom.getWriteableName(), custom);
14051405
} catch (NamedObjectNotFoundException ex) {
1406-
logger.warn("Skipping unknown custom object with type {}", currentFieldName);
1407-
parser.skipChildren();
1406+
if (preserveUnknownCustoms) {
1407+
logger.warn("Adding unknown custom object with type {}", currentFieldName);
1408+
builder.putCustom(currentFieldName, new UnknownGatewayOnlyCustom(parser.mapOrdered()));
1409+
} else {
1410+
logger.warn("Skipping unknown custom object with type {}", currentFieldName);
1411+
parser.skipChildren();
1412+
}
14081413
}
14091414
}
14101415
} else if (token.isValue()) {
@@ -1425,6 +1430,45 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
14251430
}
14261431
}
14271432

1433+
public static class UnknownGatewayOnlyCustom implements Custom {
1434+
1435+
private final Map<String, Object> contents;
1436+
1437+
UnknownGatewayOnlyCustom(Map<String, Object> contents) {
1438+
this.contents = contents;
1439+
}
1440+
1441+
@Override
1442+
public EnumSet<XContentContext> context() {
1443+
return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.GATEWAY);
1444+
}
1445+
1446+
@Override
1447+
public Diff<Custom> diff(Custom previousState) {
1448+
throw new UnsupportedOperationException();
1449+
}
1450+
1451+
@Override
1452+
public String getWriteableName() {
1453+
throw new UnsupportedOperationException();
1454+
}
1455+
1456+
@Override
1457+
public Version getMinimalSupportedVersion() {
1458+
throw new UnsupportedOperationException();
1459+
}
1460+
1461+
@Override
1462+
public void writeTo(StreamOutput out) throws IOException {
1463+
throw new UnsupportedOperationException();
1464+
}
1465+
1466+
@Override
1467+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
1468+
return builder.mapContents(contents);
1469+
}
1470+
}
1471+
14281472
private static final ToXContent.Params FORMAT_PARAMS;
14291473
static {
14301474
Map<String, String> params = new HashMap<>(2);
@@ -1436,16 +1480,25 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
14361480
/**
14371481
* State format for {@link MetaData} to write to and load from disk
14381482
*/
1439-
public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
1483+
public static final MetaDataStateFormat<MetaData> FORMAT = createMetaDataStateFormat(false);
14401484

1441-
@Override
1442-
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
1443-
Builder.toXContent(state, builder, FORMAT_PARAMS);
1444-
}
1485+
/**
1486+
* Special state format for {@link MetaData} to write to and load from disk, preserving unknown customs
1487+
*/
1488+
public static final MetaDataStateFormat<MetaData> FORMAT_PRESERVE_CUSTOMS = createMetaDataStateFormat(true);
14451489

1446-
@Override
1447-
public MetaData fromXContent(XContentParser parser) throws IOException {
1448-
return Builder.fromXContent(parser);
1449-
}
1450-
};
1490+
private static MetaDataStateFormat<MetaData> createMetaDataStateFormat(boolean preserveUnknownCustoms) {
1491+
return new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
1492+
1493+
@Override
1494+
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
1495+
Builder.toXContent(state, builder, FORMAT_PARAMS);
1496+
}
1497+
1498+
@Override
1499+
public MetaData fromXContent(XContentParser parser) throws IOException {
1500+
return Builder.fromXContent(parser, preserveUnknownCustoms);
1501+
}
1502+
};
1503+
}
14511504
}

‎server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.cluster.metadata.Manifest;
3030
import org.elasticsearch.cluster.node.DiscoveryNode;
3131
import org.elasticsearch.common.settings.Settings;
32+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3233
import org.elasticsearch.core.internal.io.IOUtils;
3334
import org.elasticsearch.gateway.WriteStateException;
3435

@@ -165,7 +166,7 @@ private String toIndexName(NodeEnvironment.NodePath[] nodePaths, String uuid) {
165166
indexPaths[i] = nodePaths[i].resolve(uuid);
166167
}
167168
try {
168-
IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, indexPaths);
169+
IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, indexPaths);
169170
return metaData.getIndex().getName();
170171
} catch (Exception e) {
171172
return "no name for uuid: " + uuid + ": " + e;
@@ -194,7 +195,7 @@ private void rewriteManifest(Terminal terminal, Manifest manifest, Path[] dataPa
194195

195196
private Manifest loadManifest(Terminal terminal, Path[] dataPaths) throws IOException {
196197
terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest");
197-
final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
198+
final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
198199

199200
if (manifest == null) {
200201
terminal.println(Terminal.Verbosity.SILENT, PRE_V7_MESSAGE);

‎server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.elasticsearch.Version;
2626
import org.elasticsearch.cli.Terminal;
2727
import org.elasticsearch.cluster.coordination.ElasticsearchNodeCommand;
28+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
2829

2930
import java.io.IOException;
3031
import java.nio.file.Path;
@@ -74,7 +75,7 @@ public OverrideNodeVersionCommand() {
7475
protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
7576
final Path[] nodePaths = Arrays.stream(toNodePaths(dataPaths)).map(p -> p.path).toArray(Path[]::new);
7677
final NodeMetaData nodeMetaData
77-
= new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, namedXContentRegistry, nodePaths);
78+
= new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, NamedXContentRegistry.EMPTY, nodePaths);
7879
if (nodeMetaData == null) {
7980
throw new ElasticsearchException(NO_METADATA_MESSAGE);
8081
}

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ public void testUnknownFieldClusterMetaData() throws IOException {
365365
.endObject()
366366
.endObject());
367367
try (XContentParser parser = createParser(JsonXContent.jsonXContent, metadata)) {
368-
MetaData.Builder.fromXContent(parser);
368+
MetaData.Builder.fromXContent(parser, randomBoolean());
369369
fail();
370370
} catch (IllegalArgumentException e) {
371371
assertEquals("Unexpected field [random]", e.getMessage());

‎server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
147147

148148
String metaDataSource = MetaData.Builder.toXContent(metaData);
149149

150-
MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource));
150+
MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource), false);
151151

152152
IndexMetaData indexMetaData = parsedMetaData.index("test1");
153153
assertThat(indexMetaData.primaryTerm(0), equalTo(1L));

0 commit comments

Comments
 (0)