diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index de8e02ed0c7de..2f258733e7575 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -424,7 +424,7 @@ public synchronized void start() { if (plugins.isEmpty() == false) { logToProcessStdout("Installing " + plugins.size() + " plugins"); - plugins.forEach(plugin -> runElaticsearchBinScript( + plugins.forEach(plugin -> runElasticsearchBinScript( "elasticsearch-plugin", "install", "--batch", plugin.toString()) ); @@ -432,7 +432,7 @@ public synchronized void start() { if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) { LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion()); - runElaticsearchBinScript( + runElasticsearchBinScript( "elasticsearch-plugin", "install", "--batch", "x-pack" ); @@ -440,7 +440,7 @@ public synchronized void start() { if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) { logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files"); - runElaticsearchBinScript("elasticsearch-keystore", "create"); + runElasticsearchBinScript("elasticsearch-keystore", "create"); keystoreSettings.forEach((key, value) -> runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key) @@ -452,7 +452,7 @@ public synchronized void start() { if (file.exists() == false) { throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this); } - runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath()); + runElasticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath()); } } @@ -467,7 +467,7 @@ public synchronized void start() { if (credentials.isEmpty() == false) { logToProcessStdout("Setting up " + credentials.size() + " users"); - credentials.forEach(paramMap -> runElaticsearchBinScript( + credentials.forEach(paramMap -> runElasticsearchBinScript( getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users", paramMap.entrySet().stream() .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue())) @@ -663,7 +663,7 @@ private void runElasticsearchBinScriptWithInput(String input, String tool, Strin } } - private void runElaticsearchBinScript(String tool, String... args) { + private void runElasticsearchBinScript(String tool, String... args) { runElasticsearchBinScriptWithInput("", tool, args); } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 51a4f86a0d3b2..20fde0891b6f8 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -819,7 +819,7 @@ private void unknownValue(Object value, boolean ensureNoSelfReferences) throws I } else if (value instanceof Map) { @SuppressWarnings("unchecked") final Map valueMap = (Map) value; - map(valueMap, ensureNoSelfReferences); + map(valueMap, ensureNoSelfReferences, true); } else if (value instanceof Iterable) { value((Iterable) value, ensureNoSelfReferences); } else if (value instanceof Object[]) { @@ -867,10 +867,15 @@ public XContentBuilder field(String name, Map values) throws IOE } public XContentBuilder map(Map values) throws IOException { - return map(values, true); + return map(values, true, true); } - private XContentBuilder map(Map values, boolean ensureNoSelfReferences) throws IOException { + /** writes a map without the start object and end object headers */ + public XContentBuilder mapContents(Map values) throws IOException { + return map(values, true, false); + } + + private XContentBuilder map(Map values, boolean ensureNoSelfReferences, boolean writeStartAndEndHeaders) throws IOException { if (values == null) { return nullValue(); } @@ -881,13 +886,17 @@ private XContentBuilder map(Map values, boolean ensureNoSelfReference ensureNoSelfReferences(values); } - startObject(); + if (writeStartAndEndHeaders) { + startObject(); + } for (Map.Entry value : values.entrySet()) { field(value.getKey()); // pass ensureNoSelfReferences=false as we already performed the check at a higher level unknownValue(value.getValue(), false); } - endObject(); + if (writeStartAndEndHeaders) { + endObject(); + } return this; } diff --git a/server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java b/server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java index 6fc3349c76233..1d3a31f0a72d9 100644 --- a/server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java +++ b/server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java @@ -88,14 +88,19 @@ protected void execute(Terminal terminal, OptionSet options) throws Exception { /** Create an {@link Environment} for the command to use. Overrideable for tests. */ protected Environment createEnv(final Map settings) throws UserException { + return createEnv(Settings.EMPTY, settings); + } + + /** Create an {@link Environment} for the command to use. Overrideable for tests. */ + protected final Environment createEnv(final Settings baseSettings, final Map settings) throws UserException { final String esPathConf = System.getProperty("es.path.conf"); if (esPathConf == null) { throw new UserException(ExitCodes.CONFIG, "the system property [es.path.conf] must be set"); } - return InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, settings, - getConfigPath(esPathConf), - // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available - () -> System.getenv("HOSTNAME")); + return InternalSettingsPreparer.prepareEnvironment(baseSettings, settings, + getConfigPath(esPathConf), + // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available + () -> System.getenv("HOSTNAME")); } @SuppressForbidden(reason = "need path to construct environment") diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java index a65934c767769..800269520e366 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java @@ -26,7 +26,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cli.EnvironmentAwareCommand; import org.elasticsearch.cli.Terminal; -import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.Tuple; @@ -42,7 +41,6 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand { private static final Logger logger = LogManager.getLogger(ElasticsearchNodeCommand.class); - protected final NamedXContentRegistry namedXContentRegistry; protected static final String DELIMITER = "------------------------------------------------------------------------\n"; static final String STOP_WARNING_MSG = @@ -61,7 +59,6 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand { public ElasticsearchNodeCommand(String description) { super(description); - namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables()); } protected void processNodePaths(Terminal terminal, OptionSet options, Environment env) throws IOException { @@ -80,7 +77,7 @@ protected void processNodePaths(Terminal terminal, OptionSet options, Environmen protected Tuple loadMetaData(Terminal terminal, Path[] dataPaths) throws IOException { terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file"); - final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths); + final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths); if (manifest == null) { throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG); @@ -89,8 +86,8 @@ protected Tuple loadMetaData(Terminal terminal, Path[] dataP throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG); } terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file"); - final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(), - dataPaths); + final MetaData metaData = MetaData.FORMAT_PRESERVE_CUSTOMS.loadGeneration( + logger, NamedXContentRegistry.EMPTY, manifest.getGlobalGeneration(), dataPaths); if (metaData == null) { throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]"); } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java index c15e832142eaf..05bc0116c13c6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeMetaData; import org.elasticsearch.node.Node; @@ -84,7 +85,7 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) { protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException { terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata"); - final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths); + final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths); if (nodeMetaData == null) { throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index d8fd88696e9f6..f8e1b48c6dd83 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -45,7 +45,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -1421,8 +1420,6 @@ public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOEx @Override public IndexMetaData fromXContent(XContentParser parser) throws IOException { - assert parser.getXContentRegistry() != NamedXContentRegistry.EMPTY - : "loading index metadata requires a working named xcontent registry"; return Builder.fromXContent(parser); } }; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index ce67e5b72f1d1..482d57bed54b5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -755,7 +755,7 @@ public static Diff readDiffFrom(StreamInput in) throws IOException { } public static MetaData fromXContent(XContentParser parser) throws IOException { - return Builder.fromXContent(parser); + return Builder.fromXContent(parser, false); } @Override @@ -1277,7 +1277,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon builder.endObject(); } - public static MetaData fromXContent(XContentParser parser) throws IOException { + public static MetaData fromXContent(XContentParser parser, boolean preserveUnknownCustoms) throws IOException { Builder builder = new Builder(); // we might get here after the meta-data element, or on a fresh parser @@ -1327,8 +1327,13 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { Custom custom = parser.namedObject(Custom.class, currentFieldName, null); builder.putCustom(custom.getWriteableName(), custom); } catch (NamedObjectNotFoundException ex) { - logger.warn("Skipping unknown custom object with type {}", currentFieldName); - parser.skipChildren(); + if (preserveUnknownCustoms) { + logger.warn("Adding unknown custom object with type {}", currentFieldName); + builder.putCustom(currentFieldName, new UnknownGatewayOnlyCustom(parser.mapOrdered())); + } else { + logger.warn("Skipping unknown custom object with type {}", currentFieldName); + parser.skipChildren(); + } } } } else if (token.isValue()) { @@ -1349,6 +1354,45 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { } } + public static class UnknownGatewayOnlyCustom implements Custom { + + private final Map contents; + + UnknownGatewayOnlyCustom(Map contents) { + this.contents = contents; + } + + @Override + public EnumSet context() { + return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.GATEWAY); + } + + @Override + public Diff diff(Custom previousState) { + throw new UnsupportedOperationException(); + } + + @Override + public String getWriteableName() { + throw new UnsupportedOperationException(); + } + + @Override + public Version getMinimalSupportedVersion() { + throw new UnsupportedOperationException(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.mapContents(contents); + } + } + private static final ToXContent.Params FORMAT_PARAMS; static { Map params = new HashMap<>(2); @@ -1360,16 +1404,25 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { /** * State format for {@link MetaData} to write to and load from disk */ - public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(GLOBAL_STATE_FILE_PREFIX) { + public static final MetaDataStateFormat FORMAT = createMetaDataStateFormat(false); - @Override - public void toXContent(XContentBuilder builder, MetaData state) throws IOException { - Builder.toXContent(state, builder, FORMAT_PARAMS); - } + /** + * Special state format for {@link MetaData} to write to and load from disk, preserving unknown customs + */ + public static final MetaDataStateFormat FORMAT_PRESERVE_CUSTOMS = createMetaDataStateFormat(true); - @Override - public MetaData fromXContent(XContentParser parser) throws IOException { - return Builder.fromXContent(parser); - } - }; + private static MetaDataStateFormat createMetaDataStateFormat(boolean preserveUnknownCustoms) { + return new MetaDataStateFormat(GLOBAL_STATE_FILE_PREFIX) { + + @Override + public void toXContent(XContentBuilder builder, MetaData state) throws IOException { + Builder.toXContent(state, builder, FORMAT_PARAMS); + } + + @Override + public MetaData fromXContent(XContentParser parser) throws IOException { + return Builder.fromXContent(parser, preserveUnknownCustoms); + } + }; + } } diff --git a/server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java b/server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java index 20b5552dfa8f8..25b4f79866eaa 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java +++ b/server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.gateway.WriteStateException; @@ -165,7 +166,7 @@ private String toIndexName(NodeEnvironment.NodePath[] nodePaths, String uuid) { indexPaths[i] = nodePaths[i].resolve(uuid); } try { - IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, indexPaths); + IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, indexPaths); return metaData.getIndex().getName(); } catch (Exception e) { return "no name for uuid: " + uuid + ": " + e; @@ -194,7 +195,7 @@ private void rewriteManifest(Terminal terminal, Manifest manifest, Path[] dataPa private Manifest loadManifest(Terminal terminal, Path[] dataPaths) throws IOException { terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest"); - final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths); + final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths); if (manifest == null) { terminal.println(Terminal.Verbosity.SILENT, PRE_V7_MESSAGE); diff --git a/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java b/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java index 34c7e9599e07f..f50bdf081ef85 100644 --- a/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java +++ b/server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java @@ -25,6 +25,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cluster.coordination.ElasticsearchNodeCommand; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import java.io.IOException; import java.nio.file.Path; @@ -74,7 +75,7 @@ public OverrideNodeVersionCommand() { protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException { final Path[] nodePaths = Arrays.stream(toNodePaths(dataPaths)).map(p -> p.path).toArray(Path[]::new); final NodeMetaData nodeMetaData - = new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, namedXContentRegistry, nodePaths); + = new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, NamedXContentRegistry.EMPTY, nodePaths); if (nodeMetaData == null) { throw new ElasticsearchException(NO_METADATA_MESSAGE); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index 36a78119c7665..7d2b10beb3279 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -365,7 +365,7 @@ public void testUnknownFieldClusterMetaData() throws IOException { .endObject() .endObject()); try (XContentParser parser = createParser(JsonXContent.jsonXContent, metadata)) { - MetaData.Builder.fromXContent(parser); + MetaData.Builder.fromXContent(parser, randomBoolean()); fail(); } catch (IllegalArgumentException e) { assertEquals("Unexpected field [random]", e.getMessage()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java index e2d0fcf5188a6..0338a64b6fe7e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java @@ -140,7 +140,7 @@ public void testSimpleJsonFromAndTo() throws IOException { String metaDataSource = MetaData.Builder.toXContent(metaData); - MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource)); + MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource), false); IndexMetaData indexMetaData = parsedMetaData.index("test1"); assertThat(indexMetaData.primaryTerm(0), equalTo(1L)); diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java index 40f3bd8a01623..c7dab0dc4d4a0 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java @@ -61,6 +61,8 @@ import java.util.stream.StreamSupport; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -80,7 +82,7 @@ public void toXContent(XContentBuilder builder, MetaData state) { @Override public MetaData fromXContent(XContentParser parser) throws IOException { - return MetaData.Builder.fromXContent(parser); + return MetaData.Builder.fromXContent(parser, false); } }; Path tmp = createTempDir(); @@ -233,7 +235,23 @@ public static void corruptFile(Path fileToCorrupt, Logger logger) throws IOExcep } } - public void testLoadState() throws IOException { + public void testLoadStateWithoutMissingCustoms() throws IOException { + runLoadStateTest(false, false); + } + + public void testLoadStateWithoutMissingCustomsButPreserved() throws IOException { + runLoadStateTest(false, true); + } + + public void testLoadStateWithMissingCustomsButPreserved() throws IOException { + runLoadStateTest(true, true); + } + + public void testLoadStateWithMissingCustomsAndNotPreserved() throws IOException { + runLoadStateTest(true, false); + } + + private void runLoadStateTest(boolean hasMissingCustoms, boolean preserveUnknownCustoms) throws IOException { final Path[] dirs = new Path[randomIntBetween(1, 5)]; int numStates = randomIntBetween(1, 5); List meta = new ArrayList<>(); @@ -241,7 +259,7 @@ public void testLoadState() throws IOException { meta.add(randomMeta()); } Set corruptedFiles = new HashSet<>(); - MetaDataStateFormat format = metaDataFormat(); + MetaDataStateFormat format = metaDataFormat(preserveUnknownCustoms); for (int i = 0; i < dirs.length; i++) { dirs[i] = createTempDir(); Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME)); @@ -258,11 +276,12 @@ public void testLoadState() throws IOException { } List dirList = Arrays.asList(dirs); Collections.shuffle(dirList, random()); - MetaData loadedMetaData = format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0])); + MetaData loadedMetaData = format.loadLatestState(logger, hasMissingCustoms ? + NamedXContentRegistry.EMPTY : xContentRegistry(), dirList.toArray(new Path[0])); MetaData latestMetaData = meta.get(numStates-1); assertThat(loadedMetaData.clusterUUID(), not(equalTo("_na_"))); assertThat(loadedMetaData.clusterUUID(), equalTo(latestMetaData.clusterUUID())); - ImmutableOpenMap indices = loadedMetaData.indices(); + ImmutableOpenMap indices = loadedMetaData.indices(); assertThat(indices.size(), equalTo(latestMetaData.indices().size())); for (IndexMetaData original : latestMetaData) { IndexMetaData deserialized = indices.get(original.getIndex().getName()); @@ -275,7 +294,23 @@ public void testLoadState() throws IOException { } // make sure the index tombstones are the same too - assertThat(loadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard())); + if (hasMissingCustoms) { + if (preserveUnknownCustoms) { + assertNotNull(loadedMetaData.custom(IndexGraveyard.TYPE)); + assertThat(loadedMetaData.custom(IndexGraveyard.TYPE), instanceOf(MetaData.UnknownGatewayOnlyCustom.class)); + + // check that we reserialize unknown metadata correctly again + final Path tempdir = createTempDir(); + metaDataFormat(randomBoolean()).write(loadedMetaData, tempdir); + final MetaData reloadedMetaData = metaDataFormat(randomBoolean()).loadLatestState(logger, xContentRegistry(), tempdir); + assertThat(reloadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard())); + } else { + assertNotNull(loadedMetaData.indexGraveyard()); + assertThat(loadedMetaData.indexGraveyard().getTombstones(), hasSize(0)); + } + } else { + assertThat(loadedMetaData.indexGraveyard(), equalTo(latestMetaData.indexGraveyard())); + } // now corrupt all the latest ones and make sure we fail to load the state for (int i = 0; i < dirs.length; i++) { @@ -419,7 +454,7 @@ public void testFailRandomlyAndReadAnyState() throws IOException { writeAndReadStateSuccessfully(format, paths); } - private static MetaDataStateFormat metaDataFormat() { + private static MetaDataStateFormat metaDataFormat(boolean preserveUnknownCustoms) { return new MetaDataStateFormat(MetaData.GLOBAL_STATE_FILE_PREFIX) { @Override public void toXContent(XContentBuilder builder, MetaData state) throws IOException { @@ -428,7 +463,7 @@ public void toXContent(XContentBuilder builder, MetaData state) throws IOExcepti @Override public MetaData fromXContent(XContentParser parser) throws IOException { - return MetaData.Builder.fromXContent(parser); + return MetaData.Builder.fromXContent(parser, preserveUnknownCustoms); } }; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java index d7799959f6cce..084d965a6e74b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetaDataSerializationTests.java @@ -80,7 +80,7 @@ public void testLicenseMetadataParsingDoesNotSwallowOtherMetaData() throws Excep builder = metaDataBuilder.build().toXContent(builder, params); builder.endObject(); // deserialize metadata again - MetaData metaData = MetaData.Builder.fromXContent(createParser(builder)); + MetaData metaData = MetaData.Builder.fromXContent(createParser(builder), randomBoolean()); // check that custom metadata still present assertThat(metaData.custom(licensesMetaData.getWriteableName()), notNullValue()); assertThat(metaData.custom(repositoriesMetaData.getWriteableName()), notNullValue()); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java index 0556b8535e428..75e5bc1073e69 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetaDataSerializationTests.java @@ -64,7 +64,7 @@ public void testWatcherMetadataParsingDoesNotSwallowOtherMetaData() throws Excep builder = metaDataBuilder.build().toXContent(builder, params); builder.endObject(); // deserialize metadata again - MetaData metaData = MetaData.Builder.fromXContent(createParser(builder)); + MetaData metaData = MetaData.Builder.fromXContent(createParser(builder), randomBoolean()); // check that custom metadata still present assertThat(metaData.custom(watcherMetaData.getWriteableName()), notNullValue()); assertThat(metaData.custom(repositoriesMetaData.getWriteableName()), notNullValue());