Make elasticsearch-node tools custom metadata-aware

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -305,13 +305,18 @@ public void restart() {
         nodes.forEach(ElasticsearchNode::restart);
     }
 
-    public void goToNextVersion() {
+    public void goToNextVersion(Runnable onStopped) {
         stop(false);
+        onStopped.run();
         nodes.all(ElasticsearchNode::goToNextVersion);
         start();
         writeUnicastHostsFiles();
     }
 
+    public void goToNextVersion() {
+        goToNextVersion(() -> {});
+    }
+
     public void nextNodeToNextVersion() {
         if (nodeIndex + 1 > nodes.size()) {
             throw new TestClustersException("Ran out of nodes to take to the next version");
@@ -334,6 +339,11 @@ public void extraConfigFile(String destination, File from, PropertyNormalization
         nodes.all(node -> node.extraConfigFile(destination, from, normalization));
     }
 
+    @Override
+    public void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
+        nodes.all(node -> node.runElasticsearchBinScriptWithInput(input, tool, args));
+    }
+
     @Override
     public void extraJarFile(File from) {
         nodes.all(node -> node.extraJarFile(from));

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -424,23 +424,23 @@ public synchronized void start() {
 
         if (plugins.isEmpty() == false) {
             logToProcessStdout("Installing " + plugins.size() + " plugins");
-            plugins.forEach(plugin -> runElaticsearchBinScript(
+            plugins.forEach(plugin -> runElasticsearchBinScript(
                 "elasticsearch-plugin",
                 "install", "--batch", plugin.toString())
             );
         }
 
         if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) {
             LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion());
-            runElaticsearchBinScript(
+            runElasticsearchBinScript(
                 "elasticsearch-plugin",
                 "install", "--batch", "x-pack"
             );
         }
 
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
-            runElaticsearchBinScript("elasticsearch-keystore", "create");
+            runElasticsearchBinScript("elasticsearch-keystore", "create");
 
             keystoreSettings.forEach((key, value) ->
                 runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
@@ -452,7 +452,7 @@ public synchronized void start() {
                 if (file.exists() == false) {
                     throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
                 }
-                runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
+                runElasticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
             }
         }
 
@@ -467,7 +467,7 @@ public synchronized void start() {
         if (credentials.isEmpty() == false) {
             logToProcessStdout("Setting up " + credentials.size() + " users");
 
-            credentials.forEach(paramMap -> runElaticsearchBinScript(
+            credentials.forEach(paramMap -> runElasticsearchBinScript(
                 getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
                 paramMap.entrySet().stream()
                     .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
@@ -623,7 +623,8 @@ public void user(Map<String, String> userSpec) {
         credentials.add(cred);
     }
 
-    private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
+    @Override
+    public void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
         if (
             Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false &&
                 Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false
@@ -663,7 +664,7 @@ private void runElasticsearchBinScriptWithInput(String input, String tool, Strin
         }
     }
 
-    private void runElaticsearchBinScript(String tool, String... args) {
+    private void runElasticsearchBinScript(String tool, String... args) {
         runElasticsearchBinScriptWithInput("", tool, args);
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -92,6 +92,8 @@ public interface TestClusterConfiguration {
 
     void extraConfigFile(String destination, File from, PropertyNormalization normalization);
 
+    void runElasticsearchBinScriptWithInput(String input, String tool, String... args);
+
     void extraJarFile(File from);
 
     void user(Map<String, String> userSpec);

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java

@@ -819,7 +819,7 @@ private void unknownValue(Object value, boolean ensureNoSelfReferences) throws I
         } else if (value instanceof Map) {
             @SuppressWarnings("unchecked")
             final Map<String, ?> valueMap = (Map<String, ?>) value;
-            map(valueMap, ensureNoSelfReferences);
+            map(valueMap, ensureNoSelfReferences, true);
         } else if (value instanceof Iterable) {
             value((Iterable<?>) value, ensureNoSelfReferences);
         } else if (value instanceof Object[]) {
@@ -867,10 +867,15 @@ public XContentBuilder field(String name, Map<String, Object> values) throws IOE
     }
 
     public XContentBuilder map(Map<String, ?> values) throws IOException {
-        return map(values, true);
+        return map(values, true, true);
     }
 
-    private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences) throws IOException {
+    /** writes a map without the start object and end object headers */
+    public XContentBuilder mapContents(Map<String, ?> values) throws IOException {
+        return map(values, true, false);
+    }
+
+    private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences, boolean writeStartAndEndHeaders) throws IOException {
         if (values == null) {
             return nullValue();
         }
@@ -881,13 +886,17 @@ private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReference
             ensureNoSelfReferences(values);
         }
 
-        startObject();
+        if (writeStartAndEndHeaders) {
+            startObject();
+        }
         for (Map.Entry<String, ?> value : values.entrySet()) {
             field(value.getKey());
             // pass ensureNoSelfReferences=false as we already performed the check at a higher level
             unknownValue(value.getValue(), false);
         }
-        endObject();
+        if (writeStartAndEndHeaders) {
+            endObject();
+        }
         return this;
     }
 

server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java

@@ -88,14 +88,19 @@ protected void execute(Terminal terminal, OptionSet options) throws Exception {
 
     /** Create an {@link Environment} for the command to use. Overrideable for tests. */
     protected Environment createEnv(final Map<String, String> settings) throws UserException {
+        return createEnv(Settings.EMPTY, settings);
+    }
+
+    /** Create an {@link Environment} for the command to use. Overrideable for tests. */
+    protected final Environment createEnv(final Settings baseSettings, final Map<String, String> settings) throws UserException {
         final String esPathConf = System.getProperty("es.path.conf");
         if (esPathConf == null) {
             throw new UserException(ExitCodes.CONFIG, "the system property [es.path.conf] must be set");
         }
-        return InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, settings,
-                getConfigPath(esPathConf),
-                // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
-                () -> System.getenv("HOSTNAME"));
+        return InternalSettingsPreparer.prepareEnvironment(baseSettings, settings,
+            getConfigPath(esPathConf),
+            // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
+            () -> System.getenv("HOSTNAME"));
     }
 
     @SuppressForbidden(reason = "need path to construct environment")

server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java

@@ -26,7 +26,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.collect.Tuple;
@@ -42,7 +41,6 @@
 
 public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
     private static final Logger logger = LogManager.getLogger(ElasticsearchNodeCommand.class);
-    protected final NamedXContentRegistry namedXContentRegistry;
     protected static final String DELIMITER = "------------------------------------------------------------------------\n";
 
     static final String STOP_WARNING_MSG =
@@ -61,7 +59,6 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
 
     public ElasticsearchNodeCommand(String description) {
         super(description);
-        namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
     }
 
     protected void processNodePaths(Terminal terminal, OptionSet options, Environment env) throws IOException {
@@ -80,7 +77,7 @@ protected void processNodePaths(Terminal terminal, OptionSet options, Environmen
 
     protected Tuple<Manifest, MetaData> loadMetaData(Terminal terminal, Path[] dataPaths) throws IOException {
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file");
-        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
 
         if (manifest == null) {
             throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG);
@@ -89,8 +86,8 @@ protected Tuple<Manifest, MetaData> loadMetaData(Terminal terminal, Path[] dataP
             throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG);
         }
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file");
-        final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(),
-                dataPaths);
+        final MetaData metaData = MetaData.FORMAT_PRESERVE_CUSTOMS.loadGeneration(
+            logger, NamedXContentRegistry.EMPTY, manifest.getGlobalGeneration(), dataPaths);
         if (metaData == null) {
             throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]");
         }

server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java

@@ -41,6 +41,7 @@ public NodeToolCli() {
         subcommands.put("unsafe-bootstrap", new UnsafeBootstrapMasterCommand());
         subcommands.put("detach-cluster", new DetachClusterCommand());
         subcommands.put("override-version", new OverrideNodeVersionCommand());
+        subcommands.put("read-and-write-metadata", new ReadAndWriteMetaDataCommand());
     }
 
     public static void main(String[] args) throws Exception {

server/src/main/java/org/elasticsearch/cluster/coordination/ReadAndWriteMetaDataCommand.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cluster.metadata.Manifest;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.env.Environment;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+public class ReadAndWriteMetaDataCommand extends ElasticsearchNodeCommand {
+
+    public ReadAndWriteMetaDataCommand() {
+        super("reads the metadata on disk and writes it back");
+    }
+
+    @Override
+    protected void processNodePaths(
+        Terminal terminal,
+        Path[] dataPaths,
+        Environment env) throws IOException {
+
+        final Tuple<Manifest, MetaData> manifestMetaDataTuple = loadMetaData(terminal, dataPaths);
+        final Manifest manifest = manifestMetaDataTuple.v1();
+        final MetaData metaData = manifestMetaDataTuple.v2();
+
+        confirm(terminal, "metadata successfully read for cluster state version " + manifest.getClusterStateVersion() +
+            ". Do you want to write it back out?\n");
+
+        writeNewMetaData(terminal, manifest, manifest.getCurrentTerm(), metaData, metaData, dataPaths);
+    }
+}

server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeMetaData;
 import org.elasticsearch.node.Node;
@@ -84,7 +85,7 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) {
 
     protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata");
-        final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+        final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
         if (nodeMetaData == null) {
             throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG);
         }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -45,7 +45,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -1421,8 +1420,6 @@ public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOEx
 
         @Override
         public IndexMetaData fromXContent(XContentParser parser) throws IOException {
-            assert parser.getXContentRegistry() != NamedXContentRegistry.EMPTY
-                    : "loading index metadata requires a working named xcontent registry";
             return Builder.fromXContent(parser);
         }
     };