Merge pull request #22335 from rjernst/keystore

Settings: Add infrastructure for elasticsearch keystore

Author: rjernst

core/build.gradle

@@ -94,6 +94,8 @@ dependencies {
       exclude group: 'org.elasticsearch', module: 'elasticsearch'
     }
   }
+  testCompile 'com.google.jimfs:jimfs:1.1'
+  testCompile 'com.google.guava:guava:18.0'
 }
 
 if (isEclipse) {

core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -30,17 +30,16 @@
 import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.PidFile;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.inject.CreationException;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.IfConfig;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.env.Environment;
@@ -228,13 +227,36 @@ protected void validateNodeBeforeAcceptingRequests(
         };
     }
 
-    private static Environment initialEnvironment(boolean foreground, Path pidFile, Settings initialSettings) {
+    private static KeyStoreWrapper loadKeyStore(Environment initialEnv) throws BootstrapException {
+        final KeyStoreWrapper keystore;
+        try {
+            keystore = KeyStoreWrapper.load(initialEnv.configFile());
+        } catch (IOException e) {
+            throw new BootstrapException(e);
+        }
+        if (keystore == null) {
+            return null; // no keystore
+        }
+
+        try {
+            keystore.decrypt(new char[0] /* TODO: read password from stdin */);
+        } catch (Exception e) {
+            throw new BootstrapException(e);
+        }
+        return keystore;
+    }
+
+    private static Environment createEnvironment(boolean foreground, Path pidFile,
+                                                 KeyStoreWrapper keystore, Settings initialSettings) {
         Terminal terminal = foreground ? Terminal.DEFAULT : null;
         Settings.Builder builder = Settings.builder();
         if (pidFile != null) {
             builder.put(Environment.PIDFILE_SETTING.getKey(), pidFile);
         }
         builder.put(initialSettings);
+        if (keystore != null) {
+            builder.setKeyStore(keystore);
+        }
         return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal, Collections.emptyMap());
     }
 
@@ -265,7 +287,7 @@ static void init(
             final boolean foreground,
             final Path pidFile,
             final boolean quiet,
-            final Settings initialSettings) throws BootstrapException, NodeValidationException, UserException {
+            final Environment initialEnv) throws BootstrapException, NodeValidationException, UserException {
         // Set the system property before anything has a chance to trigger its use
         initLoggerPrefix();
 
@@ -275,7 +297,8 @@ static void init(
 
         INSTANCE = new Bootstrap();
 
-        Environment environment = initialEnvironment(foreground, pidFile, initialSettings);
+        final KeyStoreWrapper keystore = loadKeyStore(initialEnv);
+        Environment environment = createEnvironment(foreground, pidFile, keystore, initialEnv.settings());
         try {
             LogConfigurator.configure(environment);
         } catch (IOException e) {
@@ -313,6 +336,13 @@ static void init(
 
             INSTANCE.setup(true, environment);
 
+            try {
+                // any secure settings must be read during node construction
+                IOUtils.close(keystore);
+            } catch (IOException e) {
+                throw new BootstrapException(e);
+            }
+
             INSTANCE.start();
 
             if (closeStandardStreams) {

core/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java

@@ -26,7 +26,7 @@
  * during bootstrap should explicitly declare the checked exceptions that they can throw, rather
  * than declaring the top-level checked exception {@link Exception}. This exception exists to wrap
  * these checked exceptions so that
- * {@link Bootstrap#init(boolean, Path, boolean, org.elasticsearch.common.settings.Settings)}
+ * {@link Bootstrap#init(boolean, Path, boolean, org.elasticsearch.env.Environment)}
  * does not have to declare all of these checked exceptions.
  */
 class BootstrapException extends Exception {

core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -111,16 +111,16 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th
         final boolean quiet = options.has(quietOption);
 
         try {
-            init(daemonize, pidFile, quiet, env.settings());
+            init(daemonize, pidFile, quiet, env);
         } catch (NodeValidationException e) {
             throw new UserException(ExitCodes.CONFIG, e.getMessage());
         }
     }
 
-    void init(final boolean daemonize, final Path pidFile, final boolean quiet, Settings initialSettings)
+    void init(final boolean daemonize, final Path pidFile, final boolean quiet, Environment initialEnv)
         throws NodeValidationException, UserException {
         try {
-            Bootstrap.init(!daemonize, pidFile, quiet, initialSettings);
+            Bootstrap.init(!daemonize, pidFile, quiet, initialEnv);
         } catch (BootstrapException | RuntimeException e) {
             // format exceptions to the console in a special way
             // to avoid 2MB stacktraces from guice, etc.

core/src/main/java/org/elasticsearch/cli/Terminal.java

@@ -27,6 +27,7 @@
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.nio.charset.Charset;
+import java.util.Locale;
 
 /**
  * A Terminal wraps access to reading input and writing output for a cli.
@@ -92,6 +93,26 @@ public final void print(Verbosity verbosity, String msg) {
         }
     }
 
+    /**
+     * Prompt for a yes or no answer from the user. This method will loop until 'y' or 'n'
+     * (or the default empty value) is entered.
+     */
+    public final boolean promptYesNo(String prompt, boolean defaultYes) {
+        String answerPrompt = defaultYes ? " [Y/n]" : " [y/N]";
+        while (true) {
+            String answer = readText(prompt + answerPrompt).toLowerCase(Locale.ROOT);
+            if (answer.isEmpty()) {
+                return defaultYes;
+            }
+            boolean answerYes = answer.equals("y");
+            if (answerYes == false && answer.equals("n") == false) {
+                println("Did not understand answer '" + answer + "'");
+                continue;
+            }
+            return answerYes;
+        }
+    }
+
     private static class ConsoleTerminal extends Terminal {
 
         private static final Console CONSOLE = System.console();

core/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli which adds a string setting.
+ */
+class AddStringKeyStoreCommand extends EnvironmentAwareCommand {
+
+    private final OptionSpec<Void> stdinOption;
+    private final OptionSpec<Void> forceOption;
+    private final OptionSpec<String> arguments;
+
+    AddStringKeyStoreCommand() {
+        super("Add a string setting to the keystore");
+        this.stdinOption = parser.acceptsAll(Arrays.asList("x", "stdin"), "Read setting value from stdin");
+        this.forceOption = parser.acceptsAll(Arrays.asList("f", "force"), "Overwrite existing setting without prompting");
+        this.arguments = parser.nonOptions("setting name");
+    }
+
+    // pkg private so tests can manipulate
+    InputStream getStdin() {
+        return System.in;
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        String setting = arguments.value(options);
+        if (keystore.getSettings().contains(setting) && options.has(forceOption) == false) {
+            if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
+                terminal.println("Exiting without modifying keystore.");
+                return;
+            }
+        }
+
+        final char[] value;
+        if (options.has(stdinOption)) {
+            BufferedReader stdinReader = new BufferedReader(new InputStreamReader(getStdin(), StandardCharsets.UTF_8));
+            value = stdinReader.readLine().toCharArray();
+        } else {
+            value = terminal.readSecret("Enter value for " + setting + ": ");
+        }
+
+        try {
+            keystore.setStringSetting(setting, value);
+        } catch (IllegalArgumentException e) {
+            throw new UserException(ExitCodes.DATA_ERROR, "String value must contain only ASCII");
+        }
+        keystore.save(env.configFile());
+    }
+}

core/src/main/java/org/elasticsearch/common/settings/CreateKeyStoreCommand.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to create a new keystore.
+ */
+class CreateKeyStoreCommand extends EnvironmentAwareCommand {
+
+    CreateKeyStoreCommand() {
+        super("Creates a new elasticsearch keystore");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+        if (Files.exists(keystoreFile)) {
+            if (terminal.promptYesNo("An elasticsearch keystore already exists. Overwrite?", false) == false) {
+                terminal.println("Exiting without creating keystore.");
+                return;
+            }
+        }
+
+
+        char[] password = new char[0];// terminal.readSecret("Enter passphrase (empty for no passphrase): ");
+        /* TODO: uncomment when entering passwords on startup is supported
+        char[] passwordRepeat = terminal.readSecret("Enter same passphrase again: ");
+        if (Arrays.equals(password, passwordRepeat) == false) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Passphrases are not equal, exiting.");
+        }*/
+
+        KeyStoreWrapper keystore = KeyStoreWrapper.create(password);
+        keystore.save(env.configFile());
+        terminal.println("Created elasticsearch keystore in " + env.configFile());
+    }
+}

core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.cli.MultiCommand;
+import org.elasticsearch.cli.Terminal;
+
+/**
+ * A cli tool for managing secrets in the elasticsearch keystore.
+ */
+public class KeyStoreCli extends MultiCommand {
+
+    private KeyStoreCli() {
+        super("A tool for managing settings stored in the elasticsearch keystore");
+        subcommands.put("create", new CreateKeyStoreCommand());
+        subcommands.put("list", new ListKeyStoreCommand());
+        subcommands.put("add", new AddStringKeyStoreCommand());
+        subcommands.put("remove", new RemoveSettingKeyStoreCommand());
+    }
+
+    public static void main(String[] args) throws Exception {
+        exit(new KeyStoreCli().main(args, Terminal.DEFAULT));
+    }
+}