Settings: Migrate ec2 discovery sensitive settings to elasticsearch keystore

This change adds secure settings for access/secret keys and proxy
username/password to ec2 discovery.  It adds the new settings with the
prefix `discovery.ec2`, copies other relevant ec2 client settings to the
same prefix, and deprecates all other settings (`cloud.aws.*` and
`cloud.aws.ec2.*`).  Note that this is simpler than the client configs
in repository-s3 because discovery is only initialized once for the
entire node, so there is no reason to complicate the configuration with
the ability to have multiple sets of client settings.

relates elastic#22475

Author: rjernst

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java

@@ -22,6 +22,8 @@
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import com.amazonaws.services.ec2.AmazonEC2;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -42,50 +44,52 @@ interface AwsEc2Service {
     /**
      * cloud.aws.access_key: AWS Access key. Shared with repository-s3 plugin
      */
-    Setting<String> KEY_SETTING =
-        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered, Property.Shared);
+    Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.access_key", "", SecureString::new,
+            Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.secret_key: AWS Secret key. Shared with repository-s3 plugin
      */
-    Setting<String> SECRET_SETTING =
-        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered, Property.Shared);
+    Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.secret_key", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with repository-s3 plugin
      */
     Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-        Property.NodeScope, Property.Shared);
+        Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared);
+    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host",
+        Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with repository-s3 plugin
      */
-    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope,
-        Property.Shared);
+    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16,
+        Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope, Property.Shared);
+    Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.proxy.username", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_PASSWORD_SETTING =
-        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered, Property.Shared);
+    Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.proxy.password", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with repository-s3 plugin
      */
-    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared);
+    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.region: Region. Shared with repository-s3 plugin
      */
     Setting<String> REGION_SETTING =
-        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared);
+        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.read_timeout: Socket read timeout. Shared with repository-s3 plugin
      */
     Setting<TimeValue> READ_TIMEOUT = Setting.timeSetting("cloud.aws.read_timeout",
-        TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Shared);
+        TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Shared, Property.Deprecated);
 
     /**
      * Defines specific ec2 settings starting with cloud.aws.ec2.
@@ -95,69 +99,70 @@ interface CLOUD_EC2 {
          * cloud.aws.ec2.access_key: AWS Access key specific for EC2 API calls. Defaults to cloud.aws.access_key.
          * @see AwsEc2Service#KEY_SETTING
          */
-        Setting<String> KEY_SETTING = new Setting<>("cloud.aws.ec2.access_key", AwsEc2Service.KEY_SETTING, Function.identity(),
-            Property.NodeScope, Property.Filtered);
+        Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.ec2.access_key", AwsEc2Service.KEY_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
+
         /**
          * cloud.aws.ec2.secret_key: AWS Secret key specific for EC2 API calls. Defaults to cloud.aws.secret_key.
          * @see AwsEc2Service#SECRET_SETTING
          */
-        Setting<String> SECRET_SETTING = new Setting<>("cloud.aws.ec2.secret_key", AwsEc2Service.SECRET_SETTING, Function.identity(),
-            Property.NodeScope, Property.Filtered);
+        Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.ec2.secret_key", AwsEc2Service.SECRET_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
         /**
          * cloud.aws.ec2.protocol: Protocol for AWS API specific for EC2 API calls: http or https.  Defaults to cloud.aws.protocol.
          * @see AwsEc2Service#PROTOCOL_SETTING
          */
         Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.ec2.protocol", AwsEc2Service.PROTOCOL_SETTING,
-            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
+            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.proxy.host: In case of proxy, define its hostname/IP specific for EC2 API calls. Defaults to cloud.aws.proxy.host.
          * @see AwsEc2Service#PROXY_HOST_SETTING
          */
         Setting<String> PROXY_HOST_SETTING = new Setting<>("cloud.aws.ec2.proxy.host", AwsEc2Service.PROXY_HOST_SETTING,
-            Function.identity(), Property.NodeScope);
+            Function.identity(), Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.proxy.port: In case of proxy, define its port specific for EC2 API calls.  Defaults to cloud.aws.proxy.port.
          * @see AwsEc2Service#PROXY_PORT_SETTING
          */
         Setting<Integer> PROXY_PORT_SETTING = new Setting<>("cloud.aws.ec2.proxy.port", AwsEc2Service.PROXY_PORT_SETTING,
-            s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.ec2.proxy.port"), Property.NodeScope);
+            s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.ec2.proxy.port"), Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.proxy.username: In case of proxy with auth, define the username specific for EC2 API calls.
          * Defaults to cloud.aws.proxy.username.
          * @see AwsEc2Service#PROXY_USERNAME_SETTING
          */
-        Setting<String> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.ec2.proxy.username", AwsEc2Service.PROXY_USERNAME_SETTING,
-            Function.identity(), Property.NodeScope);
+        Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.ec2.proxy.username", AwsEc2Service.PROXY_USERNAME_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
         /**
          * cloud.aws.ec2.proxy.password: In case of proxy with auth, define the password specific for EC2 API calls.
          * Defaults to cloud.aws.proxy.password.
          * @see AwsEc2Service#PROXY_PASSWORD_SETTING
          */
-        Setting<String> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.ec2.proxy.password", AwsEc2Service.PROXY_PASSWORD_SETTING,
-            Function.identity(), Property.NodeScope, Property.Filtered);
+        Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.ec2.proxy.password", AwsEc2Service.PROXY_PASSWORD_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
         /**
          * cloud.aws.ec2.signer: If you are using an old AWS API version, you can define a Signer. Specific for EC2 API calls.
          * Defaults to cloud.aws.signer.
          * @see AwsEc2Service#SIGNER_SETTING
          */
         Setting<String> SIGNER_SETTING = new Setting<>("cloud.aws.ec2.signer", AwsEc2Service.SIGNER_SETTING, Function.identity(),
-            Property.NodeScope);
+            Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.region: Region specific for EC2 API calls. Defaults to cloud.aws.region.
          * @see AwsEc2Service#REGION_SETTING
          */
         Setting<String> REGION_SETTING = new Setting<>("cloud.aws.ec2.region", AwsEc2Service.REGION_SETTING,
-            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
+            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
          */
-        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.ec2.endpoint", Property.NodeScope);
+        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.ec2.endpoint", Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.read_timeout: Socket read timeout. Defaults to cloud.aws.read_timeout
          * @see AwsEc2Service#READ_TIMEOUT
          */
         Setting<TimeValue> READ_TIMEOUT =
-            Setting.timeSetting("cloud.aws.ec2.read_timeout", AwsEc2Service.READ_TIMEOUT, Property.NodeScope);
+            Setting.timeSetting("cloud.aws.ec2.read_timeout", AwsEc2Service.READ_TIMEOUT, Property.NodeScope, Property.Deprecated);
     }
 
     /**
@@ -172,6 +177,40 @@ class HostType {
             public static final String TAG_PREFIX = "tag:";
         }
 
+        /** The access key (ie login id) for connecting to ec2. */
+        Setting<SecureString> ACCESS_KEY_SETTING = SecureSetting.secureString("discovery.ec2.access_key", CLOUD_EC2.KEY_SETTING, false);
+
+        /** The secret key (ie password) for connecting to ec2. */
+        Setting<SecureString> SECRET_KEY_SETTING = SecureSetting.secureString("discovery.ec2.secret_key", CLOUD_EC2.SECRET_SETTING, false);
+
+        /** An override for the ec2 endpoint to connect to. */
+        Setting<String> ENDPOINT_SETTING = new Setting<>("discovery.ec2.endpoint", CLOUD_EC2.ENDPOINT_SETTING,
+            s -> s.toLowerCase(Locale.ROOT), Setting.Property.NodeScope);
+
+        /** The protocol to use to connect to to ec2. */
+        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("discovery.ec2.protocol", CLOUD_EC2.PROTOCOL_SETTING,
+            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Setting.Property.NodeScope);
+
+        /** The host name of a proxy to connect to ec2 through. */
+        Setting<String> PROXY_HOST_SETTING = new Setting<>("discovery.ec2.proxy.host", CLOUD_EC2.PROXY_HOST_SETTING,
+            Function.identity(), Setting.Property.NodeScope);
+
+        /** The port of a proxy to connect to ec2 through. */
+        Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("discovery.ec2.proxy.port", CLOUD_EC2.PROXY_PORT_SETTING,
+            0, Setting.Property.NodeScope);
+
+        /** The username of a proxy to connect to s3 through. */
+        Setting<SecureString> PROXY_USERNAME_SETTING = SecureSetting.secureString("discovery.ec2.proxy.username",
+            CLOUD_EC2.PROXY_USERNAME_SETTING, false);
+
+        /** The password of a proxy to connect to s3 through. */
+        Setting<SecureString> PROXY_PASSWORD_SETTING =  SecureSetting.secureString("discovery.ec2.proxy.password",
+            CLOUD_EC2.PROXY_PASSWORD_SETTING, false);
+
+        /** The socket timeout for connecting to s3. */
+        Setting<TimeValue> READ_TIMEOUT_SETTING = Setting.timeSetting("discovery.ec2.read_timeout",
+            CLOUD_EC2.READ_TIMEOUT, Setting.Property.NodeScope);
+
         /**
          * discovery.ec2.host_type: The type of host type to use to communicate with other instances.
          * Can be one of private_ip, public_ip, private_dns, public_dns or tag:XXXX where

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 
 class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service, Closeable {
@@ -68,14 +69,15 @@ public synchronized AmazonEC2 client() {
     protected static AWSCredentialsProvider buildCredentials(Logger logger, Settings settings) {
         AWSCredentialsProvider credentials;
 
-        String key = CLOUD_EC2.KEY_SETTING.get(settings);
-        String secret = CLOUD_EC2.SECRET_SETTING.get(settings);
-        if (key.isEmpty() && secret.isEmpty()) {
-            logger.debug("Using either environment variables, system properties or instance profile credentials");
-            credentials = new DefaultAWSCredentialsProviderChain();
-        } else {
-            logger.debug("Using basic key/secret credentials");
-            credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key, secret));
+        try (SecureString key = DISCOVERY_EC2.ACCESS_KEY_SETTING.get(settings);
+             SecureString secret = DISCOVERY_EC2.SECRET_KEY_SETTING.get(settings)) {
+            if (key.length() == 0 && secret.length() == 0) {
+                logger.debug("Using either environment variables, system properties or instance profile credentials");
+                credentials = new DefaultAWSCredentialsProviderChain();
+            } else {
+                logger.debug("Using basic key/secret credentials");
+                credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key.toString(), secret.toString()));
+            }
         }
 
         return credentials;
@@ -86,19 +88,20 @@ protected static ClientConfiguration buildConfiguration(Logger logger, Settings
         // the response metadata cache is only there for diagnostics purposes,
         // but can force objects from every response to the old generation.
         clientConfiguration.setResponseMetadataCacheSize(0);
-        clientConfiguration.setProtocol(CLOUD_EC2.PROTOCOL_SETTING.get(settings));
-
-        if (PROXY_HOST_SETTING.exists(settings) || CLOUD_EC2.PROXY_HOST_SETTING.exists(settings)) {
-            String proxyHost = CLOUD_EC2.PROXY_HOST_SETTING.get(settings);
-            Integer proxyPort = CLOUD_EC2.PROXY_PORT_SETTING.get(settings);
-            String proxyUsername = CLOUD_EC2.PROXY_USERNAME_SETTING.get(settings);
-            String proxyPassword = CLOUD_EC2.PROXY_PASSWORD_SETTING.get(settings);
-
-            clientConfiguration
-                .withProxyHost(proxyHost)
-                .withProxyPort(proxyPort)
-                .withProxyUsername(proxyUsername)
-                .withProxyPassword(proxyPassword);
+        clientConfiguration.setProtocol(DISCOVERY_EC2.PROTOCOL_SETTING.get(settings));
+
+        if (PROXY_HOST_SETTING.exists(settings) || DISCOVERY_EC2.PROXY_HOST_SETTING.exists(settings)) {
+            String proxyHost = DISCOVERY_EC2.PROXY_HOST_SETTING.get(settings);
+            Integer proxyPort = DISCOVERY_EC2.PROXY_PORT_SETTING.get(settings);
+            try (SecureString proxyUsername = DISCOVERY_EC2.PROXY_USERNAME_SETTING.get(settings);
+                 SecureString proxyPassword = DISCOVERY_EC2.PROXY_PASSWORD_SETTING.get(settings)) {
+
+                clientConfiguration
+                    .withProxyHost(proxyHost)
+                    .withProxyPort(proxyPort)
+                    .withProxyUsername(proxyUsername.toString())
+                    .withProxyPassword(proxyPassword.toString());
+            }
         }
 
         // #155: we might have 3rd party users using older EC2 API version
@@ -125,15 +128,15 @@ public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest,
             10,
             false);
         clientConfiguration.setRetryPolicy(retryPolicy);
-        clientConfiguration.setSocketTimeout((int) CLOUD_EC2.READ_TIMEOUT.get(settings).millis());
+        clientConfiguration.setSocketTimeout((int) DISCOVERY_EC2.READ_TIMEOUT_SETTING.get(settings).millis());
 
         return clientConfiguration;
     }
 
     protected static String findEndpoint(Logger logger, Settings settings) {
         String endpoint = null;
-        if (CLOUD_EC2.ENDPOINT_SETTING.exists(settings)) {
-            endpoint = CLOUD_EC2.ENDPOINT_SETTING.get(settings);
+        if (DISCOVERY_EC2.ENDPOINT_SETTING.exists(settings) || CLOUD_EC2.ENDPOINT_SETTING.exists(settings)) {
+            endpoint = DISCOVERY_EC2.ENDPOINT_SETTING.get(settings);
             logger.debug("using explicit ec2 endpoint [{}]", endpoint);
         } else if (REGION_SETTING.exists(settings) || CLOUD_EC2.REGION_SETTING.exists(settings)) {
             final String region = CLOUD_EC2.REGION_SETTING.get(settings);