Settings: Migrate ec2 discovery sensitive settings to elasticsearch keystore (#23961)

This change adds secure settings for access/secret keys and proxy
username/password to ec2 discovery.  It adds the new settings with the
prefix `discovery.ec2`, copies other relevant ec2 client settings to the
same prefix, and deprecates all other settings (`cloud.aws.*` and
`cloud.aws.ec2.*`).  Note that this is simpler than the client configs
in repository-s3 because discovery is only initialized once for the
entire node, so there is no reason to complicate the configuration with
the ability to have multiple sets of client settings.

relates #22475

Author: rjernst

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2Service.java

@@ -22,6 +22,8 @@
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import com.amazonaws.services.ec2.AmazonEC2;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -42,50 +44,52 @@ interface AwsEc2Service {
     /**
      * cloud.aws.access_key: AWS Access key. Shared with repository-s3 plugin
      */
-    Setting<String> KEY_SETTING =
-        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered, Property.Shared);
+    Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.access_key", "", SecureString::new,
+            Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.secret_key: AWS Secret key. Shared with repository-s3 plugin
      */
-    Setting<String> SECRET_SETTING =
-        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered, Property.Shared);
+    Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.secret_key", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with repository-s3 plugin
      */
     Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-        Property.NodeScope, Property.Shared);
+        Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared);
+    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host",
+        Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with repository-s3 plugin
      */
-    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope,
-        Property.Shared);
+    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16,
+        Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope, Property.Shared);
+    Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.proxy.username", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_PASSWORD_SETTING =
-        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered, Property.Shared);
+    Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.proxy.password", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with repository-s3 plugin
      */
-    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared);
+    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.region: Region. Shared with repository-s3 plugin
      */
     Setting<String> REGION_SETTING =
-        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared);
+        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared, Property.Deprecated);
     /**
      * cloud.aws.read_timeout: Socket read timeout. Shared with repository-s3 plugin
      */
     Setting<TimeValue> READ_TIMEOUT = Setting.timeSetting("cloud.aws.read_timeout",
-        TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Shared);
+        TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Shared, Property.Deprecated);
 
     /**
      * Defines specific ec2 settings starting with cloud.aws.ec2.
@@ -95,69 +99,70 @@ interface CLOUD_EC2 {
          * cloud.aws.ec2.access_key: AWS Access key specific for EC2 API calls. Defaults to cloud.aws.access_key.
          * @see AwsEc2Service#KEY_SETTING
          */
-        Setting<String> KEY_SETTING = new Setting<>("cloud.aws.ec2.access_key", AwsEc2Service.KEY_SETTING, Function.identity(),
-            Property.NodeScope, Property.Filtered);
+        Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.ec2.access_key", AwsEc2Service.KEY_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
+
         /**
          * cloud.aws.ec2.secret_key: AWS Secret key specific for EC2 API calls. Defaults to cloud.aws.secret_key.
          * @see AwsEc2Service#SECRET_SETTING
          */
-        Setting<String> SECRET_SETTING = new Setting<>("cloud.aws.ec2.secret_key", AwsEc2Service.SECRET_SETTING, Function.identity(),
-            Property.NodeScope, Property.Filtered);
+        Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.ec2.secret_key", AwsEc2Service.SECRET_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
         /**
          * cloud.aws.ec2.protocol: Protocol for AWS API specific for EC2 API calls: http or https.  Defaults to cloud.aws.protocol.
          * @see AwsEc2Service#PROTOCOL_SETTING
          */
         Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.ec2.protocol", AwsEc2Service.PROTOCOL_SETTING,
-            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
+            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.proxy.host: In case of proxy, define its hostname/IP specific for EC2 API calls. Defaults to cloud.aws.proxy.host.
          * @see AwsEc2Service#PROXY_HOST_SETTING
          */
         Setting<String> PROXY_HOST_SETTING = new Setting<>("cloud.aws.ec2.proxy.host", AwsEc2Service.PROXY_HOST_SETTING,
-            Function.identity(), Property.NodeScope);
+            Function.identity(), Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.proxy.port: In case of proxy, define its port specific for EC2 API calls.  Defaults to cloud.aws.proxy.port.
          * @see AwsEc2Service#PROXY_PORT_SETTING
          */
         Setting<Integer> PROXY_PORT_SETTING = new Setting<>("cloud.aws.ec2.proxy.port", AwsEc2Service.PROXY_PORT_SETTING,
-            s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.ec2.proxy.port"), Property.NodeScope);
+            s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.ec2.proxy.port"), Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.proxy.username: In case of proxy with auth, define the username specific for EC2 API calls.
          * Defaults to cloud.aws.proxy.username.
          * @see AwsEc2Service#PROXY_USERNAME_SETTING
          */
-        Setting<String> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.ec2.proxy.username", AwsEc2Service.PROXY_USERNAME_SETTING,
-            Function.identity(), Property.NodeScope);
+        Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.ec2.proxy.username", AwsEc2Service.PROXY_USERNAME_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
         /**
          * cloud.aws.ec2.proxy.password: In case of proxy with auth, define the password specific for EC2 API calls.
          * Defaults to cloud.aws.proxy.password.
          * @see AwsEc2Service#PROXY_PASSWORD_SETTING
          */
-        Setting<String> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.ec2.proxy.password", AwsEc2Service.PROXY_PASSWORD_SETTING,
-            Function.identity(), Property.NodeScope, Property.Filtered);
+        Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.ec2.proxy.password", AwsEc2Service.PROXY_PASSWORD_SETTING,
+            SecureString::new, Property.NodeScope, Property.Filtered, Property.Deprecated);
         /**
          * cloud.aws.ec2.signer: If you are using an old AWS API version, you can define a Signer. Specific for EC2 API calls.
          * Defaults to cloud.aws.signer.
          * @see AwsEc2Service#SIGNER_SETTING
          */
         Setting<String> SIGNER_SETTING = new Setting<>("cloud.aws.ec2.signer", AwsEc2Service.SIGNER_SETTING, Function.identity(),
-            Property.NodeScope);
+            Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.region: Region specific for EC2 API calls. Defaults to cloud.aws.region.
          * @see AwsEc2Service#REGION_SETTING
          */
         Setting<String> REGION_SETTING = new Setting<>("cloud.aws.ec2.region", AwsEc2Service.REGION_SETTING,
-            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
+            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
          */
-        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.ec2.endpoint", Property.NodeScope);
+        Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.ec2.endpoint", Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.ec2.read_timeout: Socket read timeout. Defaults to cloud.aws.read_timeout
          * @see AwsEc2Service#READ_TIMEOUT
          */
         Setting<TimeValue> READ_TIMEOUT =
-            Setting.timeSetting("cloud.aws.ec2.read_timeout", AwsEc2Service.READ_TIMEOUT, Property.NodeScope);
+            Setting.timeSetting("cloud.aws.ec2.read_timeout", AwsEc2Service.READ_TIMEOUT, Property.NodeScope, Property.Deprecated);
     }
 
     /**
@@ -172,6 +177,40 @@ class HostType {
             public static final String TAG_PREFIX = "tag:";
         }
 
+        /** The access key (ie login id) for connecting to ec2. */
+        Setting<SecureString> ACCESS_KEY_SETTING = SecureSetting.secureString("discovery.ec2.access_key", CLOUD_EC2.KEY_SETTING, false);
+
+        /** The secret key (ie password) for connecting to ec2. */
+        Setting<SecureString> SECRET_KEY_SETTING = SecureSetting.secureString("discovery.ec2.secret_key", CLOUD_EC2.SECRET_SETTING, false);
+
+        /** An override for the ec2 endpoint to connect to. */
+        Setting<String> ENDPOINT_SETTING = new Setting<>("discovery.ec2.endpoint", CLOUD_EC2.ENDPOINT_SETTING,
+            s -> s.toLowerCase(Locale.ROOT), Setting.Property.NodeScope);
+
+        /** The protocol to use to connect to to ec2. */
+        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("discovery.ec2.protocol", CLOUD_EC2.PROTOCOL_SETTING,
+            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Setting.Property.NodeScope);
+
+        /** The host name of a proxy to connect to ec2 through. */
+        Setting<String> PROXY_HOST_SETTING = new Setting<>("discovery.ec2.proxy.host", CLOUD_EC2.PROXY_HOST_SETTING,
+            Function.identity(), Setting.Property.NodeScope);
+
+        /** The port of a proxy to connect to ec2 through. */
+        Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("discovery.ec2.proxy.port", CLOUD_EC2.PROXY_PORT_SETTING,
+            0, Setting.Property.NodeScope);
+
+        /** The username of a proxy to connect to s3 through. */
+        Setting<SecureString> PROXY_USERNAME_SETTING = SecureSetting.secureString("discovery.ec2.proxy.username",
+            CLOUD_EC2.PROXY_USERNAME_SETTING, false);
+
+        /** The password of a proxy to connect to s3 through. */
+        Setting<SecureString> PROXY_PASSWORD_SETTING =  SecureSetting.secureString("discovery.ec2.proxy.password",
+            CLOUD_EC2.PROXY_PASSWORD_SETTING, false);
+
+        /** The socket timeout for connecting to s3. */
+        Setting<TimeValue> READ_TIMEOUT_SETTING = Setting.timeSetting("discovery.ec2.read_timeout",
+            CLOUD_EC2.READ_TIMEOUT, Setting.Property.NodeScope);
+
         /**
          * discovery.ec2.host_type: The type of host type to use to communicate with other instances.
          * Can be one of private_ip, public_ip, private_dns, public_dns or tag:XXXX where

plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 
 class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service, Closeable {
@@ -68,14 +69,15 @@ public synchronized AmazonEC2 client() {
     protected static AWSCredentialsProvider buildCredentials(Logger logger, Settings settings) {
         AWSCredentialsProvider credentials;
 
-        String key = CLOUD_EC2.KEY_SETTING.get(settings);
-        String secret = CLOUD_EC2.SECRET_SETTING.get(settings);
-        if (key.isEmpty() && secret.isEmpty()) {
-            logger.debug("Using either environment variables, system properties or instance profile credentials");
-            credentials = new DefaultAWSCredentialsProviderChain();
-        } else {
-            logger.debug("Using basic key/secret credentials");
-            credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key, secret));
+        try (SecureString key = DISCOVERY_EC2.ACCESS_KEY_SETTING.get(settings);
+             SecureString secret = DISCOVERY_EC2.SECRET_KEY_SETTING.get(settings)) {
+            if (key.length() == 0 && secret.length() == 0) {
+                logger.debug("Using either environment variables, system properties or instance profile credentials");
+                credentials = new DefaultAWSCredentialsProviderChain();
+            } else {
+                logger.debug("Using basic key/secret credentials");
+                credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key.toString(), secret.toString()));
+            }
         }
 
         return credentials;
@@ -86,19 +88,20 @@ protected static ClientConfiguration buildConfiguration(Logger logger, Settings
         // the response metadata cache is only there for diagnostics purposes,
         // but can force objects from every response to the old generation.
         clientConfiguration.setResponseMetadataCacheSize(0);
-        clientConfiguration.setProtocol(CLOUD_EC2.PROTOCOL_SETTING.get(settings));
-
-        if (PROXY_HOST_SETTING.exists(settings) || CLOUD_EC2.PROXY_HOST_SETTING.exists(settings)) {
-            String proxyHost = CLOUD_EC2.PROXY_HOST_SETTING.get(settings);
-            Integer proxyPort = CLOUD_EC2.PROXY_PORT_SETTING.get(settings);
-            String proxyUsername = CLOUD_EC2.PROXY_USERNAME_SETTING.get(settings);
-            String proxyPassword = CLOUD_EC2.PROXY_PASSWORD_SETTING.get(settings);
-
-            clientConfiguration
-                .withProxyHost(proxyHost)
-                .withProxyPort(proxyPort)
-                .withProxyUsername(proxyUsername)
-                .withProxyPassword(proxyPassword);
+        clientConfiguration.setProtocol(DISCOVERY_EC2.PROTOCOL_SETTING.get(settings));
+
+        if (PROXY_HOST_SETTING.exists(settings) || DISCOVERY_EC2.PROXY_HOST_SETTING.exists(settings)) {
+            String proxyHost = DISCOVERY_EC2.PROXY_HOST_SETTING.get(settings);
+            Integer proxyPort = DISCOVERY_EC2.PROXY_PORT_SETTING.get(settings);
+            try (SecureString proxyUsername = DISCOVERY_EC2.PROXY_USERNAME_SETTING.get(settings);
+                 SecureString proxyPassword = DISCOVERY_EC2.PROXY_PASSWORD_SETTING.get(settings)) {
+
+                clientConfiguration
+                    .withProxyHost(proxyHost)
+                    .withProxyPort(proxyPort)
+                    .withProxyUsername(proxyUsername.toString())
+                    .withProxyPassword(proxyPassword.toString());
+            }
         }
 
         // #155: we might have 3rd party users using older EC2 API version
@@ -125,15 +128,15 @@ public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest,
             10,
             false);
         clientConfiguration.setRetryPolicy(retryPolicy);
-        clientConfiguration.setSocketTimeout((int) CLOUD_EC2.READ_TIMEOUT.get(settings).millis());
+        clientConfiguration.setSocketTimeout((int) DISCOVERY_EC2.READ_TIMEOUT_SETTING.get(settings).millis());
 
         return clientConfiguration;
     }
 
     protected static String findEndpoint(Logger logger, Settings settings) {
         String endpoint = null;
-        if (CLOUD_EC2.ENDPOINT_SETTING.exists(settings)) {
-            endpoint = CLOUD_EC2.ENDPOINT_SETTING.get(settings);
+        if (DISCOVERY_EC2.ENDPOINT_SETTING.exists(settings) || CLOUD_EC2.ENDPOINT_SETTING.exists(settings)) {
+            endpoint = DISCOVERY_EC2.ENDPOINT_SETTING.get(settings);
             logger.debug("using explicit ec2 endpoint [{}]", endpoint);
         } else if (REGION_SETTING.exists(settings) || CLOUD_EC2.REGION_SETTING.exists(settings)) {
             final String region = CLOUD_EC2.REGION_SETTING.get(settings);