Provide an Option to Use Path-Style-Access with S3 Repo

* As discussed, added the option to use path style access back again and
deprecated it.
* Defaulted to `false`
* Added warning to docs

* Closes elastic#41816

Author: original-brownbear

docs/plugins/repository-s3.asciidoc

@@ -145,6 +145,17 @@ settings belong in the `elasticsearch.yml` file.
     Whether retries should be throttled (i.e. should back off). Must be `true`
     or `false`. Defaults to `true`.
 
+`path_style_access`::
+
+   Whether to use the path style access pattern. Defaults to `false`.
+
+NOTE: This setting is deprecated and only intended as a stop-gap solution to provide
+   compatibility with alternative S3 implementations that do not provide compatibility with the domain style access pattern.
+   AWS S3 will stop supporting the path style access pattern from
+   https://forums.aws.amazon.com/ann.jspa?annID=6776[September 30th, 2020]. Releases of Elasticsearch will likely stop supporting
+   this setting before that data. Any setups using the path style access pattern should be switched to the domain style access pattern
+   as soon as possible to ensure continued compatibility with the S3 repository plugin.
+
 [float]
 [[repository-s3-compatible-services]]
 ===== S3-compatible services
@@ -381,10 +392,6 @@ bucket, in this example, named "foo".
 The bucket needs to exist to register a repository for snapshots. If you did not
 create the bucket then the repository registration will fail.
 
-Note: Starting in version 7.0, all bucket operations are using the path style
-access pattern. In previous versions the decision to use virtual hosted style or
-path style access was made by the AWS Java SDK.
-
 [[repository-s3-aws-vpc]]
 [float]
 ==== AWS VPC Bandwidth Settings

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java

@@ -95,6 +95,10 @@ final class S3ClientSettings {
     static final Setting.AffixSetting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "use_throttle_retries",
         key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope));
 
+    /** Whether the s3 client should use path style access. */
+    static final Setting.AffixSetting<Boolean> USE_PATH_STYLE_ACCESS = Setting.affixKeySetting(PREFIX, "path_style_access",
+        key -> Setting.boolSetting(key, false, Property.NodeScope, Property.Deprecated));
+
     /** Credentials to authenticate with s3. */
     final S3BasicCredentials credentials;
 
@@ -127,9 +131,13 @@ final class S3ClientSettings {
     /** Whether the s3 client should use an exponential backoff retry policy. */
     final boolean throttleRetries;
 
+    /** Whether the s3 client should use path style access. */
+    final boolean pathStyleAccess;
+
     private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol,
                              String proxyHost, int proxyPort, String proxyUsername, String proxyPassword,
-                             int readTimeoutMillis, int maxRetries, boolean throttleRetries) {
+                             int readTimeoutMillis, int maxRetries, boolean throttleRetries,
+                             boolean pathStyleAccess) {
         this.credentials = credentials;
         this.endpoint = endpoint;
         this.protocol = protocol;
@@ -140,6 +148,7 @@ private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protoc
         this.readTimeoutMillis = readTimeoutMillis;
         this.maxRetries = maxRetries;
         this.throttleRetries = throttleRetries;
+        this.pathStyleAccess = pathStyleAccess;
     }
 
     /**
@@ -162,6 +171,7 @@ S3ClientSettings refine(RepositoryMetaData metadata) {
             getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis());
         final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries);
         final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries);
+        final boolean usePathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess);
         final S3BasicCredentials newCredentials;
         if (checkDeprecatedCredentials(repoSettings)) {
             newCredentials = loadDeprecatedCredentials(repoSettings);
@@ -183,7 +193,8 @@ S3ClientSettings refine(RepositoryMetaData metadata) {
             proxyPassword,
             newReadTimeoutMillis,
             newMaxRetries,
-            newThrottleRetries
+            newThrottleRetries,
+            usePathStyleAccess
         );
     }
 
@@ -270,7 +281,8 @@ static S3ClientSettings getClientSettings(final Settings settings, final String
                 proxyPassword.toString(),
                 Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()),
                 getConfigValue(settings, clientName, MAX_RETRIES_SETTING),
-                getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING)
+                getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING),
+                getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS)
             );
         }
     }

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java

@@ -105,6 +105,7 @@ public List<Setting<?>> getSettings() {
             S3ClientSettings.READ_TIMEOUT_SETTING,
             S3ClientSettings.MAX_RETRIES_SETTING,
             S3ClientSettings.USE_THROTTLE_RETRIES_SETTING,
+            S3ClientSettings.USE_PATH_STYLE_ACCESS,
             S3Repository.ACCESS_KEY_SETTING,
             S3Repository.SECRET_KEY_SETTING);
     }

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -153,8 +153,10 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) {
         //
         // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too)
         // so this change removes that usage of a deprecated API.
-        builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null))
-            .enablePathStyleAccess();
+        builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
+        if (clientSettings.pathStyleAccess) {
+            builder.enablePathStyleAccess();
+        }
 
         return builder.build();
     }