Log unsuccessful attempts to get credentials from web identity tokens

docs/changelog/88241.yaml

@@ -0,0 +1,6 @@
+pr: 88241
+summary: Log unsuccessful attempts to get credentials from web identity tokens as
+  warnings
+area: Allocation
+type: enhancement
+issues: []

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -9,9 +9,9 @@
 package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.ClientConfiguration;
+import com.amazonaws.SdkClientException;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSCredentialsProviderChain;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.AnonymousAWSCredentials;
 import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
@@ -39,6 +39,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Clock;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -223,7 +224,9 @@ static AWSCredentialsProvider buildCredentials(
             if (webIdentityTokenCredentialsProvider.isActive()) {
                 logger.debug("Using a custom provider chain of Web Identity Token and instance profile credentials");
                 return new PrivilegedAWSCredentialsProvider(
-                    new AWSCredentialsProviderChain(webIdentityTokenCredentialsProvider, new EC2ContainerCredentialsProviderWrapper())
+                    new CustomAWSCredentialsProviderChain(
+                        List.of(webIdentityTokenCredentialsProvider, new EC2ContainerCredentialsProviderWrapper())
+                    )
                 );
             } else {
                 logger.debug("Using instance profile credentials");
@@ -375,6 +378,45 @@ public void shutdown() throws IOException {
         }
     }
 
+    /**
+     * Customized {@link  com.amazonaws.auth.AWSCredentialsProviderChain} that logs
+     * unsuccessful authentication attempts as warnings instead of debug messages
+     */
+    static class CustomAWSCredentialsProviderChain implements AWSCredentialsProvider {
+
+        private final List<AWSCredentialsProvider> credentialsProviders;
+        private volatile AWSCredentialsProvider lastUsedProvider;
+
+        CustomAWSCredentialsProviderChain(List<AWSCredentialsProvider> credentialsProviders) {
+            this.credentialsProviders = List.copyOf(credentialsProviders);
+        }
+
+        @Override
+        public AWSCredentials getCredentials() {
+            if (lastUsedProvider != null) {
+                return lastUsedProvider.getCredentials();
+            }
+
+            for (AWSCredentialsProvider provider : credentialsProviders) {
+                try {
+                    AWSCredentials credentials = provider.getCredentials();
+                    if (credentials.getAWSAccessKeyId() != null && credentials.getAWSSecretKey() != null) {
+                        lastUsedProvider = provider;
+                        return credentials;
+                    }
+                } catch (Exception e) {
+                    LOGGER.warn("Unable to load credentials from " + provider, e);
+                }
+            }
+            throw new SdkClientException("Unable to load AWS credentials from any provider in the chain");
+        }
+
+        @Override
+        public void refresh() {
+            credentialsProviders.forEach(AWSCredentialsProvider::refresh);
+        }
+    }
+
     @FunctionalInterface
     interface SystemEnvironment {
         String getEnv(String name);

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java

@@ -10,24 +10,27 @@
 
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
+import com.amazonaws.SdkClientException;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSCredentialsProviderChain;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
 
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Assert;
 import org.mockito.Mockito;
 
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
 
 public class AwsS3ServiceImplTests extends ESTestCase {
 
@@ -60,12 +63,48 @@ public void testSupportsWebIdentityTokenCredentials() {
         );
         assertThat(credentialsProvider, instanceOf(S3Service.PrivilegedAWSCredentialsProvider.class));
         var privilegedAWSCredentialsProvider = (S3Service.PrivilegedAWSCredentialsProvider) credentialsProvider;
-        assertThat(privilegedAWSCredentialsProvider.getCredentialsProvider(), instanceOf(AWSCredentialsProviderChain.class));
+        assertThat(
+            privilegedAWSCredentialsProvider.getCredentialsProvider(),
+            instanceOf(S3Service.CustomAWSCredentialsProviderChain.class)
+        );
         AWSCredentials credentials = privilegedAWSCredentialsProvider.getCredentials();
         assertEquals("sts_access_key_id", credentials.getAWSAccessKeyId());
         assertEquals("sts_secret_key", credentials.getAWSSecretKey());
     }
 
+    public void testCustomCredentialsProviderChainThrowsSdkErrorIfUnableToLoadCredentials() {
+        AWSCredentialsProvider provider1 = Mockito.mock(AWSCredentialsProvider.class);
+        AWSCredentialsProvider provider2 = Mockito.mock(AWSCredentialsProvider.class);
+        Mockito.when(provider1.getCredentials()).thenThrow(new IllegalStateException("credentialProvider1 is unavailable"));
+        Mockito.when(provider2.getCredentials()).thenThrow(new IllegalStateException("credentialProvider2 is unavailable"));
+        var credentialsProvider = new S3Service.CustomAWSCredentialsProviderChain(List.of(provider1, provider2));
+        try {
+            credentialsProvider.getCredentials();
+            Assert.fail("Shouldn't load any credentials");
+        } catch (SdkClientException e) {
+            assertThat(e.getMessage(), startsWith("Unable to load AWS credentials from any provider in the chain"));
+        }
+    }
+
+    public void testCustomCredentialsProviderChainReusesSuccessfulProvider() {
+        AWSCredentialsProvider provider1 = Mockito.mock(AWSCredentialsProvider.class);
+        AWSCredentialsProvider provider2 = Mockito.mock(AWSCredentialsProvider.class);
+        Mockito.when(provider1.getCredentials()).thenThrow(new IllegalStateException("credentialProvider1 is unavailable"));
+        Mockito.when(provider2.getCredentials()).thenReturn(new BasicAWSCredentials("sts_access_key_id", "sts_secret_key"));
+
+        var credentialsProviderChain = new S3Service.CustomAWSCredentialsProviderChain(List.of(provider1, provider2));
+        assertCredentials(credentialsProviderChain.getCredentials(), "sts_access_key_id", "sts_secret_key");
+        assertCredentials(credentialsProviderChain.getCredentials(), "sts_access_key_id", "sts_secret_key");
+
+        Mockito.verify(provider1, Mockito.times(1)).getCredentials();
+        Mockito.verify(provider2, Mockito.times(2)).getCredentials();
+    }
+
+    private void assertCredentials(AWSCredentials credentials, String accessKeyId, String secretKey) {
+        assertEquals(accessKeyId, credentials.getAWSAccessKeyId());
+        assertEquals(secretKey, credentials.getAWSSecretKey());
+    }
+
     public void testAWSCredentialsFromKeystore() {
         final MockSecureSettings secureSettings = new MockSecureSettings();
         final String clientNamePrefix = "some_client_name_";