Differentiate service account tokens in GCS tests (#48382)

This commit changes the test so that each node use a specific 
service account and private key. It also changes how unique 
request ids are generated for refresh token request using the 
token itself, so that error count will be specific per node (each 
node should execute a single refresh token request as tokens 
are valid for 1 hour).

Author: tlrx

plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java

@@ -78,8 +78,6 @@
 @SuppressForbidden(reason = "this test uses a HttpServer to emulate a Google Cloud Storage endpoint")
 public class GoogleCloudStorageBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTestCase {
 
-    private static byte[] serviceAccount;
-
     @Override
     protected String repositoryType() {
         return GoogleCloudStorageRepository.TYPE;
@@ -114,16 +112,13 @@ protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        if (serviceAccount == null) {
-            serviceAccount = TestUtils.createServiceAccount(random());
-        }
-
         final Settings.Builder settings = Settings.builder();
         settings.put(super.nodeSettings(nodeOrdinal));
         settings.put(ENDPOINT_SETTING.getConcreteSettingForNamespace("test").getKey(), httpServerUrl());
         settings.put(TOKEN_URI_SETTING.getConcreteSettingForNamespace("test").getKey(), httpServerUrl() + "/token");
 
         final MockSecureSettings secureSettings = new MockSecureSettings();
+        final byte[] serviceAccount = TestUtils.createServiceAccount(random());
         secureSettings.setFile(CREDENTIALS_FILE_SETTING.getConcreteSettingForNamespace("test").getKey(), serviceAccount);
         settings.setSecureSettings(secureSettings);
         return settings.build();
@@ -410,6 +405,15 @@ private static class GoogleErroneousHttpHandler extends ErroneousHttpHandler {
 
         @Override
         protected String requestUniqueId(HttpExchange exchange) {
+            if ("/token".equals(exchange.getRequestURI().getPath())) {
+                try {
+                    // token content is unique per node (not per request)
+                    return Streams.readFully(exchange.getRequestBody()).utf8ToString();
+                } catch (IOException e) {
+                    throw new AssertionError("Unable to read token request body", e);
+                }
+            }
+
             final String range = exchange.getRequestHeaders().getFirst("Content-Range");
             return exchange.getRemoteAddress().toString()
                 + " " + exchange.getRequestMethod()