Support fixture in repository-gcs:thirdPartyTest and fix GCS fixture (#44885)

It turns out that today :plugins:repository-gcs:thirdPartyTest can only
run against real GCS.
Moreover, thirdPartyTest is not a part of check, so these tests are not
running on intake build.
This commit addresses the issue and makes repository-gcs:thirdPartyTest
work with both fixture and real GCS.
To do that, except adjusting build and test itself, I had to make
changes to the fixture, because previously it was ignoring
BlobListOption.currentDirectory() in the list call.

Author: Andrey Ershov

plugins/repository-gcs/build.gradle

@@ -125,21 +125,49 @@ check {
   dependsOn 'qa:google-cloud-storage:check'
 }
 
+test {
+  exclude '**/GoogleCloudStorageThirdPartyTests.class'
+}
+
+task thirdPartyTest(type: Test) {
+}
+
 String gcsServiceAccount = System.getenv("google_storage_service_account")
 String gcsBucket = System.getenv("google_storage_bucket")
 String gcsBasePath = System.getenv("google_storage_base_path")
 
-test {
-  exclude '**/GoogleCloudStorageThirdPartyTests.class'
+if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
+  thirdPartyTest.dependsOn('qa:google-cloud-storage:createServiceAccountFile', 'qa:google-cloud-storage:googleCloudStorageFixture')
+  thirdPartyTest.finalizedBy('qa:google-cloud-storage:googleCloudStorageFixture#stop')
+  gcsServiceAccount = new File(project('qa:google-cloud-storage').buildDir, 'generated-resources/service_account_test.json').path
+  gcsBucket = 'bucket_test'
+  gcsBasePath = 'integration_test'
+
+  def fixtureEndpoint = {
+    "http://${project('qa:google-cloud-storage').googleCloudStorageFixture.addressAndPort}"
+  }
+
+  def tokenURI = {
+    "http://${project('qa:google-cloud-storage').googleCloudStorageFixture.addressAndPort}/o/oauth2/token"
+  }
+  
+  thirdPartyTest {
+    nonInputProperties.systemProperty 'test.google.endpoint', "${ -> fixtureEndpoint.call() }"
+    nonInputProperties.systemProperty 'test.google.tokenURI', "${ -> tokenURI.call() }"
+  }
+} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
+  throw new IllegalArgumentException("not all options specified to run thirdPartyTest against external GCS service are present")
 }
 
-task thirdPartyTest(type: Test) {
+def encodedCredentials = {
+  Base64.encoder.encodeToString(Files.readAllBytes(file(gcsServiceAccount).toPath()))
+}
+
+thirdPartyTest {
   include '**/GoogleCloudStorageThirdPartyTests.class'
-  systemProperty 'test.google.account', gcsServiceAccount ? Base64.encoder.encodeToString(Files.readAllBytes(file(gcsServiceAccount).toPath())) : ""
-  systemProperty 'test.google.bucket', gcsBucket ? gcsBucket : ""
-  systemProperty 'test.google.base', gcsBasePath ? gcsBasePath : "/"
+  systemProperty 'test.google.bucket', gcsBucket
+  systemProperty 'test.google.base', gcsBasePath
+  nonInputProperties.systemProperty 'test.google.account', "${ -> encodedCredentials.call() }"
 }
 
-if (gcsServiceAccount || gcsBucket || gcsBasePath) {
-  check.dependsOn(thirdPartyTest)
-}
+check.dependsOn(thirdPartyTest)

plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java

@@ -38,9 +38,11 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
@@ -343,18 +345,40 @@ private static PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket
             final XContentBuilder builder = jsonBuilder();
             builder.startObject();
             builder.field("kind", "storage#objects");
+            final Set<String> prefixes = new HashSet<>();
             {
                 builder.startArray("items");
 
                 final String prefixParam = request.getParam("prefix");
+                final String delimiter = request.getParam("delimiter");
+
                 for (final Map.Entry<String, byte[]> object : bucket.objects.entrySet()) {
-                    if ((prefixParam != null) && (object.getKey().startsWith(prefixParam) == false)) {
+                    String objectKey = object.getKey();
+                    if ((prefixParam != null) && (objectKey.startsWith(prefixParam) == false)) {
                         continue;
                     }
-                    buildObjectResource(builder, bucket.name, object.getKey(), object.getValue());
+
+                    if (Strings.isNullOrEmpty(delimiter)) {
+                        buildObjectResource(builder, bucket.name, objectKey, object.getValue());
+                    } else {
+                        int prefixLength = prefixParam.length();
+                        String rest = objectKey.substring(prefixLength);
+                        int delimiterPos;
+                        if ((delimiterPos = rest.indexOf(delimiter)) != -1) {
+                            String key = objectKey.substring(0, prefixLength + delimiterPos + 1);
+                            prefixes.add(key);
+                        } else {
+                            buildObjectResource(builder, bucket.name, objectKey, object.getValue());
+                        }
+                    }
                 }
                 builder.endArray();
             }
+            {
+                if (prefixes.isEmpty() == false) {
+                    builder.array("prefixes", prefixes.toArray());
+                }
+            }
             builder.endObject();
             return newResponse(RestStatus.OK, emptyMap(), builder);
         });

plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.repositories.gcs;
 
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -40,6 +41,21 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(GoogleCloudStoragePlugin.class);
     }
 
+    @Override
+    protected Settings nodeSettings() {
+        Settings.Builder builder = Settings.builder().put(super.nodeSettings());
+
+        if (Strings.isNullOrEmpty(System.getProperty("test.google.endpoint")) == false) {
+            builder.put("gcs.client.default.endpoint", System.getProperty("test.google.endpoint"));
+        }
+
+        if (Strings.isNullOrEmpty(System.getProperty("test.google.tokenURI")) == false) {
+            builder.put("gcs.client.default.token_uri", System.getProperty("test.google.tokenURI"));
+        }
+
+        return builder.build();
+    }
+
     @Override
     protected SecureSettings credentials() {
         assertThat(System.getProperty("test.google.account"), not(blankOrNullString()));