Add docker-composed based test fixture for GCS (#48903)

Similarly to what has be done for Azure in #48636, this commit
adds a new :test:fixtures:gcs-fixture project which provides two
docker-compose based fixtures that emulate a Google Cloud
Storage service.

Some code has been extracted from existing tests and placed
into this new project so that it can be easily reused in other
projects.

Author: tlrx

plugins/repository-gcs/build.gradle

@@ -54,6 +54,8 @@ dependencies {
   compile 'io.opencensus:opencensus-api:0.18.0'
   compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
   compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'
+
+  testCompile project(':test:fixtures:gcs-fixture')
 }
 
 dependencyLicenses {

plugins/repository-gcs/qa/google-cloud-storage/build.gradle

@@ -19,7 +19,6 @@
 
 
 import org.elasticsearch.gradle.MavenFilteringHack
-import org.elasticsearch.gradle.test.AntFixture
 
 import java.security.KeyPair
 import java.security.KeyPairGenerator
@@ -28,7 +27,14 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.test.fixtures'
 
+// TODO think about flattening qa:google-cloud-storage project into parent
+dependencies {
+    testCompile project(path: ':plugins:repository-gcs')
+}
+
+testFixtures.useFixture(':test:fixtures:gcs-fixture')
 boolean useFixture = false
 
 String gcsServiceAccount = System.getenv("google_storage_service_account")
@@ -38,7 +44,7 @@ String gcsBasePath = System.getenv("google_storage_base_path")
 File serviceAccountFile = null
 if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
     serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
-    gcsBucket = 'bucket_test'
+    gcsBucket = 'bucket'
     gcsBasePath = 'integration_test'
     useFixture = true
 } else {
@@ -48,12 +54,11 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
     }
 }
 
-/** A task to start the GoogleCloudStorageFixture which emulates a Google Cloud Storage service **/
-task googleCloudStorageFixture(type: AntFixture) {
-    dependsOn testClasses
-    env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
-    executable = new File(project.runtimeJavaHome, 'bin/java')
-    args 'org.elasticsearch.repositories.gcs.GoogleCloudStorageFixture', baseDir, 'bucket_test'
+def fixtureAddress = { fixture ->
+    assert useFixture : 'closure should not be used without a fixture'
+    int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
+    assert ephemeralPort > 0
+    'http://127.0.0.1:' + ephemeralPort
 }
 
 /** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
@@ -96,10 +101,10 @@ testClusters.integTest {
     keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
 
     if (useFixture) {
-        tasks.integTest.dependsOn createServiceAccountFile, googleCloudStorageFixture
+        tasks.integTest.dependsOn createServiceAccountFile
         /* Use a closure on the string to delay evaluation until tests are executed */
-        setting 'gcs.client.integration_test.endpoint', { "http://${googleCloudStorageFixture.addressAndPort}" }, IGNORE_VALUE
-        setting 'gcs.client.integration_test.token_uri', { "http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token" }, IGNORE_VALUE
+        setting 'gcs.client.integration_test.endpoint', { "${ -> fixtureAddress('gcs-fixture') }" }, IGNORE_VALUE
+        setting 'gcs.client.integration_test.token_uri', { "${ -> fixtureAddress('gcs-fixture') }/o/oauth2/token" }, IGNORE_VALUE
     } else {
         println "Using an external service to test the repository-gcs plugin"
     }

plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java

@@ -24,6 +24,7 @@
 import com.google.cloud.storage.StorageOptions;
 import com.sun.net.httpserver.HttpContext;
 import com.sun.net.httpserver.HttpServer;
+import fixture.gcs.FakeOAuth2HttpHandler;
 import org.apache.http.HttpStatus;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
@@ -68,6 +69,10 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeEnd;
+import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeLimit;
+import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeStart;
+import static fixture.gcs.GoogleCloudStorageHttpHandler.parseMultipartRequestBody;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING;
@@ -144,13 +149,7 @@ StorageOptions createStorageOptions(final GoogleCloudStorageClientSettings clien
 
         final List<HttpContext> httpContexts = Arrays.asList(
             // Auth
-            httpServer.createContext("/token", exchange -> {
-                byte[] response = ("{\"access_token\":\"foo\",\"token_type\":\"Bearer\",\"expires_in\":3600}").getBytes(UTF_8);
-                exchange.getResponseHeaders().add("Content-Type", "application/json");
-                exchange.sendResponseHeaders(HttpStatus.SC_OK, response.length);
-                exchange.getResponseBody().write(response);
-                exchange.close();
-            }),
+            httpServer.createContext("/token", new FakeOAuth2HttpHandler()),
             // Does bucket exists?
             httpServer.createContext("/storage/v1/b/bucket", exchange -> {
                 byte[] response = ("{\"kind\":\"storage#bucket\",\"name\":\"bucket\",\"id\":\"0\"}").getBytes(UTF_8);
@@ -244,7 +243,7 @@ public void testWriteBlobWithRetries() throws Exception {
         httpServer.createContext("/upload/storage/v1/b/bucket/o", exchange -> {
             assertThat(exchange.getRequestURI().getQuery(), containsString("uploadType=multipart"));
             if (countDown.countDown()) {
-                Optional<Tuple<String, BytesArray>> content = TestUtils.parseMultipartRequestBody(exchange.getRequestBody());
+                Optional<Tuple<String, BytesArray>> content = parseMultipartRequestBody(exchange.getRequestBody());
                 assertThat(content.isPresent(), is(true));
                 assertThat(content.get().v1(), equalTo("write_blob_max_retries"));
                 if (Objects.deepEquals(bytes, content.get().v2().array())) {
@@ -387,12 +386,12 @@ public void testWriteLargeBlob() throws IOException {
                     final long bytesRead = Streams.copy(exchange.getRequestBody(), requestBody);
                     assertThat(Math.toIntExact(bytesRead), anyOf(equalTo(defaultChunkSize), equalTo(lastChunkSize)));
 
-                    final int rangeStart = TestUtils.getContentRangeStart(range);
-                    final int rangeEnd = TestUtils.getContentRangeEnd(range);
+                    final int rangeStart = getContentRangeStart(range);
+                    final int rangeEnd = getContentRangeEnd(range);
                     assertThat(rangeEnd + 1 - rangeStart, equalTo(Math.toIntExact(bytesRead)));
                     assertArrayEquals(Arrays.copyOfRange(data, rangeStart, rangeEnd + 1), requestBody.toByteArray());
 
-                    final Integer limit = TestUtils.getContentRangeLimit(range);
+                    final Integer limit = getContentRangeLimit(range);
                     if (limit != null) {
                         exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1);
                         exchange.close();