Refactor GCS test fixture to remove docker dependency (#94755) (#96210)

Author: mark-vieira

modules/repository-gcs/build.gradle

@@ -1,22 +1,19 @@
-import org.apache.tools.ant.filters.ReplaceTokens
-import org.elasticsearch.gradle.internal.info.BuildParams
-import org.elasticsearch.gradle.internal.test.RestIntegTestTask
-import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin
-import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
-
-import java.nio.file.Files
-import java.security.KeyPair
-import java.security.KeyPairGenerator
-
-import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
  * 2.0 and the Server Side Public License, v 1; you may not use this file except
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+
+
+import org.apache.tools.ant.filters.ReplaceTokens
+import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
+
+import java.nio.file.Files
+
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-test-artifact-base'
 
@@ -61,6 +58,7 @@ dependencies {
   testImplementation "org.apache.httpcomponents:httpcore:${versions.httpcore}"
 
   testImplementation project(':test:fixtures:gcs-fixture')
+  yamlRestTestImplementation project(':test:fixtures:gcs-fixture')
 }
 
 testArtifacts {
@@ -202,60 +200,23 @@ tasks.named("thirdPartyAudit").configure {
 }
 
 boolean useFixture = false
-
-def fixtureAddress = { fixture ->
-  assert useFixture: 'closure should not be used without a fixture'
-  int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
-  assert ephemeralPort > 0
-  'http://127.0.0.1:' + ephemeralPort
-}
-
 String gcsServiceAccount = System.getenv("google_storage_service_account")
 String gcsBucket = System.getenv("google_storage_bucket")
 String gcsBasePath = System.getenv("google_storage_base_path")
-File serviceAccountFile = null
+File serviceAccountFile = gcsServiceAccount != null ? new File(gcsServiceAccount) : null
 
 if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
-  serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
   gcsBucket = 'bucket'
   gcsBasePath = 'integration_test'
   useFixture = true
-
-  apply plugin: 'elasticsearch.test.fixtures'
-  testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture')
-  testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party')
-  testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-with-application-default-credentials')
-
 } else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
   throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
-} else {
-  serviceAccountFile = new File(gcsServiceAccount)
 }
 
 def encodedCredentials = {
   Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
 }
 
-/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
-tasks.register("createServiceAccountFile") {
-  doLast {
-    KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
-    keyPairGenerator.initialize(2048)
-    KeyPair keyPair = keyPairGenerator.generateKeyPair()
-    String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
-
-    serviceAccountFile.parentFile.mkdirs()
-    serviceAccountFile.setText("{\n" +
-            '  "type": "service_account",\n' +
-            '  "project_id": "integration_test",\n' +
-            '  "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
-            '  "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
-            '  "client_email": "[email protected]",\n' +
-            '  "client_id": "123456789101112130594"\n' +
-            '}', 'UTF-8')
-  }
-}
-
 Map<String, Object> expansions = [
   'bucket'   : gcsBucket,
   'base_path': gcsBasePath + "_integration_tests"
@@ -271,86 +232,34 @@ tasks.named("internalClusterTest").configure {
   exclude '**/GoogleCloudStorageThirdPartyTests.class'
 }
 
-tasks.named("yamlRestTest").configure {
-  if (useFixture) {
-    dependsOn "createServiceAccountFile"
+tasks.named("yamlRestTest") {
+  systemProperty 'test.google.fixture', Boolean.toString(useFixture)
+  if (useFixture == false) {
+    systemProperty 'test.google.account', serviceAccountFile
+    // We can't run these test in parallel against a real bucket since the tests will step on each other
+    maxParallelForks = 1
   }
 }
 
-/*
- * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
- * an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
- */
-def largeBlobYamlRestTest = tasks.register("largeBlobYamlRestTest", RestIntegTestTask) {
-  if (useFixture) {
-    dependsOn "createServiceAccountFile"
-  }
-  SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-  SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
-  setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
-  setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
-
-  // We have to wait for configure the cluster here as it might not have been created otherwise yet.
-  testClusters {
-    largeBlobYamlRestTest {
-      module tasks.named("explodedBundlePlugin")
-
-      // force large blob uploads by setting the threshold small, forcing this code path to be tested
-      systemProperty 'es.repository_gcs.large_blob_threshold_byte_size', '256'
-    }
-  }
-}
-
-def gcsThirdPartyTest = tasks.register("gcsThirdPartyTest", Test) {
-  SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+def gcsThirdPartyTest = tasks.register("gcsThirdPartyUnitTest", Test) {
+  SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class)
   SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
   setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
   setClasspath(internalTestSourceSet.getRuntimeClasspath())
   include '**/GoogleCloudStorageThirdPartyTests.class'
   systemProperty 'tests.security.manager', false
   systemProperty 'test.google.bucket', gcsBucket
+  systemProperty 'test.google.fixture', Boolean.toString(useFixture)
   nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed
-  nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}"
-  if (useFixture) {
-    dependsOn "createServiceAccountFile"
-    nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}"
-    nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token"
-  }
-}
-
-testClusters.matching {
-  it.name == "yamlRestTest" ||
-  it.name == "largeBlobYamlRestTest" ||
-  it.name == "gcsThirdPartyTest" }.configureEach {
-  keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
-
-  if (useFixture) {
-    /* Use a closure on the string to delay evaluation until tests are executed */
-    setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
-    setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
-  } else {
-    println "Using an external service to test the repository-gcs plugin"
+  if (useFixture == false) {
+    nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}"
   }
 }
 
-
-// Application Default Credentials
-if (useFixture) {
-  tasks.register("yamlRestTestApplicationDefaultCredentials", RestIntegTestTask.class) {
-    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
-    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
-    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
-  }
-  tasks.named("check").configure { dependsOn("yamlRestTestApplicationDefaultCredentials") }
-
-  testClusters.matching { it.name == "yamlRestTestApplicationDefaultCredentials" }.configureEach {
-    setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture-with-application-default-credentials')}" }, IGNORE_VALUE
-    module tasks.named("explodedBundlePlugin")
-    environment 'GCE_METADATA_HOST', { "${-> fixtureAddress('gcs-fixture-with-application-default-credentials')}".replace("http://", "") }, IGNORE_VALUE
-  }
+tasks.register('gcsThirdPartyTest') {
+  dependsOn 'yamlRestTest', gcsThirdPartyTest
 }
 
-tasks.named("check").configure {
-  dependsOn(largeBlobYamlRestTest, gcsThirdPartyTest)
+tasks.named('check') {
+  dependsOn gcsThirdPartyTest
 }

modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java

@@ -8,13 +8,17 @@
 
 package org.elasticsearch.repositories.gcs;
 
+import fixture.gcs.GoogleCloudStorageHttpFixture;
+import fixture.gcs.TestUtils;
+
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
+import org.junit.ClassRule;
 
 import java.util.Base64;
 import java.util.Collection;
@@ -24,6 +28,10 @@
 import static org.hamcrest.Matchers.not;
 
 public class GoogleCloudStorageThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
+    private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.google.fixture", "true"));
+
+    @ClassRule
+    public static GoogleCloudStorageHttpFixture fixture = new GoogleCloudStorageHttpFixture(USE_FIXTURE, "bucket", "o/oauth2/token");
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
@@ -34,27 +42,30 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     protected Settings nodeSettings() {
         Settings.Builder builder = Settings.builder().put(super.nodeSettings());
 
-        if (Strings.isNullOrEmpty(System.getProperty("test.google.endpoint")) == false) {
-            builder.put("gcs.client.default.endpoint", System.getProperty("test.google.endpoint"));
-        }
-
-        if (Strings.isNullOrEmpty(System.getProperty("test.google.tokenURI")) == false) {
-            builder.put("gcs.client.default.token_uri", System.getProperty("test.google.tokenURI"));
+        if (USE_FIXTURE) {
+            builder.put("gcs.client.default.endpoint", fixture.getAddress());
+            builder.put("gcs.client.default.token_uri", fixture.getAddress() + "/o/oauth2/token");
         }
 
         return builder.build();
     }
 
     @Override
     protected SecureSettings credentials() {
-        assertThat(System.getProperty("test.google.account"), not(blankOrNullString()));
+        if (USE_FIXTURE == false) {
+            assertThat(System.getProperty("test.google.account"), not(blankOrNullString()));
+        }
         assertThat(System.getProperty("test.google.bucket"), not(blankOrNullString()));
 
         MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setFile(
-            "gcs.client.default.credentials_file",
-            Base64.getDecoder().decode(System.getProperty("test.google.account"))
-        );
+        if (USE_FIXTURE) {
+            secureSettings.setFile("gcs.client.default.credentials_file", TestUtils.createServiceAccount(random()));
+        } else {
+            secureSettings.setFile(
+                "gcs.client.default.credentials_file",
+                Base64.getDecoder().decode(System.getProperty("test.google.account"))
+            );
+        }
         return secureSettings;
     }
 

modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/DefaultCredentialsRepositoryGcsClientYamlTestSuiteIT.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import fixture.gcs.GoogleCloudStorageHttpFixture;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+public class DefaultCredentialsRepositoryGcsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+    private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.google.fixture", "true"));
+
+    private static GoogleCloudStorageHttpFixture fixture = new GoogleCloudStorageHttpFixture(
+        true,
+        "bucket",
+        "computeMetadata/v1/instance/service-accounts/default/token"
+    );
+
+    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-gcs")
+        .setting("gcs.client.integration_test.endpoint", () -> fixture.getAddress())
+        .environment("GCE_METADATA_HOST", () -> fixture.getAddress().replace("http://", ""))
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
+
+    public DefaultCredentialsRepositoryGcsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @BeforeClass
+    public static void checkFixtureEnabled() {
+        assumeTrue("Only run against test fixture", USE_FIXTURE);
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+}

modules/repository-gcs/src/yamlRestTest/java/org/elasticsearch/repositories/gcs/LargeBlobRepositoryGcsClientYamlTestSuiteIT.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+
+public class LargeBlobRepositoryGcsClientYamlTestSuiteIT extends RepositoryGcsClientYamlTestSuiteIT {
+
+    static {
+        clusterConfig = c -> c.systemProperty("es.repository_gcs.large_blob_threshold_byte_size", "256");
+    }
+
+    public LargeBlobRepositoryGcsClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+}