Use 3rd party task to run integration tests on external service (#56588)

Backport of #56587 for 7.x

Author: tlrx

plugins/repository-azure/build.gradle

@@ -1,5 +1,9 @@
+import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.info.BuildParams
 
+import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -34,7 +38,7 @@ dependencies {
 
 restResources {
   restApi {
-    includeCore '_common', 'cluster', 'nodes'
+    includeCore '_common', 'cluster', 'nodes', 'snapshot', 'bulk', 'count', 'indices'
   }
 }
 
@@ -69,16 +73,13 @@ thirdPartyAudit {
   )
 }
 
-check {
-  // also execute the QA tests when testing the plugin
-  dependsOn 'qa:microsoft-azure-storage:check'
-}
+boolean useFixture = false
 
-testClusters {
-  integTest {
-    keystore 'azure.client.integration_test.account', 'azure_account'
-    keystore 'azure.client.integration_test.key', 'azure_key'
-  }
+def azureAddress = {
+  assert useFixture: 'closure should not be used without a fixture'
+  int ephemeralPort = project(':test:fixtures:azure-fixture').postProcessFixture.ext."test.fixtures.azure-fixture.tcp.8091"
+  assert ephemeralPort > 0
+  return 'ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:' + ephemeralPort + '/'
 }
 
 String azureAccount = System.getenv("azure_storage_account")
@@ -87,19 +88,60 @@ String azureContainer = System.getenv("azure_storage_container")
 String azureBasePath = System.getenv("azure_storage_base_path")
 String azureSasToken = System.getenv("azure_storage_sas_token")
 
+if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
+  azureAccount = 'azure_integration_test_account'
+  azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
+  azureContainer = 'container'
+  azureBasePath = ''
+  azureSasToken = ''
+  useFixture = true
+
+  apply plugin: 'elasticsearch.test.fixtures'
+  testFixtures.useFixture ':test:fixtures:azure-fixture', 'azure-fixture'
+}
+
+Map<String, Object> expansions = [
+  'container': azureContainer,
+  'base_path': azureBasePath + "_integration_tests"
+]
+
+processTestResources {
+  inputs.properties(expansions)
+  MavenFilteringHack.filter(it, expansions)
+}
+
 test {
+  // this is tested explicitly in a separate test task
   exclude '**/AzureStorageCleanupThirdPartyTests.class'
 }
 
-task thirdPartyTest(type: Test) {
+testClusters {
+  integTest {
+    keystore 'azure.client.integration_test.account', azureAccount
+    if (azureKey != null && azureKey.isEmpty() == false) {
+      keystore 'azure.client.integration_test.key', azureKey
+    }
+    if (azureSasToken != null && azureSasToken.isEmpty() == false) {
+      keystore 'azure.client.integration_test.sas_token', azureSasToken
+    }
+    if (useFixture) {
+      setting 'azure.client.integration_test.endpoint_suffix', azureAddress
+      String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0)
+      setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE
+    }
+  }
+}
+
+task azureThirdPartyTest(type: Test) {
+  dependsOn tasks.integTest
   include '**/AzureStorageCleanupThirdPartyTests.class'
   systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
   systemProperty 'test.azure.key', azureKey ? azureKey : ""
   systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : ""
   systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
   systemProperty 'test.azure.base', (azureBasePath ? azureBasePath : "") + "_third_party_tests_" + BuildParams.testSeed
+  if (useFixture) {
+    nonInputProperties.systemProperty 'test.azure.endpoint_suffix', "${-> azureAddress.call() }"
+  }
 }
-
-if (azureAccount || azureKey || azureContainer || azureBasePath || azureSasToken) {
-  check.dependsOn(thirdPartyTest)
-}
+check.dependsOn(azureThirdPartyTest)

plugins/repository-azure/qa/build.gradle

@@ -50,6 +50,18 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(AzureRepositoryPlugin.class);
     }
 
+    @Override
+    protected Settings nodeSettings() {
+        final String endpoint = System.getProperty("test.azure.endpoint_suffix");
+        if (Strings.hasText(endpoint)) {
+            return Settings.builder()
+                .put(super.nodeSettings())
+                .put("azure.client.default.endpoint_suffix", endpoint)
+                .build();
+        }
+        return super.nodeSettings();
+    }
+
     @Override
     protected SecureSettings credentials() {
         assertThat(System.getProperty("test.azure.account"), not(blankOrNullString()));