Fixture for Minio testing

plugins/repository-s3/build.gradle

@@ -1,3 +1,12 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.LoggedExec
+import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.test.AntFixture
+import org.elasticsearch.gradle.test.ClusterConfiguration
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+import java.lang.reflect.Field
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -64,14 +73,213 @@ test {
   exclude '**/*CredentialsTests.class'
 }
 
-check {
-  // also execute the QA tests when testing the plugin
-  dependsOn 'qa:amazon-s3:check'
+boolean useFixture = false
+
+String s3AccessKey = System.getenv("amazon_s3_access_key")
+String s3SecretKey = System.getenv("amazon_s3_secret_key")
+String s3Bucket = System.getenv("amazon_s3_bucket")
+String s3BasePath = System.getenv("amazon_s3_base_path")
+
+if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
+  s3AccessKey = 's3_integration_test_access_key'
+  s3SecretKey = 's3_integration_test_secret_key'
+  s3Bucket = 'bucket-test'
+  s3BasePath = 'integration_test'
+  useFixture = true
+} else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) {
+  throw new IllegalArgumentException("not all options specified to run against external S3 service")
+}
+
+final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
+final String minioBinDir = "${buildDir}/minio/bin"
+final String minioDataDir = "${buildDir}/minio/data"
+final String minioAddress = "127.0.0.1:60920"
+
+final String minioDistribution
+final String minioCheckSum
+if (Os.isFamily(Os.FAMILY_MAC)) {
+  minioDistribution = 'darwin-amd64'
+  minioCheckSum = '96b0bcb2f590e8e65fb83d5c3e221f9bd1106b49fa6f22c6b726b80b845d7c60'
+} else if (Os.isFamily(Os.FAMILY_UNIX)) {
+  minioDistribution = 'linux-amd64'
+  minioCheckSum = '713dac7c105285eab3b92649be92b5e793b29d3525c7929fa7aaed99374fad99'
+} else {
+  minioDistribution = null
+  minioCheckSum = null
+}
+
+buildscript {
+  repositories {
+    maven {
+      url 'https://plugins.gradle.org/m2/'
+    }
+  }
+  dependencies {
+    classpath 'de.undercouch:gradle-download-task:3.4.3'
+  }
+}
+
+if (useFixture && minioDistribution) {
+  apply plugin: 'de.undercouch.download'
+
+  final String minioFileName = "minio.${minioVersion}"
+  final String minioDownloadURL = "https://dl.minio.io/server/minio/release/${minioDistribution}/archive/${minioFileName}"
+  final String minioFilePath = "${gradle.gradleUserHomeDir}/downloads/minio/${minioDistribution}/${minioFileName}"
+
+  task downloadMinio(type: Download) {
+    src minioDownloadURL
+    dest minioFilePath
+    onlyIfModified true
+  }
+
+  task verifyMinioChecksum(type: Verify, dependsOn: downloadMinio) {
+    src minioFilePath
+    algorithm 'SHA-256'
+    checksum minioCheckSum
+  }
+
+  task installMinio(type: Sync, dependsOn: verifyMinioChecksum) {
+    from minioFilePath
+    into minioBinDir
+    fileMode 0755
+  }
+
+  task startMinio {
+    dependsOn installMinio
+
+    ext.minioPid = 0L
+
+    doLast {
+      new File("${minioDataDir}/${s3Bucket}").mkdirs()
+      // we skip these tests on Windows so we do no need to worry about compatibility here
+      final ProcessBuilder minio = new ProcessBuilder(
+              "${minioBinDir}/${minioFileName}",
+              "server",
+              "--address",
+              minioAddress,
+              minioDataDir)
+      minio.environment().put('MINIO_ACCESS_KEY', s3AccessKey)
+      minio.environment().put('MINIO_SECRET_KEY', s3SecretKey)
+      final Process process = minio.start()
+      if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+        try {
+          Class<?> cProcessImpl = process.getClass()
+          Field fPid = cProcessImpl.getDeclaredField("pid")
+          if (!fPid.isAccessible()) {
+            fPid.setAccessible(true)
+          }
+          minioPid = fPid.getInt(process)
+        } catch (Exception e) {
+          logger.error("failed to read pid from minio process", e)
+          process.destroyForcibly()
+          throw e
+        }
+      } else {
+        minioPid = process.pid()
+      }
+
+      new BufferedReader(new InputStreamReader(process.getInputStream())).withReader { br ->
+        String line
+        int httpPort = 0
+        while ((line = br.readLine()) != null) {
+          logger.info(line)
+          if (line.matches('.*Endpoint.*:\\d+$')) {
+            assert httpPort == 0
+            final int index = line.lastIndexOf(":")
+            assert index >= 0
+            httpPort = Integer.parseInt(line.substring(index + 1))
+
+            final File script = new File(project.buildDir, "minio/minio.killer.sh")
+            script.setText(
+                    ["function shutdown {",
+                     "  kill ${minioPid}",
+                     "}",
+                     "trap shutdown EXIT",
+                     // will wait indefinitely for input, but we never pass input, and the pipe is only closed when the build dies
+                     "read line\n"].join('\n'), 'UTF-8')
+            final ProcessBuilder killer = new ProcessBuilder("bash", script.absolutePath)
+            killer.start()
+            break
+          }
+        }
+
+        assert httpPort > 0
+      }
+    }
+  }
+
+  task stopMinio(type: LoggedExec) {
+    onlyIf { startMinio.minioPid > 0 }
+
+    doFirst {
+      logger.info("Shutting down minio with pid ${startMinio.minioPid}")
+    }
+
+    final Object pid = "${ -> startMinio.minioPid }"
+
+    // we skip these tests on Windows so we do no need to worry about compatibility here
+    executable = 'kill'
+    args('-9', pid)
+  }
+
+  RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) {
+    description = "Runs REST tests using the Minio repository."
+  }
+
+  // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
+  project.afterEvaluate {
+    ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration
+    cluster.dependsOn(project.bundlePlugin)
+    cluster.keystoreSetting 's3.client.integration_test.access_key', s3AccessKey
+    cluster.keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey
+    cluster.setting 's3.client.integration_test.endpoint', "http://${minioAddress}"
+
+    Task restIntegTestTask = project.tasks.getByName('integTestMinio')
+    restIntegTestTask.clusterConfig.plugin(project.path)
+
+    // Default jvm arguments for all test clusters
+    String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
+            " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
+            " " + System.getProperty('tests.jvm.argline', '')
+
+    restIntegTestTask.clusterConfig.jvmArgs = jvmArgs
+  }
+
+  integTestMinioRunner.dependsOn(startMinio)
+  integTestMinioRunner.finalizedBy(stopMinio)
+
+  project.check.dependsOn(integTestMinio)
+}
+
+/** A task to start the AmazonS3Fixture which emulates an S3 service **/
+task s3Fixture(type: AntFixture) {
+  dependsOn testClasses
+  env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
+  executable = new File(project.runtimeJavaHome, 'bin/java')
+  args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3Bucket
+}
+
+Map<String, Object> expansions = [
+  'bucket': s3Bucket,
+  'base_path': s3BasePath
+]
+
+processTestResources {
+  inputs.properties(expansions)
+  MavenFilteringHack.filter(it, expansions)
 }
 
 integTestCluster {
-  keystoreSetting 's3.client.integration_test.access_key', "s3_integration_test_access_key"
-  keystoreSetting 's3.client.integration_test.secret_key', "s3_integration_test_secret_key"
+  keystoreSetting 's3.client.integration_test.access_key', s3AccessKey
+  keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey
+
+  if (useFixture) {
+    dependsOn s3Fixture
+    /* Use a closure on the string to delay evaluation until tests are executed */
+    setting 's3.client.integration_test.endpoint', "http://${-> s3Fixture.addressAndPort}"
+  } else {
+    println "Using an external service to test the repository-s3 plugin"
+  }
 }
 
 thirdPartyAudit.excludes = [