Add minio support

Author: ywelsch

plugins/repository-s3/build.gradle

@@ -1,5 +1,9 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.AntFixture
+import org.elasticsearch.gradle.test.ClusterConfiguration
+import org.elasticsearch.gradle.test.RestIntegTestTask
 
 /*
  * Licensed to Elasticsearch under one or more contributor
@@ -77,11 +81,158 @@ String s3BasePath = System.getenv("amazon_s3_base_path")
 if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
   s3AccessKey = 's3_integration_test_access_key'
   s3SecretKey = 's3_integration_test_secret_key'
-  s3Bucket = 'bucket_test'
+  s3Bucket = 'bucket-test'
   s3BasePath = 'integration_test'
   useFixture = true
 }
 
+final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
+final String minioBinDir = "${buildDir}/minio/bin"
+final String minioDataDir = "${buildDir}/minio/data"
+final String minioAddress = "localhost:60920"
+
+final String minioDistribution
+final String minioCheckSum
+if (Os.isFamily(Os.FAMILY_MAC)) {
+  minioDistribution = 'darwin-amd64'
+  minioCheckSum = '96b0bcb2f590e8e65fb83d5c3e221f9bd1106b49fa6f22c6b726b80b845d7c60'
+} else if (Os.isFamily(Os.FAMILY_UNIX)) {
+  minioDistribution = 'linux-amd64'
+  minioCheckSum = '713dac7c105285eab3b92649be92b5e793b29d3525c7929fa7aaed99374fad99'
+} else {
+  minioDistribution = null
+  minioCheckSum = null
+}
+
+buildscript {
+  repositories {
+    maven {
+      url 'https://plugins.gradle.org/m2/'
+    }
+  }
+  dependencies {
+    classpath 'de.undercouch:gradle-download-task:3.4.3'
+  }
+}
+
+if (minioDistribution) {
+  apply plugin: 'de.undercouch.download'
+
+  final String minioFileName = "minio.${minioVersion}"
+  final String minioDownloadURL = "https://dl.minio.io/server/minio/release/${minioDistribution}/archive/${minioFileName}"
+  final String minioFilePath = "${gradle.gradleUserHomeDir}/downloads/minio/${minioDistribution}/${minioFileName}"
+
+  task downloadMinio(type: Download) {
+    src minioDownloadURL
+    dest minioFilePath
+    onlyIfModified true
+  }
+
+  task verifyMinioChecksum(type: Verify, dependsOn: downloadMinio) {
+    src minioFilePath
+    algorithm 'SHA-256'
+    checksum minioCheckSum
+  }
+
+  task installMinio(type: Sync, dependsOn: verifyMinioChecksum) {
+    from minioFilePath
+    into minioBinDir
+    fileMode 0755
+  }
+
+  long minioPid
+
+  task startMinio {
+    dependsOn installMinio
+    doLast {
+      new File("${minioDataDir}/${s3Bucket}").mkdirs()
+      // we skip these tests on Windows so we do no need to worry about compatibility here
+      final ProcessBuilder minio = new ProcessBuilder(
+              "${minioBinDir}/${minioFileName}",
+              "server",
+              "--address",
+              minioAddress,
+              minioDataDir)
+      minio.addShutdownHook { minio.destroy() }
+      minio.environment().put('MINIO_ACCESS_KEY', s3AccessKey)
+      minio.environment().put('MINIO_SECRET_KEY', s3SecretKey)
+      minio.environment().put('MINIO_DOMAIN', 'localhost')
+      final Process process = minio.start()
+      minioPid = process.pid()
+
+      new BufferedReader(new InputStreamReader(process.getInputStream())).withReader { br ->
+        String line
+        int httpPort = 0
+        while ((line = br.readLine()) != null) {
+          logger.info(line)
+          if (line.matches('.*Endpoint.*:\\d+$')) {
+            assert httpPort == 0
+            final int index = line.lastIndexOf(":")
+            assert index >= 0
+            httpPort = Integer.parseInt(line.substring(index + 1))
+
+            final File script = new File(project.buildDir, "minio/minio.killer.sh")
+            script.setText(
+                    ["function shutdown {",
+                     "  kill ${minioPid}",
+                     "}",
+                     "trap shutdown EXIT",
+                     // will wait indefinitely for input, but we never pass input, and the pipe is only closed when the build dies
+                     "read line\n"].join('\n'), 'UTF-8')
+            final ProcessBuilder killer = new ProcessBuilder("bash", script.absolutePath)
+            killer.start()
+            break
+          }
+        }
+
+        assert httpPort > 0
+      }
+    }
+  }
+
+  task stopMinio(type: LoggedExec) {
+    onlyIf { minioPid > 0 }
+
+    doFirst {
+      logger.info("Shutting down minio with pid ${minioPid}")
+    }
+
+    final Object pid = "${ -> minioPid }"
+
+    // we skip these tests on Windows so we do no need to worry about compatibility here
+    executable = 'kill'
+    args('-9', pid)
+  }
+
+  RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) {
+    description = "Runs REST tests using the Minio repository."
+  }
+
+  // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
+  project.afterEvaluate {
+    ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration
+    cluster.dependsOn(project.bundlePlugin)
+    cluster.keystoreSetting 's3.client.integration_test.access_key', s3AccessKey
+    cluster.keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey
+    cluster.setting 's3.client.integration_test.endpoint', "http://${minioAddress}"
+
+    Task restIntegTestTask = project.tasks.getByName('integTestMinio')
+    restIntegTestTask.clusterConfig.plugin(project.path)
+
+    // Default jvm arguments for all test clusters
+    String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
+            " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
+            " " + System.getProperty('tests.jvm.argline', '')
+
+    restIntegTestTask.clusterConfig.jvmArgs = jvmArgs
+  }
+
+  integTestMinioRunner.dependsOn(startMinio)
+  integTestMinioRunner.finalizedBy(stopMinio)
+
+  project.check.dependsOn(integTestMinio)
+}
+
 /** A task to start the AmazonS3Fixture which emulates an S3 service **/
 task s3Fixture(type: AntFixture) {
   dependsOn testClasses