Add support for Kerberized Elasticsearch (#1244)

Elasticsearch supports using Kerberos to authenticate over HTTP by means of SPNEGO. 
Since most secure Hadoop environments make use of Kerberos to secure their 
environments, naturally ES-Hadoop should support Kerberos authentication.

With the recent addition of API Key authentication to Elasticsearch, we now have a stable 
means for obtaining long lived and stable tokens for worker process authentication when 
using Kerberos.

Author: jbaiera

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy

@@ -214,6 +214,7 @@ class BuildPlugin implements Plugin<Project>  {
                 exclude group: "org.elasticsearch", module: "elasticsearch-core"
                 exclude group: "org.elasticsearch", module: "elasticsearch-secure-sm"
             }
+
             testRuntime "org.slf4j:slf4j-log4j12:1.7.6"
             testRuntime "org.apache.logging.log4j:log4j-api:${project.ext.log4jVersion}"
             testRuntime "org.apache.logging.log4j:log4j-core:${project.ext.log4jVersion}"
@@ -224,6 +225,11 @@ class BuildPlugin implements Plugin<Project>  {
             testRuntime "com.vividsolutions:jts:1.13"
 
             // TODO: Remove when we merge ITests to test dirs
+            itestCompile("org.apache.hadoop:hadoop-minikdc:${project.ext.minikdcVersion}") {
+                // For some reason, the dependencies that are pulled in with MiniKDC have multiple resource files
+                // that cause issues when they are loaded. We exclude the ldap schema data jar to get around this.
+                exclude group: "org.apache.directory.api", module: "api-ldap-schema-data"
+            }
             itestCompile project.sourceSets.main.output
             itestCompile project.configurations.testCompile
             itestCompile project.configurations.provided

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/ElasticsearchFixturePlugin.groovy

@@ -1,15 +1,16 @@
 package org.elasticsearch.hadoop.gradle.fixture
 
-import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.test.ClusterConfiguration
 import org.elasticsearch.gradle.test.ClusterFormationTasks
 import org.elasticsearch.gradle.test.NodeInfo
+import org.elasticsearch.hadoop.gradle.util.PlaceholderTask
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.execution.TaskExecutionAdapter
 import org.gradle.api.tasks.TaskState
+import org.gradle.util.ConfigureUtil
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
@@ -27,6 +28,26 @@ import java.util.stream.Stream
  */
 class ElasticsearchFixturePlugin implements Plugin<Project> {
 
+    static class ElasticsearchCluster {
+
+        Project project
+        ClusterConfiguration configuration
+        List<Task> tasks = []
+
+        ElasticsearchCluster(Project project) {
+            this.project = project
+            this.configuration = new ClusterConfiguration(project)
+        }
+
+        void clusterConf(Closure configClosure) {
+            ConfigureUtil.configure(configClosure, configuration)
+        }
+
+        void addTask(Task task) {
+            tasks.add(task)
+        }
+    }
+
     @Override
     void apply(Project project) {
 
@@ -55,7 +76,9 @@ class ElasticsearchFixturePlugin implements Plugin<Project> {
         Task clusterInit = project.tasks.create(name: "esCluster#init", dependsOn: project.testClasses)
         integrationTest.dependsOn(clusterInit)
 
-        ClusterConfiguration clusterConfig = project.extensions.create("esCluster", ClusterConfiguration.class, project)
+        ElasticsearchCluster cluster = project.extensions.create("esCluster", ElasticsearchCluster.class, project)
+        cluster.tasks.add(integrationTest)
+        ClusterConfiguration clusterConfig = cluster.configuration
 
         // default settings:
         clusterConfig.clusterName = "elasticsearch-fixture"
@@ -96,18 +119,21 @@ class ElasticsearchFixturePlugin implements Plugin<Project> {
         // Also write a script to a file for use in tests
         File scriptsDir = new File(project.buildDir, 'scripts')
         scriptsDir.mkdirs()
-        File script
+        File script = null
         if (majorVersion <= 2) {
             scriptsDir.mkdirs()
             script = new File(scriptsDir, "increment.groovy").setText("ctx._source.counter+=1", 'UTF-8')
         } else if (majorVersion == 5) {
             scriptsDir.mkdirs()
             script = new File(scriptsDir, "increment.painless").setText("ctx._source.counter = ctx._source.getOrDefault('counter', 0) + 1", 'UTF-8')
         }
-        clusterConfig.extraConfigFile("script", script)
+        if (script != null) {
+            clusterConfig.extraConfigFile("script", script)
+        }
 
         project.gradle.projectsEvaluated {
-            List<NodeInfo> nodes = ClusterFormationTasks.setup(project, "esCluster", integrationTest, clusterConfig)
+            Task clusterMain = new PlaceholderTask()
+            List<NodeInfo> nodes = ClusterFormationTasks.setup(project, "esCluster", clusterMain, clusterConfig)
             project.tasks.getByPath("esCluster#wait").doLast {
                 integrationTest.systemProperty('tests.rest.cluster', "${nodes.collect{it.httpUri()}.join(",")}")
             }
@@ -123,11 +149,19 @@ class ElasticsearchFixturePlugin implements Plugin<Project> {
                     }
                 }
             }
-            integrationTest.doFirst {
-                project.gradle.addListener(logDumpListener)
-            }
-            integrationTest.doLast {
-                project.gradle.removeListener(logDumpListener)
+            for (Task clusterTask : cluster.tasks) {
+                for (Object dependency : clusterMain.taskDeps) {
+                    clusterTask.dependsOn(dependency)
+                }
+                for (Object finalizer : clusterMain.taskFinalizers) {
+                    clusterTask.finalizedBy(finalizer)
+                }
+                clusterTask.doFirst {
+                    project.gradle.addListener(logDumpListener)
+                }
+                clusterTask.doLast {
+                    project.gradle.removeListener(logDumpListener)
+                }
             }
         }
     }

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/ConfigFormats.groovy

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.hadoop.gradle.fixture.hadoop
+
+class ConfigFormats {
+
+    static Closure<String> hadoopXML() {
+        return { Map conf ->
+            String props = conf.collect { key, value ->
+                "<property>\n\t\t<name>${key}</name>\n\t\t<value>${value}</value>\n\t</property>"
+            }.join("\n\t")
+            return "<configuration>\n\t${props}\n</configuration>"
+        }
+    }
+
+    static Closure<String> propertyFile() {
+        return { Map conf ->
+            conf.collect { key, value ->
+                "${key}=${value}"
+            }.join("\n")
+        }
+    }
+
+    static Closure<String> whiteSpaced() {
+        return { Map conf ->
+            conf.collect { key, value ->
+                "${key}   ${value}"
+            }.join("\n")
+        }
+    }
+}