Build: Shadow x-pack:protocol into x-pack:plugin:core

build.gradle

@@ -516,6 +516,32 @@ allprojects {
   tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
 }
 
+allprojects {
+  /*
+   * IntelliJ and, ironically, Eclipse don't know about the shadow plugin so
+   * when we're in "IntelliJ mode" or "Eclipse mode" add "runtime"
+   * dependencies eveywhere where we see a "shadow" dependency which will
+   * cause them to reference shadowed projects directly rather than rely
+   * on the shadowing to include them. This is the correct thing for
+   * it to do because it doesn't run the jar shadowing at all. This isn't
+   * needed for the project itself because the IDE configuration is done
+   * by SourceSets but it is *is* needed for projects that depends on the
+   * project doing the shadowing. Without this they won't properly depend
+   * on the shadowed project.
+   */
+  if (isEclipse || isIdea) {
+	configurations.all { Configuration configuration ->
+	  dependencies.all { Dependency dep ->
+		if (dep instanceof ProjectDependency) {
+		  if (dep.getTargetConfiguration() == 'shadow') {
+			configuration.dependencies.add(project.dependencies.project(path: dep.dependencyProject.path, configuration: 'runtime'))
+		  }
+		}
+	  }
+	}
+  }
+}
+
 // we need to add the same --debug-jvm option as
 // the real RunTask has, so we can pass it through
 class Run extends DefaultTask {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -379,6 +379,9 @@ class BuildPlugin implements Plugin<Project> {
         project.configurations.compile.dependencies.all(disableTransitiveDeps)
         project.configurations.testCompile.dependencies.all(disableTransitiveDeps)
         project.configurations.compileOnly.dependencies.all(disableTransitiveDeps)
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            project.configurations.shadow.dependencies.all(disableTransitiveDeps)
+        }
     }
 
     /** Adds repositories used by ES dependencies */
@@ -867,11 +870,20 @@ class BuildPlugin implements Plugin<Project> {
         project.dependencyLicenses.dependencies = project.configurations.runtime.fileCollection {
             it.group.startsWith('org.elasticsearch') == false
         } - project.configurations.compileOnly
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            project.dependencyLicenses.dependencies += project.configurations.shadow.fileCollection {
+                it.group.startsWith('org.elasticsearch') == false
+            }
+        }
     }
 
     private static configureDependenciesInfo(Project project) {
         Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class)
         deps.runtimeConfiguration = project.configurations.runtime
+        project.plugins.withType(ShadowPlugin).whenPluginAdded {
+            deps.runtimeConfiguration = project.configurations.create('infoDeps')
+            deps.runtimeConfiguration.extendsFrom(project.configurations.runtime, project.configurations.shadow)
+        }
         deps.compileOnlyConfiguration = project.configurations.compileOnly
         project.afterEvaluate {
             deps.mappings = project.dependencyLicenses.mappings

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -48,18 +48,6 @@ public class PluginBuildPlugin extends BuildPlugin {
     @Override
     public void apply(Project project) {
         super.apply(project)
-        project.plugins.withType(ShadowPlugin).whenPluginAdded {
-            /*
-             * We've not tested these plugins together and we're fairly sure
-             * they aren't going to work properly as is *and* we're not really
-             * sure *why* you'd want to shade stuff in plugins. So we throw an
-             * exception here to make you come and read this comment. If you
-             * have a need for shadow while building plugins then know that you
-             * are probably going to have to fight with gradle for a while....
-             */
-            throw new InvalidUserDataException('elasticsearch.esplugin is not '
-                    + 'compatible with com.github.johnrengelman.shadow');
-        }
         configureDependencies(project)
         // this afterEvaluate must happen before the afterEvaluate added by integTest creation,
         // so that the file name resolution for installing the plugin will be setup
@@ -153,8 +141,24 @@ public class PluginBuildPlugin extends BuildPlugin {
                 include(buildProperties.descriptorOutput.name)
             }
             from pluginMetadata // metadata (eg custom security policy)
-            from project.jar // this plugin's jar
-            from project.configurations.runtime - project.configurations.compileOnly // the dep jars
+            /*
+             * We bundle the plugin's jar file and its dependencies. We have
+             * to wait until the project is evaluated before we know if the
+             * plug *has* the shadow plugin. If it does we need the shadow
+             * jar and the deps in the shadow configuration because those are
+             * the jars that are not bundled into the plugin's jar. Otherwise
+             * we're fine with the jar an all of the non-provided runtime
+             * dependencies.
+             */
+            project.afterEvaluate {
+                if (project.plugins.hasPlugin(ShadowPlugin)) {
+                    from project.shadowJar
+                    from project.configurations.shadow
+                } else {
+                    from project.jar
+                    from project.configurations.runtime - project.configurations.compileOnly
+                }
+            }
             // extra files for the plugin to go into the zip
             from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging
             from('src/main') {

x-pack/docs/build.gradle

@@ -30,7 +30,7 @@ buildRestTests.expectedUnconvertedCandidates = [
 ]
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }
@@ -264,7 +264,7 @@ setups['farequote_index'] = '''
                   airline:
                      type: keyword
                   doc_count:
-                     type: integer       
+                     type: integer
 '''
 setups['farequote_data'] = setups['farequote_index'] + '''
   - do:
@@ -277,7 +277,7 @@ setups['farequote_data'] = setups['farequote_index'] + '''
             {"airline":"JZA","responsetime":990.4628,"time":"2016-02-07T00:00:00+0000", "doc_count": 5}
             {"index": {"_id":"2"}}
             {"airline":"JBU","responsetime":877.5927,"time":"2016-02-07T00:00:00+0000", "doc_count": 23}
-            {"index": {"_id":"3"}}            
+            {"index": {"_id":"3"}}
             {"airline":"KLM","responsetime":1355.4812,"time":"2016-02-07T00:00:00+0000", "doc_count": 42}
 '''
 setups['farequote_job'] = setups['farequote_data'] + '''
@@ -309,7 +309,7 @@ setups['farequote_datafeed'] = setups['farequote_job'] + '''
           "job_id":"farequote",
           "indexes":"farequote"
           }
-'''          
+'''
 setups['server_metrics_index'] = '''
   - do:
         indices.create:

x-pack/license-tools/build.gradle

@@ -1,7 +1,7 @@
 apply plugin: 'elasticsearch.build'
 
 dependencies {
-  compile project(xpackModule('core'))
+  compile project(path: xpackModule('core'), configuration: 'shadow')
   compile "org.elasticsearch:elasticsearch:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
 }
@@ -17,7 +17,7 @@ task buildZip(type: Zip, dependsOn: jar) {
   into(parentDir + '/lib') {
     from jar
     from configurations.runtime
-  } 
+  }
   into(parentDir + '/bin') {
     from 'bin'
   }

x-pack/plugin/core/build.gradle

@@ -8,6 +8,7 @@ import java.nio.file.StandardCopyOption
 apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
+apply plugin: 'com.github.johnrengelman.shadow'
 
 archivesBaseName = 'x-pack-core'
 
@@ -28,19 +29,19 @@ dependencyLicenses {
 dependencies {
     compileOnly "org.elasticsearch:elasticsearch:${version}"
     compile project(':x-pack:protocol')
-    compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
-    compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
-    compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
-    compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
+    shadow "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+    shadow "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+    shadow "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
+    shadow "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
 
-    compile "commons-logging:commons-logging:${versions.commonslogging}"
-    compile "commons-codec:commons-codec:${versions.commonscodec}"
+    shadow "commons-logging:commons-logging:${versions.commonslogging}"
+    shadow "commons-codec:commons-codec:${versions.commonscodec}"
 
     // security deps
-    compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
-    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
-    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
-    compile project(path: ':modules:transport-netty4', configuration: 'runtime')
+    shadow 'com.unboundid:unboundid-ldapsdk:3.2.0'
+    shadow 'org.bouncycastle:bcprov-jdk15on:1.59'
+    shadow 'org.bouncycastle:bcpkix-jdk15on:1.59'
+    shadow project(path: ':modules:transport-netty4', configuration: 'runtime')
 
     testCompile 'org.elasticsearch:securemock:1.2'
     testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
@@ -110,7 +111,8 @@ test {
 // TODO: don't publish test artifacts just to run messy tests, fix the tests!
 // https://github.com/elastic/x-plugins/issues/724
 configurations {
-    testArtifacts.extendsFrom testRuntime
+    testArtifacts.extendsFrom(testRuntime, shadow)
+    testArtifacts.exclude(group: project(':x-pack:protocol').group, module: project(':x-pack:protocol').name)
 }
 task testJar(type: Jar) {
     appendix 'test'

x-pack/plugin/deprecation/build.gradle

@@ -10,7 +10,7 @@ esplugin {
 archivesBaseName = 'x-pack-deprecation'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
 }
 
 run {

x-pack/plugin/graph/build.gradle

@@ -10,7 +10,7 @@ esplugin {
 archivesBaseName = 'x-pack-graph'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 

x-pack/plugin/logstash/build.gradle

@@ -10,7 +10,7 @@ esplugin {
 archivesBaseName = 'x-pack-logstash'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 
 }

x-pack/plugin/ml/build.gradle

@@ -40,7 +40,7 @@ compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,
 compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     // This should not be here
     testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')

x-pack/plugin/monitoring/build.gradle

@@ -13,7 +13,7 @@ esplugin {
 archivesBaseName = 'x-pack-monitoring'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 
     // monitoring deps
@@ -66,7 +66,7 @@ task internalClusterTest(type: RandomizedTestingTask,
   include '**/*IT.class'
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
-check.dependsOn internalClusterTest 
+check.dependsOn internalClusterTest
 internalClusterTest.mustRunAfter test
 
 // also add an "alias" task to make typing on the command line easier task icTest {

x-pack/plugin/rollup/build.gradle

@@ -16,7 +16,7 @@ compileTestJava.options.compilerArgs << "-Xlint:-rawtypes"
 dependencies {
     compileOnly "org.elasticsearch:elasticsearch:${version}"
 
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 

x-pack/plugin/security/build.gradle

@@ -12,7 +12,7 @@ esplugin {
 archivesBaseName = 'x-pack-security'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     compileOnly project(path: ':modules:transport-netty4', configuration: 'runtime')
     compileOnly project(path: ':plugins:transport-nio', configuration: 'runtime')
 
@@ -52,7 +52,7 @@ dependencies {
     compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
     compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
     compile "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}"
-    compile 'com.google.guava:guava:19.0' 
+    compile 'com.google.guava:guava:19.0'
 
     testCompile 'org.elasticsearch:securemock:1.2'
     testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"

x-pack/plugin/sql/build.gradle

@@ -19,7 +19,7 @@ archivesBaseName = 'x-pack-sql'
 integTest.enabled = false
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     compileOnly(project(':modules:lang-painless')) {
         // exclude ASM to not affect featureAware task on Java 10+
         exclude group: "org.ow2.asm"

x-pack/plugin/upgrade/build.gradle

@@ -14,7 +14,7 @@ esplugin {
 archivesBaseName = 'x-pack-upgrade'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
@@ -39,7 +39,7 @@ task internalClusterTest(type: RandomizedTestingTask,
   include '**/*IT.class'
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
-check.dependsOn internalClusterTest 
+check.dependsOn internalClusterTest
 internalClusterTest.mustRunAfter test
 
 // also add an "alias" task to make typing on the command line easier

x-pack/plugin/watcher/build.gradle

@@ -25,7 +25,7 @@ dependencyLicenses {
 dependencies {
     compileOnly "org.elasticsearch:elasticsearch:${version}"
 
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(path: xpackModule('core'), configuration: 'shadow')
     compileOnly project(path: ':modules:transport-netty4', configuration: 'runtime')
     compileOnly project(path: ':plugins:transport-nio', configuration: 'runtime')
 

x-pack/qa/full-cluster-restart/build.gradle

@@ -11,7 +11,7 @@ apply plugin: 'elasticsearch.build'
 test.enabled = false
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile (project(path: xpackModule('security'), configuration: 'runtime')) {
         // Need to drop the guava dependency here or we get a conflict with watcher's guava dependency.
         // This is total #$%, but the solution is to get the SAML realm (which uses guava) out of security proper
@@ -249,7 +249,7 @@ subprojects {
   check.dependsOn(integTest)
 
   dependencies {
-      testCompile project(path: xpackModule('core'), configuration: 'runtime')
+      testCompile project(path: xpackModule('core'), configuration: 'shadow')
       testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
       testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
       testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')

x-pack/qa/ml-basic-multi-node/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
 

x-pack/qa/ml-disabled/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
 

x-pack/qa/ml-native-multi-node-tests/build.gradle

@@ -4,7 +4,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
     testCompile project(path: xpackModule('ml'), configuration: 'testArtifacts')

x-pack/qa/ml-no-bootstrap-tests/build.gradle

@@ -1,7 +1,6 @@
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
-    testCompile project(path: xpackModule('core'), configuration: 'runtime')
+    testCompile project(path: xpackModule('core'), configuration: 'shadow')
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
-

x-pack/qa/ml-single-node-tests/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile project(path: xpackModule('core'), configuration: 'runtime')
+  testCompile project(path: xpackModule('core'), configuration: 'shadow')
   testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }