Remove unnecessary usage of Gradle dependency substitution rules (elastic#42773)

Author: mark-vieira

benchmarks/build.gradle

@@ -27,7 +27,7 @@ archivesBaseName = 'elasticsearch-benchmarks'
 test.enabled = false
 
 dependencies {
-    compile("org.elasticsearch:elasticsearch:${version}") {
+    compile(project(":server")) {
         // JMH ships with the conflicting version 4.6. This prevents us from using jopt-simple in benchmarks (which should be ok) but allows
         // us to invoke the JMH uberjar as usual.
         exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'

build.gradle

@@ -209,68 +209,7 @@ allprojects {
     javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
   }
 
-  /* Sets up the dependencies that we build as part of this project but
-    register as though they were external to resolve internally. We register
-    them as external dependencies so the build plugin that we use can be used
-    to build elasticsearch plugins outside of the elasticsearch source tree. */
-  ext.projectSubstitutions = [
-    "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
-    "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
-    "org.elasticsearch:elasticsearch:${version}": ':server',
-    "org.elasticsearch:elasticsearch-cli:${version}": ':libs:elasticsearch-cli',
-    "org.elasticsearch:elasticsearch-core:${version}": ':libs:core',
-    "org.elasticsearch:elasticsearch-nio:${version}": ':libs:nio',
-    "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
-    "org.elasticsearch:elasticsearch-geo:${version}": ':libs:elasticsearch-geo',
-    "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',
-    "org.elasticsearch:elasticsearch-ssl-config:${version}": ':libs:elasticsearch-ssl-config',
-    "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
-    "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
-    "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
-    "org.elasticsearch.client:test:${version}": ':client:test',
-    "org.elasticsearch.client:transport:${version}": ':client:transport',
-    "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
-    "org.elasticsearch.test:framework:${version}": ':test:framework',
-    "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
-    "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
-    // for transport client
-    "org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4',
-    "org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex',
-    "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
-    "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
-    "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
-    "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
-    "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
-    // for security example plugins
-    "org.elasticsearch.plugin:x-pack-core:${version}": ':x-pack:plugin:core'
-  ]
-
-  /*
-   * Gradle only resolve project substitutions during dependency resolution but
-   * we sometimes want to do the resolution at other times. This creates a
-   * convenient method we can call to do it.
-   */
-  ext.dependencyToProject = { Dependency dep ->
-    if (dep instanceof ProjectDependency) {
-      return dep.dependencyProject
-    } else {
-      String substitution = projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
-      if (substitution != null) {
-        return findProject(substitution)
-      }
-      return null
-    }
-  }
-
   project.afterEvaluate {
-    configurations.matching { it.canBeResolved }.all {
-      resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
-        projectSubstitutions.each { k,v ->
-          subs.substitute(subs.module(k)).with(subs.project(v))
-        }
-      }
-    }
-
     // Handle javadoc dependencies across projects. Order matters: the linksOffline for
     // org.elasticsearch:elasticsearch must be the last one or all the links for the
     // other packages (e.g org.elasticsearch.client) will point to server rather than
@@ -279,10 +218,10 @@ allprojects {
       String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
       Closure sortClosure = { a, b -> b.group <=> a.group }
       Closure depJavadocClosure = { shadowed, dep ->
-        if (dep.group == null || false == dep.group.startsWith('org.elasticsearch')) {
+        if ((dep instanceof ProjectDependency) == false) {
           return
         }
-        Project upstreamProject = project.ext.dependencyToProject(dep)
+        Project upstreamProject = dep.dependencyProject
         if (upstreamProject == null) {
           return
         }
@@ -338,8 +277,8 @@ gradle.projectsEvaluated {
       integTest.mustRunAfter test
     }
     configurations.matching { it.canBeResolved }.all { Configuration configuration ->
-      dependencies.all { Dependency dep ->
-        Project upstreamProject = dependencyToProject(dep)
+      dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
+        Project upstreamProject = dep.dependencyProject
         if (upstreamProject != null) {
           if (project.path == upstreamProject.path) {
             // TODO: distribution integ tests depend on themselves (!), fix that

buildSrc/build.gradle

@@ -148,6 +148,11 @@ if (project != rootProject) {
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:oss-linux-tar')
   }
+    
+  // for external projects we want to remove the marker file indicating we are running the Elasticsearch project
+  processResources {
+      exclude 'buildSrc.marker'
+  }
 
   String localDownloads = "${rootProject.buildDir}/local-downloads"
   task setupLocalDownloads(type:Copy) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -26,6 +26,7 @@ import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
+import org.elasticsearch.gradle.tool.ClasspathUtils
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
@@ -136,8 +137,13 @@ class PluginBuildPlugin implements Plugin<Project> {
 
     private static void configureDependencies(Project project) {
         project.dependencies {
-            compileOnly "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}"
-            testCompile "org.elasticsearch.test:framework:${project.versions.elasticsearch}"
+            if (ClasspathUtils.isElasticsearchProject()) {
+                compileOnly project.project(':server')
+                testCompile project.project(':test:framework')
+            } else {
+                compileOnly "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}"
+                testCompile "org.elasticsearch.test:framework:${project.versions.elasticsearch}"
+            }
             // we "upgrade" these optional deps to provided for plugins, since they will run
             // with a full elasticsearch server that includes optional deps
             compileOnly "org.locationtech.spatial4j:spatial4j:${project.versions.spatial4j}"

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -23,11 +23,13 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.tool.ClasspathUtils
 import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.quality.Checkstyle
+
 /**
  * Validation tasks which should be run before committing. These run before tests.
  */
@@ -40,18 +42,18 @@ class PrecommitTasks {
     public static Task create(Project project, boolean includeDependencyLicenses) {
         project.configurations.create("forbiddenApisCliJar")
         project.dependencies {
-            forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.6')
+            forbiddenApisCliJar('de.thetaphi:forbiddenapis:2.6')
         }
 
         List<Task> precommitTasks = [
-            configureCheckstyle(project),
-            configureForbiddenApisCli(project),
-            project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
-            project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
-            project.tasks.create('filepermissions', FilePermissionsTask.class),
-            configureJarHell(project),
-            configureThirdPartyAudit(project),
-            configureTestingConventions(project)
+                configureCheckstyle(project),
+                configureForbiddenApisCli(project),
+                project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
+                project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
+                project.tasks.create('filepermissions', FilePermissionsTask.class),
+                configureJarHell(project),
+                configureThirdPartyAudit(project),
+                configureTestingConventions(project)
         ]
 
         // tasks with just tests don't need dependency licenses, so this flag makes adding
@@ -85,10 +87,10 @@ class PrecommitTasks {
         }
 
         return project.tasks.create([
-            name: 'precommit',
-            group: JavaBasePlugin.VERIFICATION_GROUP,
-            description: 'Runs all non-test checks.',
-            dependsOn: precommitTasks
+                name       : 'precommit',
+                group      : JavaBasePlugin.VERIFICATION_GROUP,
+                description: 'Runs all non-test checks.',
+                dependsOn  : precommitTasks
         ])
     }
 
@@ -168,7 +170,7 @@ class PrecommitTasks {
                 )
             }
         }
-        Task forbiddenApis =  project.tasks.getByName("forbiddenApis")
+        Task forbiddenApis = project.tasks.getByName("forbiddenApis")
         forbiddenApis.group = ""
         return forbiddenApis
     }
@@ -211,7 +213,7 @@ class PrecommitTasks {
         project.checkstyle {
             config = project.resources.text.fromFile(checkstyleConf, 'UTF-8')
             configProperties = [
-                suppressions: checkstyleSuppressions
+                    suppressions: checkstyleSuppressions
             ]
             toolVersion = CHECKSTYLE_VERSION
         }
@@ -229,9 +231,11 @@ class PrecommitTasks {
     }
 
     private static Task configureLoggerUsage(Project project) {
+        Object dependency = ClasspathUtils.isElasticsearchProject() ? project.project(':test:logger-usage') :
+                "org.elasticsearch.test:logger-usage:${VersionProperties.elasticsearch}"
+
         project.configurations.create('loggerUsagePlugin')
-        project.dependencies.add('loggerUsagePlugin',
-                "org.elasticsearch.test:logger-usage:${VersionProperties.elasticsearch}")
+        project.dependencies.add('loggerUsagePlugin', dependency)
         return project.tasks.create('loggerUsageCheck', LoggerUsageTask.class) {
             classpath = project.configurations.loggerUsagePlugin
         }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -252,7 +252,7 @@ class RestIntegTestTask extends DefaultTask {
             restSpec
         }
         project.dependencies {
-            restSpec "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
+            restSpec project.project(':rest-api-spec')
         }
         Task copyRestSpec = project.tasks.findByName('copyRestSpec')
         if (copyRestSpec != null) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -79,7 +79,7 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
 
         // create a compileOnly configuration as others might expect it
         project.configurations.create("compileOnly")
-        project.dependencies.add('testCompile', "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}")
+        project.dependencies.add('testCompile', project.project(':test:framework'))
 
         EclipseModel eclipse = project.extensions.getByType(EclipseModel)
         eclipse.classpath.sourceSets = [testSourceSet]

buildSrc/src/main/java/org/elasticsearch/gradle/tool/ClasspathUtils.java

@@ -0,0 +1,23 @@
+package org.elasticsearch.gradle.tool;
+
+public class ClasspathUtils {
+    private static boolean isElasticsearchProject;
+
+    static {
+        // look for buildSrc marker file, if it exists then we are running in the context of the elastic/elasticsearch build
+        isElasticsearchProject = ClasspathUtils.class.getResource("/buildSrc.marker") != null;
+    }
+
+    private ClasspathUtils() {
+    }
+
+    /**
+     * Determine if we are running in the context of the `elastic/elasticsearch` project. This method will return {@code false} when
+     * the build-tools project is pulled in as an external dependency.
+     *
+     * @return if we are currently running in the `elastic/elasticsearch` project
+     */
+    public static boolean isElasticsearchProject() {
+        return isElasticsearchProject;
+    }
+}

buildSrc/src/main/resources/buildSrc.marker

@@ -34,12 +34,12 @@ test.enabled = false
 dependencies {
   compile 'org.apache.commons:commons-math3:3.2'
 
-  compile("org.elasticsearch.client:elasticsearch-rest-client:${version}")
+  compile project(":client:rest")
   // bottleneck should be the client, not Elasticsearch
   compile project(path: ':client:client-benchmark-noop-api-plugin')
   // for transport client
-  compile("org.elasticsearch:elasticsearch:${version}")
-  compile("org.elasticsearch.client:transport:${version}")
+  compile project(":server")
+  compile project(":client:transport")
   compile project(path: ':modules:transport-netty4', configuration: 'runtime')
   compile project(path: ':modules:reindex', configuration: 'runtime')
   compile project(path: ':modules:lang-mustache', configuration: 'runtime')

client/benchmark/build.gradle

@@ -46,34 +46,34 @@ idea {
 }
 
 dependencies {
-  compile "org.elasticsearch:elasticsearch:${version}"
-  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
+  compile project(':server')
+  compile project(':client:rest')
   compile project(':modules:parent-join')
   compile project(':modules:aggs-matrix-stats')
   compile project(':modules:rank-eval')
   compile project(':modules:lang-mustache')
-  shadow "org.elasticsearch:elasticsearch:${version}"
-  shadow "org.elasticsearch.client:elasticsearch-rest-client:${version}"
+  shadow project(':server')
+  shadow project(':client:rest')
   bundle project(':modules:parent-join')
   bundle project(':modules:aggs-matrix-stats')
   bundle project(':modules:rank-eval')
   bundle project(':modules:lang-mustache')
 
-  testCompile "org.elasticsearch.client:test:${version}"
-  testCompile "org.elasticsearch.test:framework:${version}"
+  testCompile project(':client:test')
+  testCompile project(':test:framework')
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   //this is needed to make RestHighLevelClientTests#testApiNamingConventions work from IDEs
-  testCompile "org.elasticsearch:rest-api-spec:${version}"
+  testCompile project(":rest-api-spec")
   // Needed for serialization tests:
   // (In order to serialize a server side class to a client side class or the other way around)
   if (isEclipse == false || project.path == ":client:rest-high-level-tests") {
-    testCompile("org.elasticsearch.plugin:x-pack-core:${version}") {
+    testCompile(project(':x-pack:plugin:core')) {
         exclude group: 'org.elasticsearch', module: 'elasticsearch-rest-high-level-client'
     }
   }
 
-  restSpec "org.elasticsearch:rest-api-spec:${version}"
+  restSpec project(':rest-api-spec')
 }
 
 //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions)

client/rest-high-level/build.gradle

@@ -44,7 +44,7 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
 
-  testCompile "org.elasticsearch.client:test:${version}"
+  testCompile project(":client:test")
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest:${versions.hamcrest}"
@@ -68,7 +68,7 @@ forbiddenApisTest {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:core
+// TODO: Not anymore. Now in :libs:elasticsearch-core
 jarHell.enabled=false
 
 testingConventions {

client/rest/build.gradle

@@ -35,14 +35,14 @@ publishing {
 }
 
 dependencies {
-  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
+  compile project(":client:rest")
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
 
-  testCompile "org.elasticsearch.client:test:${version}"
+  testCompile project(":client:test")
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
@@ -68,7 +68,7 @@ dependencyLicenses {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:core
+// TODO: Not anymore. Now in :libs:elasticsearch-core
 jarHell.enabled=false
 
 testingConventions {

client/sniffer/build.gradle

@@ -44,7 +44,7 @@ forbiddenApisTest {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:core
+// TODO: Not anymore. Now in :libs:elasticsearch-core
 jarHell.enabled=false
 
 // TODO: should we have licenses for our test deps?