Fix projects that failed to build within Intellij (elastic#62258)

This commit address some build failures from the perspective of Intellij.
These changes include:
* changing an order of a dependency definition that seems to can cause Intellij build to fail.
* introduction of an abstract class out of the test source set (seems to be an issue sharing
  classes cross projects with non-standard source sets.
* a couple of missing dependency definitions (not sure how the command line worked prior to this)
# Conflicts:
#	x-pack/qa/security-example-spi-extension/build.gradle

Author: jakelandis

buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java

@@ -155,7 +155,7 @@ public static void setupIdeForTestSourceSet(Project project, SourceSet testSourc
         project.getPluginManager().withPlugin("idea", p -> {
             IdeaModel idea = project.getExtensions().getByType(IdeaModel.class);
             idea.getModule().setTestSourceDirs(testSourceSet.getJava().getSrcDirs());
-            idea.getModule().getScopes().put("TEST", Map.of("plus", List.of(runtimeClasspathConfiguration)));
+            idea.getModule().getScopes().put(testSourceSet.getName(), Map.of("plus", List.of(runtimeClasspathConfiguration)));
         });
         project.getPluginManager().withPlugin("eclipse", p -> {
             EclipseModel eclipse = project.getExtensions().getByType(EclipseModel.class);
@@ -196,6 +196,19 @@ public static void extendSourceSet(Project project, String parentSourceSetName,
         child.setRuntimeClasspath(project.getObjects().fileCollection().from(child.getRuntimeClasspath(), parent.getOutput()));
     }
 
+    /**
+     * Extends one configuration from another and refreshes the classpath of a provided Test.
+     * The Test parameter is only needed for eagerly defined test tasks.
+     */
+    public static void extendSourceSet(Project project, String parentSourceSetName, String childSourceSetName, Test test) {
+        extendSourceSet(project, parentSourceSetName, childSourceSetName);
+        if (test != null) {
+            SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+            SourceSet child = sourceSets.getByName(childSourceSetName);
+            test.setClasspath(child.getRuntimeClasspath());
+        }
+    }
+
     public static Dependency projectDependency(Project project, String projectPath, String projectConfig) {
         if (project.findProject(projectPath) == null) {
             throw new GradleException("no project [" + projectPath + "], project names: " + project.getRootProject().getAllprojects());

qa/die-with-dignity/build.gradle

@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.util.GradleUtils
 
 apply plugin: 'elasticsearch.java-rest-test'
 apply plugin: 'elasticsearch.esplugin'
@@ -8,10 +9,8 @@ esplugin {
   classname 'org.elasticsearch.DieWithDignityPlugin'
 }
 
-dependencies {
-  // let the javaRestTest see the classpath of main
-  javaRestTestImplementation project.sourceSets.main.runtimeClasspath
-}
+// let the javaRestTest see the classpath of main
+GradleUtils.extendSourceSet(project, "main", "javaRestTest", javaRestTest)
 
 javaRestTest {
   systemProperty 'tests.security.manager', 'false'

x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java

@@ -17,7 +17,7 @@
 import org.elasticsearch.test.rest.yaml.ClientYamlTestClient;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
-import org.elasticsearch.xpack.test.rest.XPackRestIT;
+import org.elasticsearch.xpack.test.rest.AbstractXPackRestTest;
 import org.junit.After;
 
 import java.util.List;
@@ -29,7 +29,7 @@
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.is;
 
-public class XDocsClientYamlTestSuiteIT extends XPackRestIT {
+public class XDocsClientYamlTestSuiteIT extends AbstractXPackRestTest {
     private static final String USER_TOKEN = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
 
     public XDocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

x-pack/plugin/build.gradle

@@ -12,9 +12,11 @@ apply plugin: 'elasticsearch.validate-rest-spec'
 archivesBaseName = 'x-pack'
 
 dependencies {
-  yamlRestTestImplementation project(xpackModule('core')) // this redundant dependency is here to make IntelliJ happy
-  yamlRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  testImplementation project(xpackModule('core'))
+  testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
   javaRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
+  // let the yamlRestTest see the classpath of test
+  yamlRestTestImplementation project.sourceSets.test.runtimeClasspath
 }
 
 subprojects {
@@ -61,18 +63,13 @@ subprojects {
 configurations {
   testArtifacts.extendsFrom testRuntime
   testArtifacts.extendsFrom testImplementation
-  testArtifacts.extendsFrom yamlRestTestImplementation
-  testArtifacts.extendsFrom javaRestTestImplementation
 }
 
 restResources {
   restApi {
     includeCore '*'
     includeXpack '*'
   }
-  restTests {
-    includeXpack '*'
-  }
 }
 
 //The api and tests need to stay at src/test/... since some external tooling depends on that exact file path.
@@ -83,10 +80,8 @@ artifacts {
 
 def testJar = tasks.register("testJar", Jar) {
   appendix 'test'
-  duplicatesStrategy = DuplicatesStrategy.INCLUDE
   from sourceSets.test.output
-  from sourceSets.yamlRestTest.output
-  from sourceSets.javaRestTest.output
+
   /*
    * Stick the license and notice file in the jar. This isn't strictly
    * needed because we don't publish it but it makes our super-paranoid

x-pack/plugin/deprecation/qa/rest/build.gradle

@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.util.GradleUtils
+
 apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.java-rest-test'
 
@@ -9,10 +11,12 @@ esplugin {
 dependencies {
   javaRestTestImplementation("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
   javaRestTestImplementation("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
-  // let the javaRestTest see the classpath of main
-  javaRestTestImplementation project.sourceSets.main.runtimeClasspath
 }
 
+// let the javaRestTest see the classpath of main
+GradleUtils.extendSourceSet(project, "main", "javaRestTest", javaRestTest)
+
+
 restResources {
   restApi {
     includeCore '_common', 'indices', 'index'

x-pack/plugin/ilm/qa/multi-node/build.gradle

@@ -1,9 +1,15 @@
+import org.elasticsearch.gradle.util.GradleUtils
+
 apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }
 
+// let the javaRestTest see the classpath of main
+GradleUtils.extendSourceSet(project, "main", "javaRestTest", javaRestTest)
+
+
 File repoDir = file("$buildDir/testclusters/repo")
 
 javaRestTest {

x-pack/plugin/ilm/qa/with-security/build.gradle

@@ -1,6 +1,7 @@
 apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
+  javaRestTestImplementation project(path: xpackModule('core'))
   javaRestTestImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
   testImplementation project(":client:rest-high-level")
 }

x-pack/plugin/ml/qa/ml-with-security/build.gradle

@@ -1,8 +1,8 @@
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 dependencies {
-  yamlRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
   yamlRestTestImplementation project(path: xpackModule('core'))
+  yamlRestTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
   yamlRestTestImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }
 

x-pack/plugin/ml/qa/ml-with-security/src/yamlRestTest/java/org/elasticsearch/smoketest/MlWithSecurityIT.java

@@ -11,15 +11,15 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.xpack.test.rest.XPackRestIT;
+import org.elasticsearch.xpack.test.rest.AbstractXPackRestTest;
 
 import java.util.Collections;
 import java.util.Map;
 
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 
 
-public class MlWithSecurityIT extends XPackRestIT {
+public class MlWithSecurityIT extends AbstractXPackRestTest {
 
     private static final String TEST_ADMIN_USERNAME = "x_pack_rest_user";
 

x-pack/plugin/runtime-fields/qa/with-security/build.gradle

@@ -1,6 +1,7 @@
 apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
+  javaRestTestImplementation project(path: xpackModule('core'))
   javaRestTestImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }