Remove internal build logic from public build tool plugins (#72470) (7.x backport) (#72832)

back porting #72470 to 7.x
Extract usage of internal API from TestClustersPlugin and PluginBuildPlugin and
related plugins and build logic

This includes a refactoring of ElasticsearchDistribution to handle types
better in a way we can differentiate between supported Elasticsearch
Distribution types supported in TestCkustersPlugin and types only supported
in internal plugins.

It also introduces a set of internal versions of public plugins.

As part of this we also generate the plugin descriptors now.

As a follow up on this we can actually move these public used classes into
an extra project (declared as included build)

We keep LoggedExec and VersionProperties effectively public And workaround for RestTestBase

Author: breskeby

build.gradle

@@ -12,7 +12,7 @@ import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.internal.BuildPlugin
 import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.VersionProperties
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.gradle.plugins.ide.eclipse.model.AccessRule
@@ -22,6 +22,9 @@ import org.gradle.util.GradleVersion
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
 import org.gradle.plugins.ide.eclipse.model.ProjectDependency
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
+import org.elasticsearch.gradle.internal.test.RestTestBasePlugin
+import org.elasticsearch.gradle.internal.InternalPluginBuildPlugin
+import org.elasticsearch.gradle.internal.InternalTestClustersPlugin
 
 plugins {
   id 'lifecycle-base'
@@ -37,7 +40,7 @@ plugins {
   id 'elasticsearch.formatting'
   id 'elasticsearch.local-distribution'
   id 'elasticsearch.fips'
-  id 'elasticsearch.testclusters'
+  id 'elasticsearch.internal-testclusters'
   id 'elasticsearch.run'
   id "com.diffplug.spotless" version "5.12.5" apply false
 }
@@ -110,6 +113,19 @@ subprojects {
     project.noticeFile = project.rootProject.file('NOTICE.txt')
   }
 
+  plugins.withType(InternalPluginBuildPlugin).whenPluginAdded {
+    project.dependencies {
+      compileOnly project(":server")
+      testImplementation project(":test:framework")
+    }
+  }
+
+  // Ultimately the RestTestBase Plugin should apply the InternalTestClusters Plugin itself instead of TestClusters
+  // but this requires major rework on the func test infrastructure.
+  // TODO: This will be addressed once we have https://github.com/elastic/elasticsearch/issues/71593 resolved
+  project.plugins.withType(RestTestBasePlugin) {
+    project.plugins.apply(InternalTestClustersPlugin)
+  }
 }
 
 /**
@@ -131,7 +147,6 @@ ext.testArtifact = { p, String name = "test" ->
   };
 }
 
-
 tasks.register("updateCIBwcVersions") {
   doLast {
     File yml = file(".ci/bwcVersions")

buildSrc/build.gradle

@@ -42,6 +42,156 @@ gradlePlugin {
   // We already configure publication and we don't need or want the one that comes
   // with the java-gradle-plugin
   automatedPublishing = false
+  plugins {
+    build {
+      id = 'elasticsearch.build'
+      implementationClass = 'org.elasticsearch.gradle.internal.BuildPlugin'
+    }
+    distributionDownload {
+      id = 'elasticsearch.distribution-download'
+      implementationClass = 'org.elasticsearch.gradle.DistributionDownloadPlugin'
+    }
+    distroTest {
+      id = 'elasticsearch.distro-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.DistroTestPlugin'
+    }
+    dockerSupport {
+      id = 'elasticsearch.docker-support'
+      implementationClass = 'org.elasticsearch.gradle.internal.docker.DockerSupportPlugin'
+    }
+    docsTest {
+      id = 'elasticsearch.docs-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.doc.DocsTestPlugin'
+    }
+    esPlugin {
+      id = 'elasticsearch.esplugin'
+      implementationClass = 'org.elasticsearch.gradle.plugin.PluginBuildPlugin'
+    }
+    globalBuildInfo {
+      id = 'elasticsearch.global-build-info'
+      implementationClass = 'org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin'
+    }
+    internalAvailablePorts {
+      id = 'elasticsearch.internal-available-ports'
+      implementationClass = 'org.elasticsearch.gradle.internal.InternalAvailableTcpPortProviderPlugin'
+    }
+    internalClusterTest {
+      id = 'elasticsearch.internal-cluster-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin'
+    }
+    internalDistributionArchiveCheck {
+      id = 'elasticsearch.internal-distribution-archive-check'
+      implementationClass = 'org.elasticsearch.gradle.internal.InternalDistributionArchiveCheckPlugin'
+    }
+    internalDistributionArchiveSetup {
+      id = 'elasticsearch.internal-distribution-archive-setup'
+      implementationClass = 'org.elasticsearch.gradle.internal.InternalDistributionArchiveSetupPlugin'
+    }
+    internalDistributionBwcSetup {
+      id = 'elasticsearch.internal-distribution-bwc-setup'
+      implementationClass = 'org.elasticsearch.gradle.internal.InternalDistributionBwcSetupPlugin'
+    }
+    internalDistributionDownload {
+      id = 'elasticsearch.internal-distribution-download'
+      implementationClass = 'org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin'
+    }
+    internalLicenseheaders {
+      id = 'elasticsearch.internal-licenseheaders'
+      implementationClass = 'org.elasticsearch.gradle.internal.precommit.LicenseHeadersPrecommitPlugin'
+    }
+    internalPlugin {
+      id = 'elasticsearch.internal-es-plugin'
+      implementationClass = 'org.elasticsearch.gradle.internal.InternalPluginBuildPlugin'
+    }
+    internalTestArtifact {
+      id = 'elasticsearch.internal-test-artifact'
+      implementationClass = 'org.elasticsearch.gradle.internal.InternalTestArtifactPlugin'
+    }
+    internalTestArtifactBase {
+      id = 'elasticsearch.internal-test-artifact-base'
+      implementationClass = 'org.elasticsearch.gradle.internal.InternalTestArtifactBasePlugin'
+    }
+    internalTestClusters {
+      id = 'elasticsearch.internal-testclusters'
+      implementationClass = 'org.elasticsearch.gradle.internal.InternalTestClustersPlugin'
+    }
+    internalTestRerun {
+      id = 'elasticsearch.internal-test-rerun'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rerun.TestRerunPlugin'
+    }
+    java {
+      id = 'elasticsearch.java'
+      implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin'
+    }
+    javaRestTest {
+      id = 'elasticsearch.java-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.JavaRestTestPlugin'
+    }
+    jdkDownload {
+      id = 'elasticsearch.jdk-download'
+      implementationClass = 'org.elasticsearch.gradle.internal.JdkDownloadPlugin'
+    }
+    publish {
+      id = 'elasticsearch.publish'
+      implementationClass = 'org.elasticsearch.gradle.internal.PublishPlugin'
+    }
+    reaper {
+      id = 'elasticsearch.reaper'
+      implementationClass = 'org.elasticsearch.gradle.ReaperPlugin'
+    }
+    repositories {
+      id = 'elasticsearch.repositories'
+      implementationClass = 'org.elasticsearch.gradle.internal.RepositoriesSetupPlugin'
+    }
+    restResources {
+      id = 'elasticsearch.rest-resources'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin'
+    }
+    restTest {
+      id = 'elasticsearch.rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.RestTestPlugin'
+    }
+    standaloneRestTest {
+      id = 'elasticsearch.standalone-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.StandaloneRestTestPlugin'
+    }
+    standaloneTest {
+      id = 'elasticsearch.standalone-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.StandaloneTestPlugin'
+    }
+    testFixtures {
+      id = 'elasticsearch.test.fixtures'
+      implementationClass = 'org.elasticsearch.gradle.internal.testfixtures.TestFixturesPlugin'
+    }
+    testBase {
+      id = 'elasticsearch.test-base'
+      implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin'
+    }
+    testWithDependencies {
+      id = 'elasticsearch.test-with-dependencies'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.TestWithDependenciesPlugin'
+    }
+    testWithSsl {
+      id = 'elasticsearch.test-with-ssl'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.TestWithSslPlugin'
+    }
+    testclusters {
+      id = 'elasticsearch.testclusters'
+      implementationClass = 'org.elasticsearch.gradle.testclusters.TestClustersPlugin'
+    }
+    validateRestSpec {
+      id = 'elasticsearch.validate-rest-spec'
+      implementationClass = 'org.elasticsearch.gradle.internal.precommit.ValidateRestSpecPlugin'
+    }
+    yamlRestCompatTest {
+      id = 'elasticsearch.yaml-rest-compat-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin'
+    }
+    yamlRestTest {
+      id = 'elasticsearch.yaml-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.YamlRestTestPlugin'
+    }
+  }
 }
 def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) {
   outputFile = "${buildDir}/version.properties"

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy

@@ -9,7 +9,6 @@
 package org.elasticsearch.gradle
 
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
-import org.elasticsearch.gradle.internal.VersionProperties
 import org.gradle.testkit.runner.TaskOutcome
 import spock.lang.Unroll
 
@@ -18,7 +17,7 @@ import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.with
 class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
     @Unroll
-    def "#distType version can be resolved"() {
+    def "extracted #distType version can be resolved"() {
         given:
         buildFile << applyPluginAndSetupDistro(version, platform)
 

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy

@@ -136,8 +136,6 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
                 description = "test module description"
             }
             
-            licenseFile = file('license.txt')
-            noticeFile = file('notice.txt')
             version = "1.0"
             group = 'org.acme'
         """

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy

@@ -10,7 +10,7 @@ package org.elasticsearch.gradle.fixtures
 
 import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.ElasticsearchDistribution
-import org.elasticsearch.gradle.internal.VersionProperties
+import org.elasticsearch.gradle.VersionProperties
 import org.gradle.testkit.runner.BuildResult
 import org.gradle.testkit.runner.GradleRunner
 

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPluginFuncTest.groovy

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.gradle.internal
 
-
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 import spock.lang.Unroll

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -9,28 +9,13 @@
 package org.elasticsearch.gradle.internal
 
 import org.elasticsearch.gradle.Architecture
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 
 
 class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
-    def "plugin application fails on non internal build"() {
-        given:
-        buildFile.text = """
-            plugins {
-             id 'elasticsearch.internal-distribution-download'
-            }
-        """
-
-        when:
-        def result = gradleRunner("tasks").buildAndFail()
-
-        then:
-        assertOutputContains(result.output, "Plugin 'elasticsearch.internal-distribution-download' is not supported. " +
-            "Use 'elasticsearch.distribution-download' plugin instead")
-    }
-
     def "resolves current version from local build"() {
         given:
         internalBuild()

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/PublishPluginFuncTest.groovy renamed to buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle
+package org.elasticsearch.gradle.internal
 
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
@@ -178,7 +178,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file('notice.txt') << "Notice file"
         buildFile << """
             plugins {
-                id 'elasticsearch.esplugin'
+                id 'elasticsearch.internal-es-plugin'
                 id 'elasticsearch.publish'
                 id 'com.github.johnrengelman.shadow'
             }
@@ -236,7 +236,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file('notice.txt') << "Notice file"
         buildFile << """
             plugins {
-                id 'elasticsearch.esplugin'
+                id 'elasticsearch.internal-es-plugin'
                 id 'elasticsearch.publish'
             }
             

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/RestResourcesPluginFuncTest.groovy renamed to buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle
+package org.elasticsearch.gradle.internal.test.rest
 
 import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
 import org.gradle.testkit.runner.TaskOutcome

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/YamlRestTestPluginFuncTest.groovy renamed to buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestTestPluginFuncTest.groovy

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle
+package org.elasticsearch.gradle.internal.test.rest
 
 import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
 import org.gradle.testkit.runner.TaskOutcome

buildSrc/src/main/groovy/elasticsearch.run.gradle

@@ -10,7 +10,7 @@ import org.elasticsearch.gradle.testclusters.RunTask
 
 // gradle has an open issue of failing applying plugins in
 // precompiled script plugins (see https://github.com/gradle/gradle/issues/17004)
-// apply plugin: 'elasticsearch.testclusters'
+// apply plugin: 'elasticsearch.internal-testclusters'
 
 testClusters {
   runTask {

buildSrc/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle

@@ -8,7 +8,7 @@
 
 import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.OS
-import org.elasticsearch.gradle.internal.VersionProperties
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 // gradle has an open issue of failing applying plugins in

buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy

@@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal
 
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.FixtureStop
+import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.internal.test.AntFixture
 import org.gradle.api.file.FileSystemOperations
 import org.gradle.api.tasks.Internal

buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy

@@ -9,7 +9,7 @@ package org.elasticsearch.gradle.internal.doc
 
 import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.VersionProperties
+import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.file.Directory
@@ -23,7 +23,7 @@ class DocsTestPlugin implements Plugin<Project> {
 
     @Override
     void apply(Project project) {
-        project.pluginManager.apply('elasticsearch.testclusters')
+        project.pluginManager.apply('elasticsearch.internal-testclusters')
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
         project.pluginManager.apply('elasticsearch.rest-test')
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -11,9 +11,9 @@ import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.internal.BuildPlugin
 import org.elasticsearch.gradle.internal.BwcVersions
-import org.elasticsearch.gradle.internal.LoggedExec
+import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.VersionProperties
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.test.Fixture
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -12,7 +12,7 @@ import com.sun.jna.Native
 import com.sun.jna.WString
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.VersionProperties
+import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.Project
 
 import java.nio.file.Files