Build: Consolidate archives and packages configuration (#28760)

This commit moves the distribution specific tasks into the respective
archives and packages builds. The collocation of common and distribution
specific tasks make it much easier to reason about what is expected in a
particular distribution.

Author: rjernst

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -119,7 +119,7 @@ class VagrantTestPlugin implements Plugin<Project> {
             } else {
                 it = "packages:${it}"
             }
-            project.dependencies.add(BATS, project.dependencies.project(path: ":distribution:${it}", configuration: 'archives'))
+            project.dependencies.add(BATS, project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
         }
 
         UPGRADE_FROM_ARCHIVES.each {

distribution/archives/build.gradle

@@ -17,70 +17,195 @@
  * under the License.
  */
 
-import org.apache.tools.ant.filters.FixCrLfFilter
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.apache.tools.ant.filters.FixCrLfFilter
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.EmptyDirTask
-import org.elasticsearch.gradle.ConcatFilesTask
 import org.elasticsearch.gradle.MavenFilteringHack
-import org.elasticsearch.gradle.NoticeTask
-import org.elasticsearch.gradle.precommit.DependencyLicensesTask
-import org.elasticsearch.gradle.precommit.UpdateShasTask
-import org.elasticsearch.gradle.test.RunTask
+import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 
-subprojects {
-  // CopySpec does not make it easy to create an empty director so we create the directory that we want, and then point CopySpec to its
-  // parent to copy to the root of the distribution
-  File logs = new File(buildDir, 'logs-hack/logs')
-  task createLogDir(type: EmptyDirTask) {
-    dir "${logs}"
-    dirMode 0755
-  }
-  File plugins = new File(buildDir, 'plugins-hack/plugins')
-  task createPluginsDir(type: EmptyDirTask) {
-    dir "${plugins}"
-    dirMode 0755
-  }
-  project.ext.archivesFiles = copySpec {
+// need this so Zip/Tar tasks get basic defaults...
+apply plugin: 'base'
+
+// CopySpec does not make it easy to create an empty directory so we
+// create the directory that we want, and then point CopySpec to its
+// parent to copy to the root of the distribution
+ext.logsDir = new File(buildDir, 'logs-hack/logs')
+task createLogsDir(type: EmptyDirTask) {
+  dir "${logsDir}"
+  dirMode 0755
+}
+ext.pluginsDir= new File(buildDir, 'plugins-hack/plugins')
+task createPluginsDir(type: EmptyDirTask) {
+  dir "${pluginsDir}"
+  dirMode 0755
+}
+
+CopySpec archiveFiles(CopySpec... innerFiles) {
+  return copySpec {
     into("elasticsearch-${version}") {
       with libFiles
       into('config') {
         dirMode 0750
         fileMode 0660
-        with configFiles
+        with configFiles('def')
       }
       into('bin') {
         with copySpec {
-          with binFiles
-          from('../../src/main/resources/bin') {
+          with binFiles('def')
+          from('../src/bin') {
             include '*.bat'
             filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
           }
-          MavenFilteringHack.filter(it, expansions)
+          MavenFilteringHack.filter(it, expansionsForDistribution('def'))
         }
       }
       into('') {
         from {
           dirMode 0755
-          logs.getParent()
+          logsDir.getParent()
         }
       }
       into('') {
         from {
           dirMode 0755
-          plugins.getParent()
+          pluginsDir.getParent()
         }
       }
       with commonFiles
       with noticeFile
-      from('../../src/main/resources') {
+      from('../src') {
         include 'bin/*.exe'
       }
-      if (project.name != 'integ-test-zip') {
-        with modulesFiles
-      } else {
-        with transportModulesFiles
+      for (CopySpec files : innerFiles) {
+        with files
       }
     }
   }
 }
+
+task buildIntegTestZip(type: Zip) {
+  dependsOn createLogsDir, createPluginsDir
+  destinationDir = file('integ-test-zip/build/distributions')
+  baseName = 'elasticsearch'
+  with archiveFiles(transportModulesFiles)
+}
+
+task buildZip(type: Zip) {
+  dependsOn createLogsDir, createPluginsDir
+  destinationDir = file('zip/build/distributions')
+  baseName = 'elasticsearch'
+  with archiveFiles(modulesFiles)
+}
+
+task buildTar(type: Tar) {
+  dependsOn createLogsDir, createPluginsDir
+  destinationDir = file('tar/build/distributions')
+  baseName = 'elasticsearch'
+  extension = 'tar.gz'
+  compression = Compression.GZIP
+  dirMode 0755
+  fileMode 0644
+  with archiveFiles(modulesFiles)
+}
+
+// This configures the default artifact for the distribution specific
+// subprojects. We have subprojects for two reasons:
+// 1. Gradle project substitutions can only bind to the default
+//    configuration of a project
+// 2. The integ-test-zip and zip distributions have the exact same
+//    filename, so they must be placed in different directories.
+subprojects {
+  apply plugin: 'distribution'
+
+  archivesBaseName = 'elasticsearch'
+
+  String buildTask = "build${it.name.replaceAll(/-[a-z]/) { it.substring(1).toUpperCase() }.capitalize()}"
+  ext.buildDist = parent.tasks.getByName(buildTask)
+  artifacts {
+    'default' buildDist
+  }
+}
+
+/*****************************************************************************
+ *                            Rest test config                               *
+ *****************************************************************************/
+subprojects {
+  apply plugin: 'elasticsearch.standalone-rest-test'
+  apply plugin: 'elasticsearch.rest-test'
+
+  if (project.name == 'integ-test-zip') {
+    integTest {
+      includePackaged true
+    }
+  }
+
+  integTestCluster {
+    dependsOn assemble
+    distribution = project.name
+  }
+  integTestRunner {
+    if (Os.isFamily(Os.FAMILY_WINDOWS) && System.getProperty('tests.timeoutSuite') == null) {
+      // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man
+      systemProperty 'tests.timeoutSuite', '1800000!'
+    }
+  }
+
+  processTestResources {
+    inputs.properties(project(':distribution').restTestExpansions)
+    MavenFilteringHack.filter(it, project(':distribution').restTestExpansions)
+  }
+}
+
+/*****************************************************************************
+ *                              Maven config                                 *
+ *****************************************************************************/
+configure(subprojects.findAll { it.name.contains('zip') }) {
+  // only zip distributions go to maven
+  BuildPlugin.configurePomGeneration(project)
+  apply plugin: 'nebula.info-scm'
+  apply plugin: 'nebula.maven-base-publish'
+  apply plugin: 'nebula.maven-scm'
+
+  // note: the group must be correct before applying the nexus plugin, or
+  // it will capture the wrong value...
+  project.group = "org.elasticsearch.distribution.${project.name}"
+
+  publishing {
+    publications {
+      nebula {
+        artifactId 'elasticsearch'
+        artifact buildDist
+      }
+      /*
+       * HUGE HACK: the underlying maven publication library refuses to
+       * deploy any attached artifacts when the packaging type is set to
+       * 'pom'. But Sonatype's OSS repositories require source files for
+       * artifacts that are of type 'zip'. We already publish the source
+       * and javadoc for Elasticsearch under the various other subprojects.
+       * So here we create another publication using the same name that
+       * has the "real" pom, and rely on the fact that gradle will execute
+       * the publish tasks in alphabetical order. This lets us publish the
+       * zip file and even though the pom says the type is 'pom' instead of
+       * 'zip'. We cannot setup a dependency between the tasks because the
+       * publishing tasks are created *extremely* late in the configuration
+       * phase, so that we cannot get ahold of the actual task. Furthermore,
+       * this entire hack only exists so we can make publishing to maven
+       * local work, since we publish to maven central externally.
+       */
+      nebulaRealPom(MavenPublication) {
+        artifactId 'elasticsearch'
+        pom.packaging = 'pom'
+        pom.withXml { XmlProvider xml ->
+          Node root = xml.asNode()
+          root.appendNode('name', 'Elasticsearch')
+          root.appendNode('description', 'A Distributed RESTful Search Engine')
+          root.appendNode('url', PluginBuildPlugin.urlFromOrigin(project.scminfo.origin))
+          Node scmNode = root.appendNode('scm')
+          scmNode.appendNode('url', project.scminfo.origin)
+        }
+      }
+    }
+  }
+}
+

distribution/archives/integ-test-zip/build.gradle

@@ -1,72 +1,2 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import org.elasticsearch.gradle.plugin.PluginBuildPlugin
-import org.apache.tools.ant.taskdefs.condition.Os
-
-task buildZip(type: Zip) {
-  dependsOn createLogDir, createPluginsDir
-  baseName = 'elasticsearch'
-  with archivesFiles
-}
-
-artifacts {
-  'default' buildZip
-  archives buildZip
-}
-
-publishing {
-  publications {
-    nebula {
-      artifactId 'elasticsearch'
-      artifact buildZip
-    }
-    /* HUGE HACK: the underlying maven publication library refuses to deploy any attached artifacts
-     * when the packaging type is set to 'pom'. But Sonatype's OSS repositories require source files
-     * for artifacts that are of type 'zip'. We already publish the source and javadoc for Elasticsearch
-     * under the various other subprojects. So here we create another publication using the same
-     * name that has the "real" pom, and rely on the fact that gradle will execute the publish tasks
-     * in alphabetical order. This lets us publish the zip file and even though the pom says the
-     * type is 'pom' instead of 'zip'. We cannot setup a dependency between the tasks because the
-     * publishing tasks are created *extremely* late in the configuration phase, so that we cannot get
-     * ahold of the actual task. Furthermore, this entire hack only exists so we can make publishing to
-     * maven local work, since we publish to maven central externally. */
-    nebulaRealPom(MavenPublication) {
-      artifactId 'elasticsearch'
-      pom.packaging = 'pom'
-      pom.withXml { XmlProvider xml ->
-        Node root = xml.asNode()
-        root.appendNode('name', 'Elasticsearch')
-        root.appendNode('description', 'A Distributed RESTful Search Engine')
-        root.appendNode('url', PluginBuildPlugin.urlFromOrigin(project.scminfo.origin))
-        Node scmNode = root.appendNode('scm')
-        scmNode.appendNode('url', project.scminfo.origin)
-      }
-    }
-  }
-}
-
-integTestRunner {
-  if (Os.isFamily(Os.FAMILY_WINDOWS) && System.getProperty('tests.timeoutSuite') == null) {
-    // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man
-    systemProperty 'tests.timeoutSuite', '1800000!'
-  }
-}
-
-integTest.dependsOn buildZip
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/archives/tar/build.gradle

@@ -1,33 +1,2 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-task buildTar(type: Tar) {
-  dependsOn createLogDir, createPluginsDir
-  baseName = 'elasticsearch'
-  extension = 'tar.gz'
-  with archivesFiles
-  compression = Compression.GZIP
-  dirMode 0755
-  fileMode 0644
-}
-
-artifacts {
-  'default' buildTar
-}
-
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.