Make xpack modules instead of a meta plugin (#30589)

This commit removes xpack from being a meta-plugin-as-a-module.
It also fixes a couple tests which were missing task dependencies, which
failed once the gradle execution order changed.

Author: rjernst

distribution/archives/build.gradle

@@ -224,7 +224,7 @@ subprojects {
       doLast {
         // this is just a small sample from the C++ notices, the idea being that if we've added these lines we've probably added all the required lines
         final List<String> expectedLines = Arrays.asList("Apache log4cxx", "Boost Software License - Version 1.0 - August 17th, 2003")
-        final Path noticePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/modules/x-pack/x-pack-ml/NOTICE.txt")
+        final Path noticePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/modules/x-pack-ml/NOTICE.txt")
         final List<String> actualLines = Files.readAllLines(noticePath)
         for (final String expectedLine : expectedLines) {
           if (actualLines.contains(expectedLine) == false) {

distribution/build.gradle

@@ -201,17 +201,14 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each {
 
 // use licenses from each of the bundled xpack plugins
 Project xpack = project(':x-pack:plugin')
-xpack.subprojects.findAll { it.name != 'bwc' }.each { Project xpackSubproject ->
-  File licenses = new File(xpackSubproject.projectDir, 'licenses')
+xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
+  File licenses = new File(xpackModule.projectDir, 'licenses')
   if (licenses.exists()) {
     buildDefaultNotice.licensesDir licenses 
   }
+  copyModule(processDefaultOutputs, xpackModule)
+  copyLog4jProperties(buildDefaultLog4jConfig, xpackModule)
 }
-// but copy just the top level meta plugin to the default modules
-copyModule(processDefaultOutputs, xpack)
-copyLog4jProperties(buildDefaultLog4jConfig, xpack)
-
-// 
 
 // make sure we have a clean task since we aren't a java project, but we have tasks that
 // put stuff in the build dir

x-pack/plugin/build.gradle

@@ -8,17 +8,11 @@ import java.nio.file.Path
 import java.nio.file.StandardCopyOption
 import org.elasticsearch.gradle.test.RunTask;
 
-apply plugin: 'elasticsearch.es-meta-plugin'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
 
 archivesBaseName = 'x-pack'
 
-es_meta_plugin {
-  name = 'x-pack'
-  description = 'Elasticsearch Expanded Pack Plugin'
-  plugins = ['core', 'deprecation', 'graph', 'logstash',
-             'ml', 'monitoring', 'security', 'upgrade', 'watcher', 'sql', 'rollup']
-}
-
 dependencies {
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }

x-pack/plugin/core/src/main/bin/x-pack-env

@@ -5,4 +5,4 @@
 # you may not use this file except in compliance with the Elastic License.
 
 # include x-pack-core jars in classpath
-ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack/x-pack-core/*"
+ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack-core/*"

x-pack/plugin/core/src/main/bin/x-pack-env.bat

@@ -2,4 +2,4 @@ rem Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 rem or more contributor license agreements. Licensed under the Elastic License;
 rem you may not use this file except in compliance with the Elastic License.
 
-set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack/x-pack-core/*
+set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack-core/*

x-pack/plugin/security/src/main/bin/x-pack-security-env

@@ -7,4 +7,4 @@
 source "`dirname "$0"`"/x-pack-env
 
 # include x-pack-security jars in classpath
-ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack/x-pack-security/*"
+ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack-security/*"

x-pack/plugin/security/src/main/bin/x-pack-security-env.bat

@@ -4,4 +4,4 @@ rem you may not use this file except in compliance with the Elastic License.
 
 call "%~dp0x-pack-env.bat" || exit /b 1
 
-set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack/x-pack-security/*
+set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack-security/*

x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env

@@ -7,4 +7,4 @@
 source "`dirname "$0"`"/x-pack-env
 
 # include x-pack-security jars in classpath
-ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack/x-pack-watcher/*"
+ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/modules/x-pack-watcher/*"

x-pack/plugin/watcher/src/main/bin/x-pack-watcher-env.bat

@@ -4,4 +4,4 @@ rem you may not use this file except in compliance with the Elastic License.
 
 call "%~dp0x-pack-env.bat" || exit /b 1
 
-set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack/x-pack-watcher/*
+set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/modules/x-pack-watcher/*

x-pack/qa/vagrant/build.gradle

@@ -11,41 +11,6 @@ esvagrant {
 }
 
 dependencies {
-    // Packaging tests use the x-pack meta plugin
-    packaging project(path: xpackProject('plugin').path, configuration: 'zip')
-
     // Inherit Bats test utils from :qa:vagrant project
     packaging project(path: ':qa:vagrant', configuration: 'packaging')
 }
-
-Map<String, List<String>> metaPlugins = [:]
-for (Project metaPlugin : project.rootProject.subprojects) {
-    if (metaPlugin.plugins.hasPlugin(MetaPluginBuildPlugin)) {
-        MetaPluginPropertiesExtension extension = metaPlugin.extensions.findByName('es_meta_plugin')
-        if (extension != null) {
-            List<String> plugins = []
-            metaPlugin.subprojects.each {
-                if (extension.plugins.contains(it.name)) {
-                    Project plugin = (Project) it
-                    if (plugin.plugins.hasPlugin(PluginBuildPlugin)) {
-                        PluginPropertiesExtension esplugin = plugin.extensions.findByName('esplugin')
-                        if (esplugin != null) {
-                            plugins.add(esplugin.name)
-                        }
-                    }
-                }
-            }
-            metaPlugins.put(extension.name, plugins.toSorted())
-        }
-    }
-}
-
-setupPackagingTest {
-    doLast {
-        metaPlugins.each{ name, plugins ->
-            File expectedMetaPlugins = file("build/plugins/${name}.expected")
-            expectedMetaPlugins.parentFile.mkdirs()
-            expectedMetaPlugins.setText(plugins.join('\n'), 'UTF-8')
-        }
-    }
-}

x-pack/qa/vagrant/src/test/resources/packaging/utils/xpack.bash

@@ -69,15 +69,6 @@ verify_xpack_installation() {
     done
     # nocommit: decide whether to check the files added by the distribution, not part of xpack...
     #assert_number_of_files "$ESCONFIG/" $configFilesCount
-
-    # Read the $name.expected file that contains all the expected
-    # plugins for the meta plugin
-    while read plugin; do
-        assert_module_or_plugin_directory "$ESMODULES/$name/$plugin"
-        assert_file_exist "$ESMODULES/$name/$plugin/$plugin"*".jar"
-        assert_file_exist "$ESMODULES/$name/$plugin/plugin-descriptor.properties"
-        assert_file_exist "$ESMODULES/$name/$plugin/plugin-security.policy"
-    done </project/build/plugins/$name.expected
 }
 
 assert_number_of_files() {