Consolidate REST API specifications and publish under Apache 2.0 license (#70940)

Author: mark-vieira

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/RestResourcesPluginFuncTest.groovy

@@ -52,7 +52,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
         file("/build/restResources/yamlSpecs/rest-api-spec/api/" + api).exists()
     }
 
-    def "restResources copies all core API (but not x-pack) by default for projects with copied tests"() {
+    def "restResources copies all API by default for projects with copied tests"() {
         given:
         internalBuild()
         buildFile << """
@@ -71,7 +71,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
         String apiXpack = "xpack.json"
         String coreTest = "foo/10_basic.yml"
         String xpackTest = "bar/10_basic.yml"
-        setupRestResources([apiCore1, apiCore2], [coreTest], [apiXpack], [xpackTest])
+        setupRestResources([apiCore1, apiCore2, apiXpack], [coreTest], [xpackTest])
         // intentionally not adding tests to project, they will be copied over via the plugin
         // this tests that the test copy happens before the api copy since the api copy will only trigger if there are tests in the project
 
@@ -83,7 +83,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
         result.task(':copyYamlTestsTask').outcome == TaskOutcome.SUCCESS
         file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiCore1).exists()
         file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiCore2).exists()
-        file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiXpack).exists() == false //x-pack specs must be explicitly configured
+        file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiXpack).exists()
         file("/build/restResources/yamlTests/rest-api-spec/test/" + coreTest).exists()
         file("/build/restResources/yamlTests/rest-api-spec/test/" + xpackTest).exists()
     }
@@ -97,16 +97,15 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
 
             restResources {
                 restApi {
-                    includeCore 'foo'
-                    includeXpack 'xpackfoo'
+                    include 'foo', 'xpackfoo'
                 }
             }
         """
         String apiFoo = "foo.json"
         String apiXpackFoo = "xpackfoo.json"
         String apiBar = "bar.json"
         String apiXpackBar = "xpackbar.json"
-        setupRestResources([apiFoo, apiBar], [], [apiXpackFoo, apiXpackBar])
+        setupRestResources([apiFoo, apiBar, apiXpackFoo, apiXpackBar])
         addRestTestsToProject(["10_basic.yml"])
 
         when:
@@ -130,8 +129,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
 
             restResources {
                 restApi {
-                    includeCore '*'
-                    includeXpack '*'
+                    include '*'
                 }
                 restTests {
                     includeCore 'foo'
@@ -144,7 +142,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
         String apiXpack = "xpack.json"
         String coreTest = "foo/10_basic.yml"
         String xpackTest = "bar/10_basic.yml"
-        setupRestResources([apiCore1, apiCore2], [coreTest], [apiXpack], [xpackTest])
+        setupRestResources([apiCore1, apiCore2, apiXpack], [coreTest], [xpackTest])
         // intentionally not adding tests to project, they will be copied over via the plugin
         // this tests that the test copy happens before the api copy since the api copy will only trigger if there are tests in the project
 

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy

@@ -11,37 +11,32 @@ package org.elasticsearch.gradle.fixtures;
 
 abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest {
 
-    void setupRestResources(List<String> apis, List<String> tests = [], List<String> xpackApis = [], List<String> xpackTests = []) {
+    void setupRestResources(List<String> apis, List<String> tests = [], List<String> xpackTests = []) {
         addSubProject(":test:framework") << "apply plugin: 'elasticsearch.java'"
         addSubProject(":distribution:archives:integ-test-zip") << "configurations { extracted }"
         addSubProject(":rest-api-spec") << """
         configurations { restSpecs\nrestTests }
         artifacts {
           restSpecs(new File(projectDir, "src/main/resources/rest-api-spec/api"))
-          restTests(new File(projectDir, "src/main/resources/rest-api-spec/test"))
+          restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
         }
         """
         addSubProject(":x-pack:plugin") << """
         configurations { restXpackSpecs\nrestXpackTests }
         artifacts {
-          //The api and tests need to stay at src/test/... since some external tooling depends on that exact file path.
-          restXpackSpecs(new File(projectDir, "src/test/resources/rest-api-spec/api"))
-          restXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test"))
+          restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
         }
         """
 
-        xpackApis.each { api ->
-            file("x-pack/plugin/src/test/resources/rest-api-spec/api/" + api) << ""
-        }
         xpackTests.each { test ->
-            file("x-pack/plugin/src/test/resources/rest-api-spec/test/" + test) << ""
+            file("x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
         }
 
         apis.each { api ->
             file("rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
         }
         tests.each { test ->
-            file("rest-api-spec/src/main/resources/rest-api-spec/test/" + test) << ""
+            file("rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java

@@ -47,22 +47,18 @@
 public class CopyRestApiTask extends DefaultTask {
     private static final String REST_API_PREFIX = "rest-api-spec/api";
     private static final String REST_TEST_PREFIX = "rest-api-spec/test";
-    private final ListProperty<String> includeCore;
-    private final ListProperty<String> includeXpack;
+    private final ListProperty<String> include;
     private final DirectoryProperty outputResourceDir;
     private final DirectoryProperty additionalYamlTestsDir;
 
     private File sourceResourceDir;
     private boolean skipHasRestTestCheck;
-    private FileCollection coreConfig;
-    private FileCollection xpackConfig;
+    private FileCollection config;
     private FileCollection additionalConfig;
-    private Function<FileCollection, FileTree> coreConfigToFileTree = FileCollection::getAsFileTree;
-    private Function<FileCollection, FileTree> xpackConfigToFileTree = FileCollection::getAsFileTree;
+    private Function<FileCollection, FileTree> configToFileTree = FileCollection::getAsFileTree;
     private Function<FileCollection, FileTree> additionalConfigToFileTree = FileCollection::getAsFileTree;
 
-    private final PatternFilterable corePatternSet;
-    private final PatternFilterable xpackPatternSet;
+    private final PatternFilterable patternSet;
     private final ProjectLayout projectLayout;
     private final FileSystemOperations fileSystemOperations;
     private final ArchiveOperations archiveOperations;
@@ -75,25 +71,18 @@ public CopyRestApiTask(
         ArchiveOperations archiveOperations,
         ObjectFactory objectFactory
     ) {
-        this.includeCore = objectFactory.listProperty(String.class);
-        this.includeXpack = objectFactory.listProperty(String.class);
+        this.include = objectFactory.listProperty(String.class);
         this.outputResourceDir = objectFactory.directoryProperty();
         this.additionalYamlTestsDir = objectFactory.directoryProperty();
-        this.corePatternSet = patternSetFactory.create();
-        this.xpackPatternSet = patternSetFactory.create();
+        this.patternSet = patternSetFactory.create();
         this.projectLayout = projectLayout;
         this.fileSystemOperations = fileSystemOperations;
         this.archiveOperations = archiveOperations;
     }
 
     @Input
-    public ListProperty<String> getIncludeCore() {
-        return includeCore;
-    }
-
-    @Input
-    public ListProperty<String> getIncludeXpack() {
-        return includeXpack;
+    public ListProperty<String> getInclude() {
+        return include;
     }
 
     @Input
@@ -105,29 +94,22 @@ public boolean isSkipHasRestTestCheck() {
     @InputFiles
     public FileTree getInputDir() {
         FileTree coreFileTree = null;
-        FileTree xpackFileTree = null;
-        if (includeXpack.get().isEmpty() == false) {
-            xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
-            xpackFileTree = xpackConfigToFileTree.apply(xpackConfig).matching(xpackPatternSet);
-        }
         boolean projectHasYamlRestTests = skipHasRestTestCheck || projectHasYamlRestTests();
-        if (includeCore.get().isEmpty() == false || projectHasYamlRestTests) {
+        if (include.get().isEmpty() == false || projectHasYamlRestTests) {
             if (BuildParams.isInternal()) {
-                corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
-                coreFileTree = coreConfigToFileTree.apply(coreConfig).matching(corePatternSet); // directory on disk
+                patternSet.setIncludes(include.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
+                coreFileTree = configToFileTree.apply(config).matching(patternSet); // directory on disk
             } else {
-                coreFileTree = coreConfig.getAsFileTree(); // jar file
+                coreFileTree = config.getAsFileTree(); // jar file
             }
         }
 
         FileCollection fileCollection = additionalConfig == null
-            ? projectLayout.files(coreFileTree, xpackFileTree)
-            : projectLayout.files(coreFileTree, xpackFileTree, additionalConfigToFileTree.apply(additionalConfig));
+            ? coreFileTree
+            : projectLayout.files(coreFileTree, additionalConfigToFileTree.apply(additionalConfig));
 
         // if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input
-        return projectHasYamlRestTests || includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false
-            ? fileCollection.getAsFileTree()
-            : null;
+        return projectHasYamlRestTests || include.get().isEmpty() == false ? fileCollection.getAsFileTree() : null;
     }
 
     @OutputDirectory
@@ -152,9 +134,9 @@ void copy() {
         if (BuildParams.isInternal()) {
             getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", projectPath);
             fileSystemOperations.copy(c -> {
-                c.from(coreConfigToFileTree.apply(coreConfig));
+                c.from(configToFileTree.apply(config));
                 c.into(restSpecOutputDir);
-                c.include(corePatternSet.getIncludes());
+                c.include(patternSet.getIncludes());
             });
         } else {
             getLogger().debug(
@@ -163,26 +145,15 @@ void copy() {
                 VersionProperties.getElasticsearch()
             );
             fileSystemOperations.copy(c -> {
-                c.from(archiveOperations.zipTree(coreConfig.getSingleFile())); // jar file
+                c.from(archiveOperations.zipTree(config.getSingleFile())); // jar file
                 c.into(outputResourceDir);
-                if (includeCore.get().isEmpty()) {
+                if (include.get().isEmpty()) {
                     c.include(REST_API_PREFIX + "/**");
                 } else {
-                    c.include(
-                        includeCore.get().stream().map(prefix -> REST_API_PREFIX + "/" + prefix + "*/**").collect(Collectors.toList())
-                    );
+                    c.include(include.get().stream().map(prefix -> REST_API_PREFIX + "/" + prefix + "*/**").collect(Collectors.toList()));
                 }
             });
         }
-        // only copy x-pack specs if explicitly instructed
-        if (includeXpack.get().isEmpty() == false) {
-            getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", projectPath);
-            fileSystemOperations.copy(c -> {
-                c.from(xpackConfigToFileTree.apply(xpackConfig));
-                c.into(restSpecOutputDir);
-                c.include(xpackPatternSet.getIncludes());
-            });
-        }
 
         // copy any additional config
         if (additionalConfig != null) {
@@ -222,38 +193,24 @@ public void setSkipHasRestTestCheck(boolean skipHasRestTestCheck) {
         this.skipHasRestTestCheck = skipHasRestTestCheck;
     }
 
-    public void setCoreConfig(FileCollection coreConfig) {
-        this.coreConfig = coreConfig;
-    }
-
-    public void setXpackConfig(FileCollection xpackConfig) {
-        this.xpackConfig = xpackConfig;
+    public void setConfig(FileCollection config) {
+        this.config = config;
     }
 
     public void setAdditionalConfig(FileCollection additionalConfig) {
         this.additionalConfig = additionalConfig;
     }
 
-    public void setCoreConfigToFileTree(Function<FileCollection, FileTree> coreConfigToFileTree) {
-        this.coreConfigToFileTree = coreConfigToFileTree;
-    }
-
-    public void setXpackConfigToFileTree(Function<FileCollection, FileTree> xpackConfigToFileTree) {
-        this.xpackConfigToFileTree = xpackConfigToFileTree;
+    public void setConfigToFileTree(Function<FileCollection, FileTree> configToFileTree) {
+        this.configToFileTree = configToFileTree;
     }
 
     public void setAdditionalConfigToFileTree(Function<FileCollection, FileTree> additionalConfigToFileTree) {
         this.additionalConfigToFileTree = additionalConfigToFileTree;
     }
 
     @Internal
-    public FileCollection getCoreConfig() {
-        return coreConfig;
+    public FileCollection getConfig() {
+        return config;
     }
-
-    @Internal
-    public FileCollection getXpackConfig() {
-        return xpackConfig;
-    }
-
 }

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java

@@ -19,29 +19,58 @@
  */
 public class RestResourcesExtension {
 
-    final RestResourcesSpec restApi;
-    final RestResourcesSpec restTests;
+    private final RestResourcesSpec restApi;
+    private final XpackRestResourcesSpec restTests;
 
     @Inject
     public RestResourcesExtension(ObjectFactory objects) {
         restApi = new RestResourcesSpec(objects);
-        restTests = new RestResourcesSpec(objects);
+        restTests = new XpackRestResourcesSpec(objects);
     }
 
     void restApi(Action<? super RestResourcesSpec> spec) {
         spec.execute(restApi);
     }
 
-    void restTests(Action<? super RestResourcesSpec> spec) {
+    void restTests(Action<? super XpackRestResourcesSpec> spec) {
+        if (BuildParams.isInternal() == false) {
+            // TODO: Separate this out into an "internal" plugin so we don't even expose this API to external folks
+            throw new UnsupportedOperationException("Including tests is not supported from external builds.");
+        }
         spec.execute(restTests);
     }
 
-    static class RestResourcesSpec {
+    public RestResourcesSpec getRestApi() {
+        return restApi;
+    }
+
+    public XpackRestResourcesSpec getRestTests() {
+        return restTests;
+    }
+
+    public static class RestResourcesSpec {
+
+        private final ListProperty<String> include;
+
+        RestResourcesSpec(ObjectFactory objects) {
+            include = objects.listProperty(String.class);
+        }
+
+        public void include(String... include) {
+            this.include.addAll(include);
+        }
+
+        public ListProperty<String> getInclude() {
+            return include;
+        }
+    }
+
+    public static class XpackRestResourcesSpec {
 
         private final ListProperty<String> includeCore;
         private final ListProperty<String> includeXpack;
 
-        RestResourcesSpec(ObjectFactory objects) {
+        XpackRestResourcesSpec(ObjectFactory objects) {
             includeCore = objects.listProperty(String.class);
             includeXpack = objects.listProperty(String.class);
         }
@@ -51,9 +80,6 @@ public void includeCore(String... include) {
         }
 
         public void includeXpack(String... include) {
-            if (BuildParams.isInternal() == false) {
-                throw new IllegalStateException("Can not include x-pack rest resources from an external build.");
-            }
             this.includeXpack.addAll(include);
         }