[7.x] Add rolling upgrade tests for component and composable templates (#58867)

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java

@@ -11,6 +11,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
@@ -19,6 +20,13 @@
 @TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
 public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    protected static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
+
+    @Override
+    protected Version overwriteEsVersion(Version esVersionFromApi) {
+        return UPGRADE_FROM_VERSION;
+    }
+
     @Override
     protected boolean preserveIndicesUponCompletion() {
         return true;

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml

@@ -80,3 +80,32 @@
        index: queries
        include_type_name: false
   - match: { queries.mappings.properties.id.type: "keyword" }
+
+---
+"Component and composable templates can be retrieved":
+  - skip:
+      version: " - 7.12.99"
+      reason: "#58969 only available from 7.13.0"
+  - do:
+      cluster.get_component_template:
+        name: my-ct
+
+  - match: {component_templates.0.name: my-ct}
+  - match: {component_templates.0.component_template.version: 2}
+  - match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}}
+  - match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}}
+  - is_true: component_templates.0.component_template.template.mappings
+  - match: {component_templates.0.component_template.template.aliases: {aliasname: {}}}
+
+  - do:
+      indices.get_index_template:
+        name: my-it
+
+  - match: {index_templates.0.index_template.index_patterns: ["test-*"]}
+  - match: {index_templates.0.index_template.template.settings.index: {number_of_shards: '1', number_of_replicas: '0'}}
+  - is_true: index_templates.0.index_template.template.mappings
+  - length: {index_templates.0.index_template.template.aliases: 3}
+  - is_true: index_templates.0.index_template.template.aliases.test_alias
+  - match: {index_templates.0.index_template.template.aliases.test_blias.index_routing: "b" }
+  - match: {index_templates.0.index_template.template.aliases.test_blias.search_routing: "b" }
+  - match: {index_templates.0.index_template.template.aliases.test_clias.filter.term.user: "kimchy" }

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml

@@ -204,3 +204,53 @@
       tasks.get:
         wait_for_completion: true
         task_id: $task
+
+---
+"Component and composable template validation":
+  - skip:
+      version: " - 7.12.99"
+      reason: "#58969 only available from 7.13.0"
+  - do:
+      cluster.put_component_template:
+        name: my-ct
+        body:
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+            mappings:
+              dynamic: false
+            aliases:
+              aliasname: {}
+          version: 2
+          _meta:
+            foo: bar
+            baz:
+              eggplant: true
+
+  - do:
+      indices.put_index_template:
+        name: my-it
+        body:
+          index_patterns: [test-*]
+          composed_of: ["my-ct"]
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+            mappings:
+              properties:
+                field:
+                  type: text
+            aliases:
+              test_alias: {}
+              test_blias: { routing: b }
+              test_clias: { filter: { term: { user: kimchy }}}
+
+  - do:
+      cluster.get_component_template:
+        name: my-ct
+
+  - do:
+      indices.get_index_template:
+        name: my-it

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml

@@ -133,3 +133,60 @@
         wait_for_completion: true
         task_id: $task_id
   - match: { task.headers.X-Opaque-Id: "Reindexing Again" }
+
+---
+"Component and composable templates can be retrieved and updated":
+  - skip:
+      version: " - 7.12.99"
+      reason: "#58969 only available from 7.13.0"
+  - do:
+      cluster.get_component_template:
+        name: my-ct
+
+  - match: {component_templates.0.name: my-ct}
+  - match: {component_templates.0.component_template.version: 2}
+  - match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}}
+  - match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}}
+  - match: {component_templates.0.component_template.template.mappings: {dynamic: false}}
+  - match: {component_templates.0.component_template.template.aliases: {aliasname: {}}}
+
+  - do:
+      indices.get_index_template:
+        name: my-it
+
+  - match: {index_templates.0.index_template.index_patterns: ["test-*"]}
+  - match: {index_templates.0.index_template.template.settings.index: {number_of_shards: '1', number_of_replicas: '0'}}
+  - is_true: index_templates.0.index_template.template.mappings
+  - length: {index_templates.0.index_template.template.aliases: 3}
+  - is_true: index_templates.0.index_template.template.aliases.test_alias
+  - match: {index_templates.0.index_template.template.aliases.test_blias.index_routing: "b" }
+  - match: {index_templates.0.index_template.template.aliases.test_blias.search_routing: "b" }
+  - match: {index_templates.0.index_template.template.aliases.test_clias.filter.term.user: "kimchy" }
+  - do:
+      cluster.put_component_template:
+        name: my-ct
+        body:
+          template:
+            settings:
+              number_of_shards:   1
+              number_of_replicas: 0
+            mappings:
+              dynamic: true
+            aliases:
+              aliasname: {}
+          version: 2
+          _meta:
+            foo: bar
+            baz:
+              eggplant: true
+
+  - do:
+      cluster.get_component_template:
+        name: my-ct
+
+  - match: {component_templates.0.name: my-ct}
+  - match: {component_templates.0.component_template.version: 2}
+  - match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}}
+  - match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}}
+  - is_true: component_templates.0.component_template.template.mappings
+  - match: {component_templates.0.component_template.template.aliases: {aliasname: {}}}

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -189,7 +189,7 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean
         }
 
         CompressedXContent mappings = template.template().mappings();
-        String stringMappings = mappings == null ? null : mappings.string();
+        String stringMappings = wrapMappingsIfNecessary(mappings == null ? null : mappings.string(), xContentRegistry);
 
         // We may need to normalize index settings, so do that also
         Settings finalSettings = template.template().settings();
@@ -228,17 +228,6 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean
             }
         }
 
-        // Mappings in component templates don't include _doc, so update the mappings to include this single type
-        if (stringMappings != null) {
-            Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
-            if (parsedMappings.size() > 0) {
-                stringMappings = Strings.toString(XContentFactory.jsonBuilder()
-                    .startObject()
-                    .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
-                    .endObject());
-            }
-        }
-
         final Template finalTemplate = new Template(finalSettings,
             stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases());
         final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata());
@@ -282,6 +271,35 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean
             .build();
     }
 
+    @Nullable
+    private static String wrapMappingsIfNecessary(@Nullable String mappings, NamedXContentRegistry xContentRegistry) throws Exception {
+        // Mappings in templates don't have to include _doc, so update
+        // the mappings to include this single type if necessary
+
+        String stringMappings = mappings;
+        if (stringMappings != null) {
+            Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
+            if (parsedMappings.size() > 0) {
+                if (parsedMappings.size() == 1) {
+                    final String keyName = parsedMappings.keySet().iterator().next();
+                    // Check if it's already wrapped in `_doc`, only rewrap if needed
+                    if (MapperService.SINGLE_MAPPING_NAME.equals(keyName) == false) {
+                        stringMappings = Strings.toString(XContentFactory.jsonBuilder()
+                            .startObject()
+                            .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
+                            .endObject());
+                    }
+                } else {
+                    stringMappings = Strings.toString(XContentFactory.jsonBuilder()
+                        .startObject()
+                        .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
+                        .endObject());
+                }
+            }
+        }
+        return stringMappings;
+    }
+
     /**
      * Remove the given component template from the cluster state. The component template name
      * supports simple regex wildcards for removing multiple component templates at a time.
@@ -486,18 +504,8 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo
             // If an inner template was specified, its mappings may need to be
             // adjusted (to add _doc) and it should be validated
             CompressedXContent mappings = innerTemplate.mappings();
-            String stringMappings = mappings == null ? null : mappings.string();
+            String stringMappings = wrapMappingsIfNecessary(mappings == null ? null : mappings.string(), xContentRegistry);
 
-            // Mappings in index templates don't include _doc, so update the mappings to include this single type
-            if (stringMappings != null) {
-                Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
-                if (parsedMappings.size() > 0) {
-                    stringMappings = Strings.toString(XContentFactory.jsonBuilder()
-                        .startObject()
-                        .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
-                        .endObject());
-                }
-            }
             final Template finalTemplate = new Template(finalSettings,
                 stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases());
             finalIndexTemplate = new ComposableIndexTemplate(template.indexPatterns(), finalTemplate, template.composedOf(),

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java

@@ -123,7 +123,7 @@ public void initAndResetContext() throws Exception {
             validateSpec(restSpec);
             final List<HttpHost> hosts = getClusterHosts();
             Tuple<Version, Version> versionVersionTuple = readVersionsFromCatNodes(adminClient());
-            final Version esVersion = versionVersionTuple.v1();
+            final Version esVersion = overwriteEsVersion(versionVersionTuple.v1());
             final Version masterVersion = versionVersionTuple.v2();
             final String os = readOsFromNodesInfo(adminClient());
 
@@ -157,6 +157,17 @@ public void initAndResetContext() throws Exception {
         restTestExecutionContext.clear();
     }
 
+    /**
+     * Allows test suites to return another version then the lowest version returned from the cat node api.
+     *
+     * For example, in rolling upgrade bwc tests, the version of the old cluster should be used, and if
+     * the cluster is fully upgraded then the cat api doesn't return the version of the old cluster. These
+     * tests provide the es version via system property.
+     */
+    protected Version overwriteEsVersion(Version esVersionFromApi) {
+        return esVersionFromApi;
+    }
+
     protected ClientYamlTestClient initClientYamlTestClient(
             final ClientYamlSuiteRestSpec restSpec,
             final RestClient restClient,