From a4a8f08aa39e31eb7619c1c13eca2d0e2c6fb96a Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 2 Jul 2020 14:13:35 -0600 Subject: [PATCH 1/4] Add rolling upgrade tests for component and composable templates (#58867) This adds rolling upgrade tests that component and composable templates can be read from older versions of the cluster. Relates to #58643 --- .../test/mixed_cluster/10_basic.yml | 26 +++++++++ .../test/old_cluster/10_basic.yml | 46 ++++++++++++++++ .../test/upgraded_cluster/10_basic.yml | 52 ++++++++++++++++++ .../MetadataIndexTemplateService.java | 54 +++++++++++-------- 4 files changed, 155 insertions(+), 23 deletions(-) diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index 1956cd56e6850..945ca80bc876a 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -80,3 +80,29 @@ index: queries include_type_name: false - match: { queries.mappings.properties.id.type: "keyword" } + +--- +"Component and composable templates can be retrieved": + - do: + cluster.get_component_template: + name: my-ct + + - match: {component_templates.0.name: my-ct} + - match: {component_templates.0.component_template.version: 2} + - match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}} + - match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}} + - is_true: component_templates.0.component_template.template.mappings + - match: {component_templates.0.component_template.template.aliases: {aliasname: {}}} + + - do: + indices.get_index_template: + name: my-it + + - match: {index_templates.0.index_template.index_patterns: ["test-*"]} + - match: {index_templates.0.index_template.template.settings.index: {number_of_shards: '1', number_of_replicas: '0'}} + - is_true: index_templates.0.index_template.template.mappings + - length: {index_templates.0.index_template.template.aliases: 3} + - is_true: index_templates.0.index_template.template.aliases.test_alias + - match: {index_templates.0.index_template.template.aliases.test_blias.index_routing: "b" } + - match: {index_templates.0.index_template.template.aliases.test_blias.search_routing: "b" } + - match: {index_templates.0.index_template.template.aliases.test_clias.filter.term.user: "kimchy" } diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml index e1ffcea930a42..a69d7d735719b 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml @@ -205,3 +205,49 @@ wait_for_completion: true task_id: $task +--- +"Component and composable template validation": + - do: + cluster.put_component_template: + name: my-ct + body: + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + dynamic: false + aliases: + aliasname: {} + version: 2 + _meta: + foo: bar + baz: + eggplant: true + + - do: + indices.put_index_template: + name: my-it + body: + index_patterns: [test-*] + composed_of: ["my-ct"] + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + field: + type: text + aliases: + test_alias: {} + test_blias: { routing: b } + test_clias: { filter: { term: { user: kimchy }}} + + - do: + cluster.get_component_template: + name: my-ct + + - do: + indices.get_index_template: + name: my-it diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 4368ffd602f58..9cb6ee7a91eca 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -126,4 +126,56 @@ task_id: $task_id - match: { task.headers.X-Opaque-Id: "Reindexing Again" } +--- +"Component and composable templates can be retrieved and updated": + - do: + cluster.get_component_template: + name: my-ct + + - match: {component_templates.0.name: my-ct} + - match: {component_templates.0.component_template.version: 2} + - match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}} + - match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}} + - match: {component_templates.0.component_template.template.mappings: {dynamic: false}} + - match: {component_templates.0.component_template.template.aliases: {aliasname: {}}} + + - do: + indices.get_index_template: + name: my-it + + - match: {index_templates.0.index_template.index_patterns: ["test-*"]} + - match: {index_templates.0.index_template.template.settings.index: {number_of_shards: '1', number_of_replicas: '0'}} + - is_true: index_templates.0.index_template.template.mappings + - length: {index_templates.0.index_template.template.aliases: 3} + - is_true: index_templates.0.index_template.template.aliases.test_alias + - match: {index_templates.0.index_template.template.aliases.test_blias.index_routing: "b" } + - match: {index_templates.0.index_template.template.aliases.test_blias.search_routing: "b" } + - match: {index_templates.0.index_template.template.aliases.test_clias.filter.term.user: "kimchy" } + - do: + cluster.put_component_template: + name: my-ct + body: + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + dynamic: true + aliases: + aliasname: {} + version: 2 + _meta: + foo: bar + baz: + eggplant: true + + - do: + cluster.get_component_template: + name: my-ct + - match: {component_templates.0.name: my-ct} + - match: {component_templates.0.component_template.version: 2} + - match: {component_templates.0.component_template._meta: {foo: bar, baz: {eggplant: true}}} + - match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}} + - is_true: component_templates.0.component_template.template.mappings + - match: {component_templates.0.component_template.template.aliases: {aliasname: {}}} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index e6e9a5f79a8d6..eaee0fa6139d1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -188,7 +188,7 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean } CompressedXContent mappings = template.template().mappings(); - String stringMappings = mappings == null ? null : mappings.string(); + String stringMappings = wrapMappingsIfNecessary(mappings == null ? null : mappings.string(), xContentRegistry); // We may need to normalize index settings, so do that also Settings finalSettings = template.template().settings(); @@ -227,17 +227,6 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean } } - // Mappings in component templates don't include _doc, so update the mappings to include this single type - if (stringMappings != null) { - Map parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings); - if (parsedMappings.size() > 0) { - stringMappings = Strings.toString(XContentFactory.jsonBuilder() - .startObject() - .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings) - .endObject()); - } - } - final Template finalTemplate = new Template(finalSettings, stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases()); final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata()); @@ -281,6 +270,35 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean .build(); } + @Nullable + private static String wrapMappingsIfNecessary(@Nullable String mappings, NamedXContentRegistry xContentRegistry) throws Exception { + // Mappings in templates don't have to include _doc, so update + // the mappings to include this single type if necessary + + String stringMappings = mappings; + if (stringMappings != null) { + Map parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings); + if (parsedMappings.size() > 0) { + if (parsedMappings.size() == 1) { + final String keyName = parsedMappings.keySet().iterator().next(); + // Check if it's already wrapped in `_doc`, only rewrap if needed + if (MapperService.SINGLE_MAPPING_NAME.equals(keyName) == false) { + stringMappings = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings) + .endObject()); + } + } else { + stringMappings = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings) + .endObject()); + } + } + } + return stringMappings; + } + /** * Remove the given component template from the cluster state. The component template name * supports simple regex wildcards for removing multiple component templates at a time. @@ -464,18 +482,8 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo // If an inner template was specified, its mappings may need to be // adjusted (to add _doc) and it should be validated CompressedXContent mappings = innerTemplate.mappings(); - String stringMappings = mappings == null ? null : mappings.string(); + String stringMappings = wrapMappingsIfNecessary(mappings == null ? null : mappings.string(), xContentRegistry); - // Mappings in index templates don't include _doc, so update the mappings to include this single type - if (stringMappings != null) { - Map parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings); - if (parsedMappings.size() > 0) { - stringMappings = Strings.toString(XContentFactory.jsonBuilder() - .startObject() - .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings) - .endObject()); - } - } final Template finalTemplate = new Template(finalSettings, stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases()); finalIndexTemplate = new ComposableIndexTemplate(template.indexPatterns(), finalTemplate, template.composedOf(), From ae634a53ddcff82fb36ea5b7e5b53378444e9049 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 20 Apr 2021 14:18:09 +0200 Subject: [PATCH 2/4] Added skip versions --- .../resources/rest-api-spec/test/mixed_cluster/10_basic.yml | 3 +++ .../test/resources/rest-api-spec/test/old_cluster/10_basic.yml | 3 +++ .../resources/rest-api-spec/test/upgraded_cluster/10_basic.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index 945ca80bc876a..873465244d225 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -83,6 +83,9 @@ --- "Component and composable templates can be retrieved": + - skip: + version: " - 7.12.99" + reason: "#58969 only available from 7.13.0" - do: cluster.get_component_template: name: my-ct diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml index a69d7d735719b..1bf743b0a2c00 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml @@ -207,6 +207,9 @@ --- "Component and composable template validation": + - skip: + version: " - 7.12.99" + reason: "#58969 only available from 7.13.0" - do: cluster.put_component_template: name: my-ct diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 1c0b87319236a..9e623d13543b7 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -136,6 +136,9 @@ --- "Component and composable templates can be retrieved and updated": + - skip: + version: " - 7.12.99" + reason: "#58969 only available from 7.13.0" - do: cluster.get_component_template: name: my-ct From 7940bec4ab13b53075ac1a0757141f7032f8936d Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 20 Apr 2021 20:39:26 +0200 Subject: [PATCH 3/4] Make `tests.upgrade_from_version` available to `UpgradeClusterClientYamlTestSuiteIT` and use that to skip tests based on versions. --- .../upgrades/UpgradeClusterClientYamlTestSuiteIT.java | 8 ++++++++ .../test/rest/yaml/ESClientYamlSuiteTestCase.java | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index 910b1e5428729..a56329baa0636 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; import org.apache.lucene.util.TimeUnits; +import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; @@ -19,6 +20,13 @@ @TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + protected static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version")); + + @Override + protected Version overwriteEsVersion(Version esVersionFromApi) { + return UPGRADE_FROM_VERSION; + } + @Override protected boolean preserveIndicesUponCompletion() { return true; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index d804f6e3b1129..544099b66b37b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -123,7 +123,7 @@ public void initAndResetContext() throws Exception { validateSpec(restSpec); final List hosts = getClusterHosts(); Tuple versionVersionTuple = readVersionsFromCatNodes(adminClient()); - final Version esVersion = versionVersionTuple.v1(); + final Version esVersion = overwriteEsVersion(versionVersionTuple.v1()); final Version masterVersion = versionVersionTuple.v2(); final String os = readOsFromNodesInfo(adminClient()); @@ -157,6 +157,10 @@ public void initAndResetContext() throws Exception { restTestExecutionContext.clear(); } + protected Version overwriteEsVersion(Version esVersionFromApi) { + return esVersionFromApi; + } + protected ClientYamlTestClient initClientYamlTestClient( final ClientYamlSuiteRestSpec restSpec, final RestClient restClient, From 846baa4ab716aed729123b477aaba69b03c19a4e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 29 Apr 2021 16:23:29 +0200 Subject: [PATCH 4/4] added jdocs --- .../test/rest/yaml/ESClientYamlSuiteTestCase.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 544099b66b37b..711e58cb49d00 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -157,6 +157,13 @@ public void initAndResetContext() throws Exception { restTestExecutionContext.clear(); } + /** + * Allows test suites to return another version then the lowest version returned from the cat node api. + * + * For example, in rolling upgrade bwc tests, the version of the old cluster should be used, and if + * the cluster is fully upgraded then the cat api doesn't return the version of the old cluster. These + * tests provide the es version via system property. + */ protected Version overwriteEsVersion(Version esVersionFromApi) { return esVersionFromApi; }