diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml new file mode 100644 index 0000000000000..8c2c7d84dc461 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml @@ -0,0 +1,188 @@ +--- +"Component and index template composition": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + + - do: + cluster.put_component_template: + name: ct_low + body: + template: + settings: + number_of_replicas: 1 + mappings: + properties: + field2: + type: text + aliases: + aliasname: + is_write_index: false + + - do: + cluster.put_component_template: + name: ct_high + body: + template: + settings: + index.number_of_replicas: 0 + mappings: + properties: + field2: + type: keyword + aliases: + aliasname: + is_write_index: true + + - do: + indices.put_index_template: + name: my-template + body: + index_patterns: ["foo", "bar-*"] + template: + settings: + index.number_of_shards: 2 + mappings: + properties: + field: + type: keyword + ignore_above: 255 + aliases: + my_alias: {} + aliasname: + filter: + match_all: {} + composed_of: ["ct_low", "ct_high"] + priority: 400 + + - do: + indices.create: + index: bar-baz + body: + settings: + index.priority: 17 + mappings: + properties: + foo: + type: keyword + aliases: + other: {} + + - do: + indices.get: + index: bar-baz + + - match: {bar-baz.settings.index.number_of_shards: "2"} + - match: {bar-baz.settings.index.number_of_replicas: "0"} + - match: {bar-baz.settings.index.priority: "17"} + - match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}} + - match: {bar-baz.mappings.properties.field2: {type: keyword}} + - match: {bar-baz.mappings.properties.foo: {type: keyword}} + - match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}} + - match: {bar-baz.aliases.my_alias: {}} + - match: {bar-baz.aliases.other: {}} + +--- +"Index template priority": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + + - do: + indices.put_index_template: + name: my-template + body: + index_patterns: ["foo", "bar-*"] + template: + settings: + index.number_of_shards: 2 + composed_of: [] + priority: 400 + + - do: + indices.put_index_template: + name: another-template + body: + index_patterns: ["bar-*"] + template: + settings: + index.number_of_shards: 3 + composed_of: [] + priority: 405 + + - do: + indices.create: + index: bar-baz + + - do: + indices.get: + index: bar-baz + + - match: {bar-baz.settings.index.number_of_shards: "3"} + +--- +"Component template only composition": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + + - do: + cluster.put_component_template: + name: ct_low + body: + template: + aliases: + alias1: {} + + - do: + cluster.put_component_template: + name: ct_high + body: + template: + mappings: + properties: + field: + type: keyword + + - do: + indices.put_index_template: + name: my-template + body: + index_patterns: ["baz*"] + composed_of: ["ct_low", "ct_high"] + + - do: + indices.create: + index: bazfoo + + - do: + indices.get: + index: bazfoo + + - match: {bazfoo.mappings.properties.field: {type: keyword}} + - match: {bazfoo.aliases.alias1: {}} + +--- +"Index template without component templates": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + + - do: + indices.put_index_template: + name: my-template + body: + index_patterns: ["eggplant"] + template: + settings: + number_of_shards: 3 + + - do: + indices.create: + index: eggplant + + - do: + indices.get: + index: eggplant + + - match: {eggplant.settings.index.number_of_shards: "3"} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 4def9e85aa76d..15f990ad435ba 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -40,7 +40,7 @@ import java.util.Locale; import java.util.regex.Pattern; -import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findTemplates; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates; public class MetadataRolloverService { private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$"); @@ -161,10 +161,9 @@ static List rolloverAliasToNewIndex(String oldIndex, String newInde * the rollover alias will point to multiple indices. This causes indexing requests to be rejected. * To avoid this, we make sure that there is no duplicated alias in index templates before creating a new index. */ - static void checkNoDuplicatedAliasInIndexTemplate( - Metadata metadata, String rolloverIndexName, String rolloverRequestAlias, - @Nullable Boolean isHidden) { - final List matchedTemplates = findTemplates(metadata, rolloverIndexName, isHidden); + static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String rolloverIndexName, String rolloverRequestAlias, + @Nullable Boolean isHidden) { + final List matchedTemplates = findV1Templates(metadata, rolloverIndexName, isHidden); for (IndexTemplateMetadata template : matchedTemplates) { if (template.aliases().containsKey(rolloverRequestAlias)) { throw new IllegalArgumentException(String.format(Locale.ROOT, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java index 7dd42893ee9ea..8a594bb5cdce5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java @@ -26,15 +26,12 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexTemplateV2; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -75,14 +72,6 @@ protected ClusterBlockException checkBlock(PutIndexTemplateV2Action.Request requ protected void masterOperation(Task task, final PutIndexTemplateV2Action.Request request, final ClusterState state, final ActionListener listener) { IndexTemplateV2 indexTemplate = request.indexTemplate(); - Template template = indexTemplate.template(); - // Normalize the index settings if necessary - if (template.settings() != null) { - Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX); - template = new Template(settings.build(), template.mappings(), template.aliases()); - indexTemplate = new IndexTemplateV2(indexTemplate.indexPatterns(), template, indexTemplate.composedOf(), - indexTemplate.priority(), indexTemplate.version(), indexTemplate.metadata()); - } indexTemplateService.putIndexTemplateV2(request.cause(), request.create(), request.name(), request.masterNodeTimeout(), indexTemplate, listener); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index af71f10a078e8..a53acdf59cad2 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -307,7 +307,7 @@ static boolean resolvePipelines(final DocWriteRequest originalRequest, final } } else if (indexRequest.index() != null) { // the index does not exist yet (and this is a valid request), so match index templates to look for pipelines - List templates = MetadataIndexTemplateService.findTemplates(metadata, indexRequest.index(), null); + List templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null); assert (templates != null); // order of templates are highest order first for (final IndexTemplateMetadata template : templates) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java index e707b84488171..06394a3a1c4d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java @@ -114,11 +114,15 @@ public List indexPatterns() { return indexPatterns; } + @Nullable public Template template() { return template; } public List composedOf() { + if (componentTemplates == null) { + return List.of(); + } return componentTemplates; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index ef8539aadf90e..ee221ede892da 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster.metadata; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -90,6 +89,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -313,39 +313,60 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd final Index recoverFromIndex = request.recoverFrom(); final IndexMetadata sourceMetadata = recoverFromIndex == null ? null : currentState.metadata().getIndexSafe(recoverFromIndex); - // we only find a template when its an API call (a new index) - // find templates, highest order are better matching - final Boolean isHiddenFromRequest = IndexMetadata.INDEX_HIDDEN_SETTING.exists(request.settings()) ? - IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; - final List templates = sourceMetadata == null ? - Collections.unmodifiableList(MetadataIndexTemplateService.findTemplates(currentState.metadata(), - request.index(), - isHiddenFromRequest)) : - List.of(); - - final Map mappings = Collections.unmodifiableMap(parseMappings(request.mappings(), templates, xContentRegistry)); - - final Settings aggregatedIndexSettings = - aggregateIndexSettings(currentState, request, templates, mappings, sourceMetadata, settings, indexScopedSettings); - int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); - - final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings); - validateDotIndex(request.index(), currentState, isHiddenAfterTemplates); - - // remove the setting it's temporary and is only relevant once we create the index - final Settings.Builder settingsBuilder = Settings.builder().put(aggregatedIndexSettings); - settingsBuilder.remove(IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); - final Settings indexSettings = settingsBuilder.build(); + if (sourceMetadata != null) { + // If source metadata was provided, it means we're recovering from an existing index, + // in which case templates don't apply, so create the index from the source metadata + return applyCreateIndexRequestWithExistingMetadata(currentState, request, silent, sourceMetadata); + } else { + // Hidden indices apply templates slightly differently (ignoring wildcard '*' + // templates), so we need to check to see if the request is creating a hidden index + // prior to resolving which templates it matches + final Boolean isHiddenFromRequest = IndexMetadata.INDEX_HIDDEN_SETTING.exists(request.settings()) ? + IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; + + // Check to see if a v2 template matched + final String v2Template = MetadataIndexTemplateService.findV2Template(currentState.metadata(), + request.index(), isHiddenFromRequest); + + if (v2Template != null) { + // If a v2 template was found, it takes precedence over all v1 templates, so create + // the index using that template and the request's specified settings + return applyCreateIndexRequestWithV2Template(currentState, request, silent, v2Template); + } else { + // A v2 template wasn't found, check the v1 templates, in the event no templates are + // found creation still works using the request's specified index settings + final List v1Templates = MetadataIndexTemplateService.findV1Templates(currentState.metadata(), + request.index(), isHiddenFromRequest); - final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); - tmpImdBuilder.setRoutingNumShards(routingNumShards); - tmpImdBuilder.settings(indexSettings); + return applyCreateIndexRequestWithV1Templates(currentState, request, silent, v1Templates); + } + } + } - // Set up everything, now locally create the index to see that things are ok, and apply - IndexMetadata tmpImd = tmpImdBuilder.build(); - validateActiveShardCount(request.waitForActiveShards(), tmpImd); + /** + * Given the state and a request as well as the metadata necessary to build a new index, + * validate the configuration with an actual index service as return a new cluster state with + * the index added (and rerouted) + * @param currentState the current state to base the new state off of + * @param request the create index request + * @param silent a boolean for whether logging should be at a lower or higher level + * @param sourceMetadata when recovering from an existing index, metadata that should be copied to the new index + * @param temporaryIndexMeta metadata for the new index built from templates, source metadata, and request settings + * @param mappings a map of mappings for the new index + * @param aliasSupplier a function that takes the real {@link IndexService} and returns a list of {@link AliasMetadata} aliases + * @param templatesApplied a list of the names of the templates applied, for logging + * @return a new cluster state with the index added + */ + private ClusterState applyCreateIndexWithTemporaryService(final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final IndexMetadata sourceMetadata, + final IndexMetadata temporaryIndexMeta, + final Map mappings, + final Function> aliasSupplier, + final List templatesApplied) throws Exception { // create the index here (on the master) to validate it can be created, as well as adding the mapping - return indicesService.withTempIndexService(tmpImd, indexService -> { + return indicesService.withTempIndexService(temporaryIndexMeta, indexService -> { try { updateIndexMappingsAndBuildSortOrder(indexService, mappings, sourceMetadata); } catch (Exception e) { @@ -353,25 +374,20 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd throw e; } - // the context is only used for validation so it's fine to pass fake values for the shard id and the current - // timestamp - final List aliases = Collections.unmodifiableList( - resolveAndValidateAliases(request.index(), request.aliases(), templates, currentState.metadata(), aliasValidator, - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)) - ); + final List aliases = aliasSupplier.apply(indexService); final IndexMetadata indexMetadata; try { - indexMetadata = buildIndexMetadata(request.index(), aliases, indexService.mapperService()::documentMapper, indexSettings, - routingNumShards, sourceMetadata); + indexMetadata = buildIndexMetadata(request.index(), aliases, indexService.mapperService()::documentMapper, + temporaryIndexMeta.getSettings(), temporaryIndexMeta.getRoutingNumShards(), sourceMetadata); } catch (Exception e) { logger.info("failed to build index metadata [{}]", request.index()); throw e; } logger.log(silent ? Level.DEBUG : Level.INFO, "[{}] creating index, cause [{}], templates {}, shards [{}]/[{}], mappings {}", - request.index(), request.cause(), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), - indexMetadata.getNumberOfShards(), indexMetadata.getNumberOfReplicas(), mappings.keySet()); + request.index(), request.cause(), templatesApplied, indexMetadata.getNumberOfShards(), + indexMetadata.getNumberOfReplicas(), mappings.keySet()); indexService.getIndexEventListener().beforeIndexAddedToCluster(indexMetadata.getIndex(), indexMetadata.getSettings()); @@ -379,21 +395,124 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd }); } + /** + * Given a state and index settings calculated after applying templates, validate metadata for + * the new index, returning an {@link IndexMetadata} for the new index + */ + private IndexMetadata buildAndValidateTemporaryIndexMetadata(final ClusterState currentState, + final Settings aggregatedIndexSettings, + final CreateIndexClusterStateUpdateRequest request, + final int routingNumShards) { + + final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings); + validateDotIndex(request.index(), currentState, isHiddenAfterTemplates); + + // remove the setting it's temporary and is only relevant once we create the index + final Settings.Builder settingsBuilder = Settings.builder().put(aggregatedIndexSettings); + settingsBuilder.remove(IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); + final Settings indexSettings = settingsBuilder.build(); + + final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); + tmpImdBuilder.setRoutingNumShards(routingNumShards); + tmpImdBuilder.settings(indexSettings); + + // Set up everything, now locally create the index to see that things are ok, and apply + IndexMetadata tempMetadata = tmpImdBuilder.build(); + validateActiveShardCount(request.waitForActiveShards(), tempMetadata); + + return tempMetadata; + } + + // TODO: this method can be removed in 9.0 because we will no longer use v1 templates to create indices (only v2 templates) + private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final List templates) throws Exception { + logger.info("applying create index request using v1 templates {}", templates); + + final Map mappings = Collections.unmodifiableMap(parseMappings(request.mappings(), + templates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), xContentRegistry)); + + final Settings aggregatedIndexSettings = + aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(templates), mappings, + null, settings, indexScopedSettings); + int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); + + return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings, + indexService -> resolveAndValidateAliases(request.index(), request.aliases(), + MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator, + // the context is only used for validation so it's fine to pass fake values for the + // shard id and the current timestamp + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + templates.stream().map(IndexTemplateMetadata::getName).collect(toList())); + } + + private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final String templateName) throws Exception { + logger.info("applying create index request using v2 template [{}]", templateName); + + final Map mappings = Collections.unmodifiableMap(parseMappings(request.mappings(), + MetadataIndexTemplateService.resolveMappings(currentState, templateName), xContentRegistry)); + + final Settings aggregatedIndexSettings = + aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(currentState, templateName), + mappings, null, settings, indexScopedSettings); + int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); + + return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings, + indexService -> resolveAndValidateAliases(request.index(), request.aliases(), + MetadataIndexTemplateService.resolveAliases(currentState, templateName), currentState.metadata(), aliasValidator, + // the context is only used for validation so it's fine to pass fake values for the + // shard id and the current timestamp + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + Collections.singletonList(templateName)); + } + + private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final IndexMetadata sourceMetadata) throws Exception { + logger.info("applying create index request using existing index [{}] metadata", sourceMetadata.getIndex().getName()); + + final Map mappings = Collections.unmodifiableMap(MapperService.parseMapping(xContentRegistry, request.mappings())); + + final Settings aggregatedIndexSettings = + aggregateIndexSettings(currentState, request, Settings.EMPTY, mappings, sourceMetadata, settings, indexScopedSettings); + final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); + + return applyCreateIndexWithTemporaryService(currentState, request, silent, sourceMetadata, tmpImd, mappings, + indexService -> resolveAndValidateAliases(request.index(), request.aliases(), Collections.emptyList(), + currentState.metadata(), aliasValidator, xContentRegistry, + // the context is only used for validation so it's fine to pass fake values for the + // shard id and the current timestamp + indexService.newQueryShardContext(0, null, () -> 0L, null)), + Collections.emptyList()); + } + /** * Parses the provided mappings json and the inheritable mappings from the templates (if any) into a map. * * The template mappings are applied in the order they are encountered in the list (clients should make sure the lower index, closer * to the head of the list, templates have the highest {@link IndexTemplateMetadata#order()}) */ - static Map parseMappings(String mappingsJson, List templates, + static Map parseMappings(String mappingsJson, List templateMappings, NamedXContentRegistry xContentRegistry) throws Exception { Map mappings = MapperService.parseMapping(xContentRegistry, mappingsJson); // apply templates, merging the mappings into the request mapping if exists - for (IndexTemplateMetadata template : templates) { - CompressedXContent mapping = template.mappings(); + for (CompressedXContent mapping : templateMappings) { if (mapping != null) { Map templateMapping = MapperService.parseMapping(xContentRegistry, mapping.string()); - assert templateMapping.size() == 1 : templateMapping; + if (templateMapping.isEmpty()) { + // Someone provided an empty '{}' for mappings, which is okay, but to avoid + // tripping the below assertion, we can safely ignore it + continue; + } + assert templateMapping.size() == 1 : "expected exactly one mapping value, got: " + templateMapping; // pre-8x templates may have a wrapper type other than _doc, so we re-wrap things here templateMapping = Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, templateMapping.values().iterator().next()); @@ -418,15 +537,12 @@ static Map parseMappings(String mappingsJson, List templates, Map mappings, + Settings templateSettings, Map mappings, @Nullable IndexMetadata sourceMetadata, Settings settings, IndexScopedSettings indexScopedSettings) { Settings.Builder indexSettingsBuilder = Settings.builder(); if (sourceMetadata == null) { - // apply templates, here, in reverse order, since first ones are better matching - for (int i = templates.size() - 1; i >= 0; i--) { - indexSettingsBuilder.put(templates.get(i).settings()); - } + indexSettingsBuilder.put(templateSettings); } // now, put the request settings, so they override templates indexSettingsBuilder.put(request.settings()); @@ -514,7 +630,7 @@ static int getIndexNumberOfRoutingShards(Settings indexSettings, @Nullable Index * @return the list of resolved aliases, with the explicitly provided aliases occurring first (having a higher priority) followed by * the ones inherited from the templates */ - static List resolveAndValidateAliases(String index, Set aliases, List templates, + static List resolveAndValidateAliases(String index, Set aliases, List> templateAliases, Metadata metadata, AliasValidator aliasValidator, NamedXContentRegistry xContentRegistry, QueryShardContext queryShardContext) { List resolvedAliases = new ArrayList<>(); @@ -530,17 +646,17 @@ static List resolveAndValidateAliases(String index, Set al } Map templatesAliases = new HashMap<>(); - for (IndexTemplateMetadata template : templates) { + for (Map templateAliasConfig : templateAliases) { // handle aliases - for (ObjectObjectCursor cursor : template.aliases()) { - AliasMetadata aliasMetadata = cursor.value; + for (Map.Entry entry : templateAliasConfig.entrySet()) { + AliasMetadata aliasMetadata = entry.getValue(); // if an alias with same name came with the create index request itself, // ignore this one taken from the index template if (aliases.contains(new Alias(aliasMetadata.alias()))) { continue; } // if an alias with same name was already processed, ignore this one - if (templatesAliases.containsKey(cursor.key)) { + if (templatesAliases.containsKey(entry.getKey())) { continue; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 6823fa8c56f5c..c396000d863f2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -46,6 +46,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperParsingException; @@ -63,6 +64,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; @@ -183,11 +185,34 @@ ClusterState addComponentTemplate(final ClusterState currentState, final boolean CompressedXContent mappings = template.template().mappings(); String stringMappings = mappings == null ? null : mappings.string(); - validateTemplate(template.template().settings(), stringMappings, indicesService, xContentRegistry); + // We may need to normalize index settings, so do that also + Settings finalSettings = template.template().settings(); + if (finalSettings != null) { + finalSettings = Settings.builder() + .put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX) + .build(); + } + + validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry); + + // Mappings in component templates don't include _doc, so update the mappings to include this single type + if (stringMappings != null) { + Map parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings); + if (parsedMappings.size() > 0) { + stringMappings = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings) + .endObject()); + } + } + + final Template finalTemplate = new Template(finalSettings, + stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases()); + final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata()); logger.info("adding component template [{}]", name); return ClusterState.builder(currentState) - .metadata(Metadata.builder(currentState.metadata()).put(name, template)) + .metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate)) .build(); } @@ -262,7 +287,7 @@ public void onFailure(String source, Exception e) { } @Override - public ClusterState execute(ClusterState currentState) { + public ClusterState execute(ClusterState currentState) throws Exception { return addIndexTemplateV2(currentState, create, name, template); } @@ -274,8 +299,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } // Package visible for testing - static ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create, - final String name, final IndexTemplateV2 template) { + ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create, + final String name, final IndexTemplateV2 template) throws Exception { if (create && currentState.metadata().templatesV2().containsKey(name)) { throw new IllegalArgumentException("index template [" + name + "] already exists"); } @@ -295,12 +320,41 @@ static ClusterState addIndexTemplateV2(final ClusterState currentState, final bo deprecationLogger.deprecated(warning); } - // TODO: validation of index template - // validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry); + IndexTemplateV2 finalIndexTemplate = template; + Template innerTemplate = template.template(); + if (innerTemplate != null) { + // We may need to normalize index settings, so do that also + Settings finalSettings = innerTemplate.settings(); + if (finalSettings != null) { + finalSettings = Settings.builder() + .put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX) + .build(); + } + // If an inner template was specified, its mappings may need to be + // adjusted (to add _doc) and it should be validated + CompressedXContent mappings = innerTemplate.mappings(); + String stringMappings = mappings == null ? null : mappings.string(); + validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry); + + // Mappings in index templates don't include _doc, so update the mappings to include this single type + if (stringMappings != null) { + Map parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings); + if (parsedMappings.size() > 0) { + stringMappings = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .field(MapperService.SINGLE_MAPPING_NAME, parsedMappings) + .endObject()); + } + } + final Template finalTemplate = new Template(finalSettings, + stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases()); + finalIndexTemplate = new IndexTemplateV2(template.indexPatterns(), finalTemplate, template.composedOf(), + template.priority(), template.version(), template.metadata()); + } logger.info("adding index template [{}]", name); return ClusterState.builder(currentState) - .metadata(Metadata.builder(currentState.metadata()).put(name, template)) + .metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate)) .build(); } @@ -532,7 +586,7 @@ static ClusterState innerPutTemplate(final ClusterState currentState, PutRequest * @return a list of templates sorted by {@link IndexTemplateMetadata#order()} descending. * */ - public static List findTemplates(Metadata metadata, String indexName, @Nullable Boolean isHidden) { + public static List findV1Templates(Metadata metadata, String indexName, @Nullable Boolean isHidden) { final Predicate patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, indexName); final List matchedTemplates = new ArrayList<>(); for (ObjectCursor cursor : metadata.templates().values()) { @@ -575,10 +629,165 @@ public static List findTemplates(Metadata metadata, Strin } } } - return matchedTemplates; + return Collections.unmodifiableList(matchedTemplates); + } + + /** + * Return the name (id) of the highest matching index template for the given index name. In + * the event that no templates are matched, {@code null} is returned. + */ + @Nullable + public static String findV2Template(Metadata metadata, String indexName, @Nullable Boolean isHidden) { + final Predicate patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, indexName); + final Map matchedTemplates = new HashMap<>(); + for (Map.Entry entry : metadata.templatesV2().entrySet()) { + final String name = entry.getKey(); + final IndexTemplateV2 template = entry.getValue(); + if (isHidden == null || isHidden == Boolean.FALSE) { + final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate); + if (matched) { + matchedTemplates.put(template, name); + } + } else { + assert isHidden == Boolean.TRUE; + final boolean isNotMatchAllTemplate = template.indexPatterns().stream().noneMatch(Regex::isMatchAllPattern); + if (isNotMatchAllTemplate) { + if (template.indexPatterns().stream().anyMatch(patternMatchPredicate)) { + matchedTemplates.put(template, name); + } + } + } + } + + if (matchedTemplates.size() == 0) { + return null; + } + + final List candidates = new ArrayList<>(matchedTemplates.keySet()); + CollectionUtil.timSort(candidates, Comparator.comparingLong(IndexTemplateV2::priority).reversed()); + + assert candidates.size() > 0 : "we should have returned early with no candidates"; + IndexTemplateV2 winner = candidates.get(0); + return matchedTemplates.get(winner); + } + + /** + * Resolve the given v2 template into an ordered list of mappings + */ + public static List resolveMappings(final ClusterState state, final String templateName) { + final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName); + assert template != null : "attempted to resolve mappings for a template [" + templateName + + "] that did not exist in the cluster state"; + if (template == null) { + return List.of(); + } + final Map componentTemplates = state.metadata().componentTemplates(); + // TODO: more fine-grained merging of component template mappings, ie, merge fields as distint entities + List mappings = template.composedOf().stream() + .map(componentTemplates::get) + .filter(Objects::nonNull) + .map(ComponentTemplate::template) + .map(Template::mappings) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + // Add the actual index template's mappings, since it takes the highest precedence + Optional.ofNullable(template.template()) + .map(Template::mappings) + .ifPresent(mappings::add); + // When actually merging mappings, the highest precedence ones should go first, so reverse the list + Collections.reverse(mappings); + return Collections.unmodifiableList(mappings); } - private static void validateTemplate(Settings settings, String mappings, + /** + * Resolve index settings for the given list of v1 templates, templates are apply in reverse + * order since they should be provided in order of priority/order + */ + public static Settings resolveSettings(final List templates) { + Settings.Builder templateSettings = Settings.builder(); + // apply templates, here, in reverse order, since first ones are better matching + for (int i = templates.size() - 1; i >= 0; i--) { + templateSettings.put(templates.get(i).settings()); + } + return templateSettings.build(); + } + + /** + * Resolve the given v2 template into a collected {@link Settings} object + */ + public static Settings resolveSettings(final ClusterState state, final String templateName) { + final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName); + assert template != null : "attempted to resolve settings for a template [" + templateName + + "] that did not exist in the cluster state"; + if (template == null) { + return Settings.EMPTY; + } + final Map componentTemplates = state.metadata().componentTemplates(); + List componentSettings = template.composedOf().stream() + .map(componentTemplates::get) + .filter(Objects::nonNull) + .map(ComponentTemplate::template) + .map(Template::settings) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + Settings.Builder templateSettings = Settings.builder(); + componentSettings.forEach(templateSettings::put); + // Add the actual index template's settings to the end, since it takes the highest precedence. + Optional.ofNullable(template.template()) + .map(Template::settings) + .ifPresent(templateSettings::put); + return templateSettings.build(); + } + + /** + * Resolve the given v1 templates into an ordered list of aliases + */ + public static List> resolveAliases(final List templates) { + final List> resolvedAliases = new ArrayList<>(); + templates.forEach(template -> { + if (template.aliases() != null) { + Map aliasMeta = new HashMap<>(); + for (ObjectObjectCursor cursor : template.aliases()) { + aliasMeta.put(cursor.key, cursor.value); + } + resolvedAliases.add(aliasMeta); + } + }); + return Collections.unmodifiableList(resolvedAliases); + } + + /** + * Resolve the given v2 template into an ordered list of aliases + */ + public static List> resolveAliases(final ClusterState state, final String templateName) { + final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName); + assert template != null : "attempted to resolve aliases for a template [" + templateName + + "] that did not exist in the cluster state"; + if (template == null) { + return List.of(); + } + final Map componentTemplates = state.metadata().componentTemplates(); + List> aliases = template.composedOf().stream() + .map(componentTemplates::get) + .filter(Objects::nonNull) + .map(ComponentTemplate::template) + .map(Template::aliases) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + // Add the actual index template's aliases to the end if they exist + Optional.ofNullable(template.template()) + .map(Template::aliases) + .ifPresent(aliases::add); + // Aliases are applied in order, but subsequent alias configuration from the same name is + // ignored, so in order for the order to be correct, alias configuration should be in order + // of precedence (with the index template first) + Collections.reverse(aliases); + return Collections.unmodifiableList(aliases); + } + + private static void validateTemplate(Settings validateSettings, String mappings, IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception { // First check to see if mappings are valid XContent if (mappings != null) { @@ -589,6 +798,12 @@ private static void validateTemplate(Settings settings, String mappings, } } + // Hard to validate settings if they're non-existent, so used empty ones if none were provided + Settings settings = validateSettings; + if (settings == null) { + settings = Settings.EMPTY; + } + Index createdIndex = null; final String temporaryIndexName = UUIDs.randomBase64UUID(); try { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index eb819bb8f3993..f86952c65fde9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -171,7 +172,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws XContentHelper.convertToMap(new BytesArray(this.mappings.uncompressed()), true, XContentType.JSON).v2(); if (uncompressedMapping.size() > 0) { builder.field(MAPPINGS.getPreferredName()); - builder.map(uncompressedMapping); + builder.map(reduceMapping(uncompressedMapping)); } } if (this.aliases != null) { @@ -184,4 +185,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + @SuppressWarnings("unchecked") + private static Map reduceMapping(Map mapping) { + if (mapping.size() == 1 && MapperService.SINGLE_MAPPING_NAME.equals(mapping.keySet().iterator().next())) { + return (Map) mapping.values().iterator().next(); + } else { + return mapping; + } + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java index af48738753a5f..2709c59863c77 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java @@ -98,7 +98,7 @@ public static Map randomAliases() { private static CompressedXContent randomMappings() { try { - return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}"); + return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}"); } catch (IOException e) { fail("got an IO exception creating fake mappings: " + e); return null; @@ -107,7 +107,12 @@ private static CompressedXContent randomMappings() { private static Settings randomSettings() { return Settings.builder() - .put(randomAlphaOfLength(4), randomAlphaOfLength(10)) + .put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean()) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10)) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 5)) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_PRIORITY, randomIntBetween(0, 100000)) .build(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java index 1940442c59adb..121c10a5338bc 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java @@ -110,7 +110,7 @@ private static Map randomAliases() { private static CompressedXContent randomMappings() { try { - return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}"); + return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}"); } catch (IOException e) { fail("got an IO exception creating fake mappings: " + e); return null; @@ -119,7 +119,12 @@ private static CompressedXContent randomMappings() { private static Settings randomSettings() { return Settings.builder() - .put(randomAlphaOfLength(4), randomAlphaOfLength(10)) + .put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean()) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10)) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 5)) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_PRIORITY, randomIntBetween(0, 100000)) .build(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 0304cfaabd2f7..0108315240f59 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -621,8 +621,8 @@ public void testParseMappingsAppliesDataFromTemplateAndRequest() throws Exceptio }); request.mappings(createMapping("mapping_from_request", "text").string()); - Map parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(), List.of(templateMetadata), - NamedXContentRegistry.EMPTY); + Map parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(), + List.of(templateMetadata.getMappings()), NamedXContentRegistry.EMPTY); assertThat(parsedMappings, hasKey("_doc")); Map doc = (Map) parsedMappings.get("_doc"); @@ -645,7 +645,7 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() { .build(); request.settings(Settings.builder().put("request_setting", "value2").build()); - Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, List.of(templateMetadata), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1")); @@ -677,11 +677,12 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { request.aliases(Set.of(new Alias("alias").searchRouting("fromRequest"))); request.settings(Settings.builder().put("key1", "requestValue").build()); - Map parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(), List.of(templateMetadata), - xContentRegistry()); - List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), List.of(templateMetadata), + Map parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(), + List.of(templateMetadata.mappings()), xContentRegistry()); + List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), + MetadataIndexTemplateService.resolveAliases(List.of(templateMetadata)), Metadata.builder().build(), aliasValidator, xContentRegistry(), queryShardContext); - Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(templateMetadata), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templateMetadata.settings(), Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(resolvedAliases.get(0).getSearchRouting(), equalTo("fromRequest")); @@ -695,14 +696,14 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { } public void testDefaultSettings() { - Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("1")); } public void testSettingsFromClusterState() { - Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("15")); @@ -725,9 +726,11 @@ public void testTemplateOrder() throws Exception { .settings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 10)) .putAlias(AliasMetadata.builder("alias1").searchRouting("1").build()) )); - Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templates, Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, + MetadataIndexTemplateService.resolveSettings(templates), Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); - List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), templates, + List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), + MetadataIndexTemplateService.resolveAliases(templates), Metadata.builder().build(), aliasValidator, xContentRegistry(), queryShardContext); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("12")); AliasMetadata alias = resolvedAliases.get(0); @@ -751,7 +754,7 @@ public void testAggregateIndexSettingsIgnoresTemplatesOnCreateFromSourceIndex() createClusterState("sourceIndex", 1, 0, Settings.builder().put("index.blocks.write", true).build()); - Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, List.of(templateMetadata), Map.of(), + Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), Map.of(), clusterState.metadata().index("sourceIndex"), Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertThat(aggregatedIndexSettings.get("templateSetting"), is(nullValue())); @@ -813,7 +816,7 @@ public void testParseMappingsWithTypedTemplateAndTypelessIndexMapping() throws E } }); - Map mappings = parseMappings("{\"_doc\":{}}", List.of(templateMetadata), xContentRegistry()); + Map mappings = parseMappings("{\"_doc\":{}}", List.of(templateMetadata.mappings()), xContentRegistry()); assertThat(mappings, Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); } @@ -826,7 +829,7 @@ public void testParseMappingsWithTypedTemplate() throws Exception { ExceptionsHelper.reThrowIfNotNull(e); } }); - Map mappings = parseMappings("", List.of(templateMetadata), xContentRegistry()); + Map mappings = parseMappings("", List.of(templateMetadata.mappings()), xContentRegistry()); assertThat(mappings, Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); } @@ -838,7 +841,7 @@ public void testParseMappingsWithTypelessTemplate() throws Exception { ExceptionsHelper.reThrowIfNotNull(e); } }); - Map mappings = parseMappings("", List.of(templateMetadata), xContentRegistry()); + Map mappings = parseMappings("", List.of(templateMetadata.mappings()), xContentRegistry()); assertThat(mappings, Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); } @@ -905,7 +908,7 @@ public void testRejectWithSoftDeletesDisabled() { final IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); }); assertThat(error.getMessage(), equalTo("Creating indices with soft-deletes disabled is no longer supported. " @@ -926,7 +929,7 @@ public void testRejectTranslogRetentionSettings() { } request.settings(settings.build()); IllegalArgumentException error = expectThrows(IllegalArgumentException.class, - () -> aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + () -> aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS)); assertThat(error.getMessage(), equalTo("Translog retention settings [index.translog.retention.age] " + "and [index.translog.retention.size] are no longer supported. Please do not specify values for these settings")); @@ -942,7 +945,7 @@ public void testDeprecateTranslogRetentionSettings() { } settings.put(SETTING_INDEX_VERSION_CREATED.getKey(), VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0)); request.settings(settings.build()); - aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(), + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Map.of(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); assertWarnings("Translog retention settings [index.translog.retention.age] " + "and [index.translog.retention.size] are deprecated and effectively ignored. They will be removed in a future version."); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 1006822595485..e67268b61ec41 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -28,21 +28,28 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexTemplateException; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; @@ -193,11 +200,11 @@ public void testFindTemplates() throws Exception { putTemplateDetail(new PutRequest("test", "foo-2").patterns(singletonList("foo-*")).order(2)); putTemplateDetail(new PutRequest("test", "bar").patterns(singletonList("bar-*")).order(between(0, 100))); final ClusterState state = client().admin().cluster().prepareState().get().getState(); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", randomBoolean()).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", randomBoolean()).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("foo-2", "foo-1")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", randomBoolean()).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", randomBoolean()).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("bar")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", randomBoolean()), empty()); + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", randomBoolean()), empty()); } public void testFindTemplatesWithHiddenIndices() throws Exception { @@ -212,32 +219,32 @@ public void testFindTemplatesWithHiddenIndices() throws Exception { final ClusterState state = client().admin().cluster().prepareState().get().getState(); // hidden - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", true).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", true).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("foo-2", "foo-1")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", true).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", true).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("bar")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", true), empty()); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "sneaky1", true).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", true), empty()); + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "sneaky1", true).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("sneaky-hidden")); // not hidden - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", false).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", false).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("foo-2", "foo-1", "global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", false).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", false).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("bar", "global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", false).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", false).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "sneaky1", false).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "sneaky1", false).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("global", "sneaky-hidden")); // unknown - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", null).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", null).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("foo-2", "foo-1", "global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", null).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", null).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("bar", "global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", null).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", null).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("global")); - assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "sneaky1", null).stream() + assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "sneaky1", null).stream() .map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("sneaky-hidden")); } @@ -285,32 +292,34 @@ public void testAddComponentTemplate() throws Exception{ () -> metadataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate4)); } - public void testAddIndexTemplateV2() { + public void testAddIndexTemplateV2() throws Exception { ClusterState state = ClusterState.EMPTY_STATE; + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateV2 template = IndexTemplateV2Tests.randomInstance(); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template); assertNotNull(state.metadata().templatesV2().get("foo")); - assertThat(state.metadata().templatesV2().get("foo"), equalTo(template)); + assertTemplatesEqual(state.metadata().templatesV2().get("foo"), template); final ClusterState throwState = ClusterState.builder(state).build(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> MetadataIndexTemplateService.addIndexTemplateV2(throwState, true, "foo", template)); + () -> metadataIndexTemplateService.addIndexTemplateV2(throwState, true, "foo", template)); assertThat(e.getMessage(), containsString("index template [foo] already exists")); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, randomBoolean(), "bar", template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, randomBoolean(), "bar", template); assertNotNull(state.metadata().templatesV2().get("bar")); } - public void testRemoveIndexTemplateV2() { + public void testRemoveIndexTemplateV2() throws Exception { IndexTemplateV2 template = IndexTemplateV2Tests.randomInstance(); + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateMissingException e = expectThrows(IndexTemplateMissingException.class, () -> MetadataIndexTemplateService.innerRemoveIndexTemplateV2(ClusterState.EMPTY_STATE, "foo")); assertThat(e.getMessage(), equalTo("index_template [foo] missing")); - final ClusterState state = MetadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); + final ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); assertNotNull(state.metadata().templatesV2().get("foo")); - assertThat(state.metadata().templatesV2().get("foo"), equalTo(template)); + assertTemplatesEqual(state.metadata().templatesV2().get("foo"), template); ClusterState updatedState = MetadataIndexTemplateService.innerRemoveIndexTemplateV2(state, "foo"); assertNull(updatedState.metadata().templatesV2().get("foo")); @@ -319,10 +328,11 @@ public void testRemoveIndexTemplateV2() { /** * Test that if we have a pre-existing v1 template and put a v2 template that would match the same indices, we generate a warning */ - public void testPuttingV2TemplateGeneratesWarning() { + public void testPuttingV2TemplateGeneratesWarning() throws Exception { IndexTemplateMetadata v1Template = IndexTemplateMetadata.builder("v1-template") .patterns(Arrays.asList("fo*", "baz")) .build(); + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder(Metadata.EMPTY_METADATA) @@ -331,7 +341,7 @@ public void testPuttingV2TemplateGeneratesWarning() { .build(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + "from existing older templates [v1-template] with patterns (v1-template => [fo*, baz]); this template [v2-template] will " + @@ -344,9 +354,10 @@ public void testPuttingV2TemplateGeneratesWarning() { /** * Test that if we have a pre-existing v2 template and put a "*" v1 template, we generate a warning */ - public void testPuttingV1StarTemplateGeneratesWarning() { + public void testPuttingV1StarTemplateGeneratesWarning() throws Exception { + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - ClusterState state = MetadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); + ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); req.patterns(Arrays.asList("*", "baz")); @@ -363,9 +374,10 @@ public void testPuttingV1StarTemplateGeneratesWarning() { /** * Test that if we have a pre-existing v2 template and put a v1 template that would match the same indices, we generate a hard error */ - public void testPuttingV1NonStarTemplateGeneratesError() { + public void testPuttingV1NonStarTemplateGeneratesError() throws Exception { + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - ClusterState state = MetadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); + ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); req.patterns(Arrays.asList("egg*", "baz")); @@ -384,7 +396,9 @@ public void testPuttingV1NonStarTemplateGeneratesError() { * Test that if we have a pre-existing v1 and v2 template, and we update the existing v1 * template without changing its index patterns, a warning is generated */ - public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() { + public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() throws Exception { + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); + IndexTemplateMetadata v1Template = IndexTemplateMetadata.builder("v1-template") .patterns(Arrays.asList("fo*", "baz")) .build(); @@ -396,7 +410,7 @@ public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() .build(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + "from existing older templates [v1-template] with patterns (v1-template => [fo*, baz]); this template [v2-template] will " + @@ -423,7 +437,8 @@ public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() * Test that if we have a pre-existing v1 and v2 template, and we update the existing v1 * template *AND* change the index patterns that an error is generated */ - public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() { + public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() throws Exception { + final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); IndexTemplateMetadata v1Template = IndexTemplateMetadata.builder("v1-template") .patterns(Arrays.asList("fo*", "baz")) .build(); @@ -435,7 +450,7 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() { .build(); IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null); - state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); + state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + "from existing older templates [v1-template] with patterns (v1-template => [fo*, baz]); this template [v2-template] will " + @@ -457,6 +472,161 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() { "templates (/_index_template) instead")); } + public void testFindV2Templates() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + assertNull(MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean() ? null : randomBoolean())); + + ComponentTemplate ct = ComponentTemplateTests.randomInstance(); + state = service.addComponentTemplate(state, true, "ct", ct); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), null, List.of("ct"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + IndexTemplateV2 it2 = new IndexTemplateV2(List.of("in*"), null, List.of("ct"), 10L, 2L, null); + state = service.addIndexTemplateV2(state, true, "my-template2", it2); + + String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean() ? null : randomBoolean()); + + assertThat(result, equalTo("my-template2")); + } + + public void testFindV2TemplatesForHiddenIndex() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + assertNull(MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true)); + + ComponentTemplate ct = ComponentTemplateTests.randomInstance(); + state = service.addComponentTemplate(state, true, "ct", ct); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), null, List.of("ct"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + IndexTemplateV2 it2 = new IndexTemplateV2(List.of("*"), null, List.of("ct"), 10L, 2L, null); + state = service.addIndexTemplateV2(state, true, "my-template2", it2); + + String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true); + + assertThat(result, equalTo("my-template")); + } + + public void testResolveMappings() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, + new CompressedXContent("{\n" + + " \"properties\": {\n" + + " \"field2\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }"), null), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(null, + new CompressedXContent("{\n" + + " \"properties\": {\n" + + " \"field2\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }"), null), null, null); + state = service.addComponentTemplate(state, true, "ct_high", ct1); + state = service.addComponentTemplate(state, true, "ct_low", ct2); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), + new Template(null, + new CompressedXContent("{\n" + + " \"properties\": {\n" + + " \"field\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }"), null), + List.of("ct_low", "ct_high"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + + List mappings = MetadataIndexTemplateService.resolveMappings(state, "my-template"); + + assertNotNull(mappings); + assertThat(mappings.size(), equalTo(3)); + List> parsedMappings = mappings.stream() + .map(m -> { + try { + return MapperService.parseMapping(new NamedXContentRegistry(List.of()), m.string()); + } catch (Exception e) { + logger.error(e); + fail("failed to parse mappings: " + m.string()); + return null; + } + }) + .collect(Collectors.toList()); + + // The order of mappings should be: + // - index template + // - ct_high + // - ct_low + // Because the first elements when merging mappings have the highest precedence + assertThat(parsedMappings.get(0), + equalTo(Map.of("_doc", Map.of("properties", Map.of("field", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(1), + equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "keyword")))))); + assertThat(parsedMappings.get(2), + equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text")))))); + } + + public void testResolveSettings() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + + ComponentTemplate ct1 = new ComponentTemplate(new Template(Settings.builder() + .put("number_of_replicas", 2) + .put("index.blocks.write", true) + .build(), + null, null), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(Settings.builder() + .put("index.number_of_replicas", 1) + .put("index.blocks.read", true) + .build(), + null, null), null, null); + state = service.addComponentTemplate(state, true, "ct_high", ct1); + state = service.addComponentTemplate(state, true, "ct_low", ct2); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), + new Template(Settings.builder() + .put("index.blocks.write", false) + .put("index.number_of_shards", 3) + .build(), null, null), + List.of("ct_low", "ct_high"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + + Settings settings = MetadataIndexTemplateService.resolveSettings(state, "my-template"); + assertThat(settings.get("index.number_of_replicas"), equalTo("2")); + assertThat(settings.get("index.blocks.write"), equalTo("false")); + assertThat(settings.get("index.blocks.read"), equalTo("true")); + assertThat(settings.get("index.number_of_shards"), equalTo("3")); + } + + public void testResolveAliases() throws Exception { + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + + Map a1 = new HashMap<>(); + a1.put("foo", AliasMetadata.newAliasMetadataBuilder("foo").build()); + Map a2 = new HashMap<>(); + a2.put("bar", AliasMetadata.newAliasMetadataBuilder("bar").build()); + Map a3 = new HashMap<>(); + a3.put("eggplant", AliasMetadata.newAliasMetadataBuilder("eggplant").build()); + a3.put("baz", AliasMetadata.newAliasMetadataBuilder("baz").build()); + + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, null, a1), null, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(null, null, a2), null, null); + state = service.addComponentTemplate(state, true, "ct_high", ct1); + state = service.addComponentTemplate(state, true, "ct_low", ct2); + IndexTemplateV2 it = new IndexTemplateV2(List.of("i*"), + new Template(null, null, a3), + List.of("ct_low", "ct_high"), 0L, 1L, null); + state = service.addIndexTemplateV2(state, true, "my-template", it); + + List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(state, "my-template"); + + // These should be order of precedence, so the index template (a3), then ct_high (a1), then ct_low (a2) + assertThat(resolvedAliases, equalTo(List.of(a3, a1, a2))); + } + private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { MetadataCreateIndexService createIndexService = new MetadataCreateIndexService( Settings.EMPTY, @@ -529,4 +699,55 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { clusterService, createIndexService, new AliasValidator(), indicesService, new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry()); } + + @SuppressWarnings("unchecked") + public static void assertTemplatesEqual(IndexTemplateV2 actual, IndexTemplateV2 expected) { + IndexTemplateV2 actualNoTemplate = new IndexTemplateV2(actual.indexPatterns(), null, + actual.composedOf(), actual.priority(), actual.version(), actual.metadata()); + IndexTemplateV2 expectedNoTemplate = new IndexTemplateV2(expected.indexPatterns(), null, + expected.composedOf(), expected.priority(), expected.version(), expected.metadata()); + + assertThat(actualNoTemplate, equalTo(expectedNoTemplate)); + Template actualTemplate = actual.template(); + Template expectedTemplate = expected.template(); + + assertThat("expected both templates to have either a template or no template", + Objects.nonNull(actualTemplate), equalTo(Objects.nonNull(expectedTemplate))); + + if (actualTemplate != null) { + assertThat(actualTemplate.settings(), equalTo(expectedTemplate.settings())); + assertThat(actualTemplate.aliases(), equalTo(expectedTemplate.aliases())); + assertThat("expected both templates to have either mappings or no mappings", + Objects.nonNull(actualTemplate.mappings()), equalTo(Objects.nonNull(expectedTemplate.mappings()))); + + if (actualTemplate.mappings() != null) { + Map actualMappings; + Map expectedMappings; + try (XContentParser parser = XContentType.JSON.xContent() + .createParser(new NamedXContentRegistry(List.of()), LoggingDeprecationHandler.INSTANCE, + actualTemplate.mappings().string())) { + actualMappings = parser.map(); + } catch (IOException e) { + throw new AssertionError(e); + } + try (XContentParser parser = XContentType.JSON.xContent() + .createParser(new NamedXContentRegistry(List.of()), LoggingDeprecationHandler.INSTANCE, + expectedTemplate.mappings().string())) { + expectedMappings = parser.map(); + } catch (IOException e) { + throw new AssertionError(e); + } + + if (actualMappings.size() == 1 && actualMappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) { + actualMappings = (Map) actualMappings.get(MapperService.SINGLE_MAPPING_NAME); + } + + if (expectedMappings.size() == 1 && expectedMappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) { + expectedMappings = (Map) expectedMappings.get(MapperService.SINGLE_MAPPING_NAME); + } + + assertThat(actualMappings, equalTo(expectedMappings)); + } + } + } }