Skip to content

Commit 814c248

Browse files
dakronedakrone
authored
[7.x] Use V2 index templates during index creation (#54669) (#54750)
* Use V2 index templates during index creation This commit changes our index creation code to use (and favor!) V2 index templates during index creation. The creation precedence goes like so, in order of precedence: - Existing source `IndexMetadata` - for example, when recovering from a peer or a shrink/split/clone where index templates should not be applied - A matching V2 index template, if one is found - When a V2 template is found, all component templates (in the `composed_of` field) are applied in the order that they appear, with the index template having the 2nd highest precedence (the create index request always has the top priority when it comes to index settings) - All matching V1 templates (the old style) This also adds index template validation when `PUT`-ing a new v2 index template (because this was required) and ensures that all index and component templates specify *no* top-level mapping type (it is automatically added when the template is added to the cluster state). This does not yet implement fine-grained component template merging of mappings, where we favor merging only a single field's configuration, that will be done in subsequent work. This also keeps the existing hidden index behavior present for v1 templates, where a hidden index will match v2 index templates unless they are global (`*`) templates. Relates to #53101
1 parent 548145f commit 814c248

File tree

12 files changed

+949
-151
lines changed

12 files changed

+949
-151
lines changed

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
---
2+
"Component and index template composition":
3+
- skip:
4+
version: " - 7.7.99"
5+
reason: "itv2 is available in 7.8+"
6+
7+
- do:
8+
cluster.put_component_template:
9+
name: ct_low
10+
body:
11+
template:
12+
settings:
13+
number_of_replicas: 1
14+
mappings:
15+
properties:
16+
field2:
17+
type: text
18+
aliases:
19+
aliasname:
20+
is_write_index: false
21+
22+
- do:
23+
cluster.put_component_template:
24+
name: ct_high
25+
body:
26+
template:
27+
settings:
28+
index.number_of_replicas: 0
29+
mappings:
30+
properties:
31+
field2:
32+
type: keyword
33+
aliases:
34+
aliasname:
35+
is_write_index: true
36+
37+
- do:
38+
indices.put_index_template:
39+
name: my-template
40+
body:
41+
index_patterns: ["foo", "bar-*"]
42+
template:
43+
settings:
44+
index.number_of_shards: 2
45+
mappings:
46+
properties:
47+
field:
48+
type: keyword
49+
ignore_above: 255
50+
aliases:
51+
my_alias: {}
52+
aliasname:
53+
filter:
54+
match_all: {}
55+
composed_of: ["ct_low", "ct_high"]
56+
priority: 400
57+
58+
- do:
59+
indices.create:
60+
index: bar-baz
61+
body:
62+
settings:
63+
index.priority: 17
64+
mappings:
65+
properties:
66+
foo:
67+
type: keyword
68+
aliases:
69+
other: {}
70+
71+
- do:
72+
indices.get:
73+
index: bar-baz
74+
75+
- match: {bar-baz.settings.index.number_of_shards: "2"}
76+
- match: {bar-baz.settings.index.number_of_replicas: "0"}
77+
- match: {bar-baz.settings.index.priority: "17"}
78+
- match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}}
79+
- match: {bar-baz.mappings.properties.field2: {type: keyword}}
80+
- match: {bar-baz.mappings.properties.foo: {type: keyword}}
81+
- match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}}
82+
- match: {bar-baz.aliases.my_alias: {}}
83+
- match: {bar-baz.aliases.other: {}}
84+
85+
---
86+
"Index template priority":
87+
- skip:
88+
version: " - 7.7.99"
89+
reason: "itv2 is available in 7.8+"
90+
91+
- do:
92+
indices.put_index_template:
93+
name: my-template
94+
body:
95+
index_patterns: ["foo", "bar-*"]
96+
template:
97+
settings:
98+
index.number_of_shards: 2
99+
composed_of: []
100+
priority: 400
101+
102+
- do:
103+
indices.put_index_template:
104+
name: another-template
105+
body:
106+
index_patterns: ["bar-*"]
107+
template:
108+
settings:
109+
index.number_of_shards: 3
110+
composed_of: []
111+
priority: 405
112+
113+
- do:
114+
indices.create:
115+
index: bar-baz
116+
117+
- do:
118+
indices.get:
119+
index: bar-baz
120+
121+
- match: {bar-baz.settings.index.number_of_shards: "3"}
122+
123+
---
124+
"Component template only composition":
125+
- skip:
126+
version: " - 7.7.99"
127+
reason: "itv2 is available in 7.8+"
128+
129+
- do:
130+
cluster.put_component_template:
131+
name: ct_low
132+
body:
133+
template:
134+
aliases:
135+
alias1: {}
136+
137+
- do:
138+
cluster.put_component_template:
139+
name: ct_high
140+
body:
141+
template:
142+
mappings:
143+
properties:
144+
field:
145+
type: keyword
146+
147+
- do:
148+
indices.put_index_template:
149+
name: my-template
150+
body:
151+
index_patterns: ["baz*"]
152+
composed_of: ["ct_low", "ct_high"]
153+
154+
- do:
155+
indices.create:
156+
index: bazfoo
157+
158+
- do:
159+
indices.get:
160+
index: bazfoo
161+
162+
- match: {bazfoo.mappings.properties.field: {type: keyword}}
163+
- match: {bazfoo.aliases.alias1: {}}
164+
165+
---
166+
"Index template without component templates":
167+
- skip:
168+
version: " - 7.7.99"
169+
reason: "itv2 is available in 7.8+"
170+
171+
- do:
172+
indices.put_index_template:
173+
name: my-template
174+
body:
175+
index_patterns: ["eggplant"]
176+
template:
177+
settings:
178+
number_of_shards: 3
179+
180+
- do:
181+
indices.create:
182+
index: eggplant
183+
184+
- do:
185+
indices.get:
186+
index: eggplant
187+
188+
- match: {eggplant.settings.index.number_of_shards: "3"}

‎server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
import java.util.Locale;
4343
import java.util.regex.Pattern;
4444

45-
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findTemplates;
45+
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
4646

4747
public class MetadataRolloverService {
4848
private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
@@ -165,7 +165,7 @@ static List<AliasAction> rolloverAliasToNewIndex(String oldIndex, String newInde
165165
*/
166166
static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String rolloverIndexName, String rolloverRequestAlias,
167167
@Nullable Boolean isHidden) {
168-
final List<IndexTemplateMetadata> matchedTemplates = findTemplates(metadata, rolloverIndexName, isHidden);
168+
final List<IndexTemplateMetadata> matchedTemplates = findV1Templates(metadata, rolloverIndexName, isHidden);
169169
for (IndexTemplateMetadata template : matchedTemplates) {
170170
if (template.aliases().containsKey(rolloverRequestAlias)) {
171171
throw new IllegalArgumentException(String.format(Locale.ROOT,

‎server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,12 @@
2626
import org.elasticsearch.cluster.ClusterState;
2727
import org.elasticsearch.cluster.block.ClusterBlockException;
2828
import org.elasticsearch.cluster.block.ClusterBlockLevel;
29-
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
30-
import org.elasticsearch.cluster.metadata.IndexMetadata;
3129
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
30+
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
3231
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
33-
import org.elasticsearch.cluster.metadata.Template;
3432
import org.elasticsearch.cluster.service.ClusterService;
3533
import org.elasticsearch.common.inject.Inject;
3634
import org.elasticsearch.common.io.stream.StreamInput;
37-
import org.elasticsearch.common.settings.Settings;
3835
import org.elasticsearch.threadpool.ThreadPool;
3936
import org.elasticsearch.transport.TransportService;
4037

@@ -74,14 +71,6 @@ protected ClusterBlockException checkBlock(PutIndexTemplateV2Action.Request requ
7471
protected void masterOperation(final PutIndexTemplateV2Action.Request request, final ClusterState state,
7572
final ActionListener<AcknowledgedResponse> listener) {
7673
IndexTemplateV2 indexTemplate = request.indexTemplate();
77-
Template template = indexTemplate.template();
78-
// Normalize the index settings if necessary
79-
if (template.settings() != null) {
80-
Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
81-
template = new Template(settings.build(), template.mappings(), template.aliases());
82-
indexTemplate = new IndexTemplateV2(indexTemplate.indexPatterns(), template, indexTemplate.composedOf(),
83-
indexTemplate.priority(), indexTemplate.version(), indexTemplate.metadata());
84-
}
8574
indexTemplateService.putIndexTemplateV2(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),
8675
indexTemplate, listener);
8776
}

‎server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,7 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
311311
}
312312
} else if (indexRequest.index() != null) {
313313
// the index does not exist yet (and this is a valid request), so match index templates to look for pipelines
314-
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findTemplates(metadata, indexRequest.index(), null);
314+
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
315315
assert (templates != null);
316316
// order of templates are highest order first
317317
for (final IndexTemplateMetadata template : templates) {

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.common.xcontent.XContentParser;
3333

3434
import java.io.IOException;
35+
import java.util.Collections;
3536
import java.util.List;
3637
import java.util.Map;
3738
import java.util.Objects;
@@ -114,11 +115,15 @@ public List<String> indexPatterns() {
114115
return indexPatterns;
115116
}
116117

118+
@Nullable
117119
public Template template() {
118120
return template;
119121
}
120122

121123
public List<String> composedOf() {
124+
if (componentTemplates == null) {
125+
return Collections.emptyList();
126+
}
122127
return componentTemplates;
123128
}
124129

0 commit comments

Comments
 (0)