Skip to content

Commit 9f9ade7

Browse files
dakronedakrone
authored
Use V2 index templates during index creation (#54669)
* Use V2 index templates during index creation This commit changes our index creation code to use (and favor!) V2 index templates during index creation. The creation precedence goes like so, in order of precedence: - Existing source `IndexMetadata` - for example, when recovering from a peer or a shrink/split/clone where index templates should not be applied - A matching V2 index template, if one is found - When a V2 template is found, all component templates (in the `composed_of` field) are applied in the order that they appear, with the index template having the 2nd highest precedence (the create index request always has the top priority when it comes to index settings) - All matching V1 templates (the old style) This also adds index template validation when `PUT`-ing a new v2 index template (because this was required) and ensures that all index and component templates specify *no* top-level mapping type (it is automatically added when the template is added to the cluster state). This does not yet implement fine-grained component template merging of mappings, where we favor merging only a single field's configuration, that will be done in subsequent work. This also keeps the existing hidden index behavior present for v1 templates, where a hidden index will match v2 index templates unless they are global (`*`) templates. Relates to #53101
1 parent 5fc9fc5 commit 9f9ade7

File tree

12 files changed

+895
-140
lines changed

12 files changed

+895
-140
lines changed

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
---
2+
"Component and index template composition":
3+
- skip:
4+
version: " - 7.99.99"
5+
reason: "not yet backported"
6+
7+
- do:
8+
cluster.put_component_template:
9+
name: ct_low
10+
body:
11+
template:
12+
settings:
13+
number_of_replicas: 1
14+
mappings:
15+
properties:
16+
field2:
17+
type: text
18+
aliases:
19+
aliasname:
20+
is_write_index: false
21+
22+
- do:
23+
cluster.put_component_template:
24+
name: ct_high
25+
body:
26+
template:
27+
settings:
28+
index.number_of_replicas: 0
29+
mappings:
30+
properties:
31+
field2:
32+
type: keyword
33+
aliases:
34+
aliasname:
35+
is_write_index: true
36+
37+
- do:
38+
indices.put_index_template:
39+
name: my-template
40+
body:
41+
index_patterns: ["foo", "bar-*"]
42+
template:
43+
settings:
44+
index.number_of_shards: 2
45+
mappings:
46+
properties:
47+
field:
48+
type: keyword
49+
ignore_above: 255
50+
aliases:
51+
my_alias: {}
52+
aliasname:
53+
filter:
54+
match_all: {}
55+
composed_of: ["ct_low", "ct_high"]
56+
priority: 400
57+
58+
- do:
59+
indices.create:
60+
index: bar-baz
61+
body:
62+
settings:
63+
index.priority: 17
64+
mappings:
65+
properties:
66+
foo:
67+
type: keyword
68+
aliases:
69+
other: {}
70+
71+
- do:
72+
indices.get:
73+
index: bar-baz
74+
75+
- match: {bar-baz.settings.index.number_of_shards: "2"}
76+
- match: {bar-baz.settings.index.number_of_replicas: "0"}
77+
- match: {bar-baz.settings.index.priority: "17"}
78+
- match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}}
79+
- match: {bar-baz.mappings.properties.field2: {type: keyword}}
80+
- match: {bar-baz.mappings.properties.foo: {type: keyword}}
81+
- match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}}
82+
- match: {bar-baz.aliases.my_alias: {}}
83+
- match: {bar-baz.aliases.other: {}}
84+
85+
---
86+
"Index template priority":
87+
- skip:
88+
version: " - 7.99.99"
89+
reason: "not yet backported"
90+
91+
- do:
92+
indices.put_index_template:
93+
name: my-template
94+
body:
95+
index_patterns: ["foo", "bar-*"]
96+
template:
97+
settings:
98+
index.number_of_shards: 2
99+
composed_of: []
100+
priority: 400
101+
102+
- do:
103+
indices.put_index_template:
104+
name: another-template
105+
body:
106+
index_patterns: ["bar-*"]
107+
template:
108+
settings:
109+
index.number_of_shards: 3
110+
composed_of: []
111+
priority: 405
112+
113+
- do:
114+
indices.create:
115+
index: bar-baz
116+
117+
- do:
118+
indices.get:
119+
index: bar-baz
120+
121+
- match: {bar-baz.settings.index.number_of_shards: "3"}
122+
123+
---
124+
"Component template only composition":
125+
- skip:
126+
version: " - 7.99.99"
127+
reason: "not yet backported"
128+
129+
- do:
130+
cluster.put_component_template:
131+
name: ct_low
132+
body:
133+
template:
134+
aliases:
135+
alias1: {}
136+
137+
- do:
138+
cluster.put_component_template:
139+
name: ct_high
140+
body:
141+
template:
142+
mappings:
143+
properties:
144+
field:
145+
type: keyword
146+
147+
- do:
148+
indices.put_index_template:
149+
name: my-template
150+
body:
151+
index_patterns: ["baz*"]
152+
composed_of: ["ct_low", "ct_high"]
153+
154+
- do:
155+
indices.create:
156+
index: bazfoo
157+
158+
- do:
159+
indices.get:
160+
index: bazfoo
161+
162+
- match: {bazfoo.mappings.properties.field: {type: keyword}}
163+
- match: {bazfoo.aliases.alias1: {}}
164+
165+
---
166+
"Index template without component templates":
167+
- skip:
168+
version: " - 7.99.99"
169+
reason: "not yet backported"
170+
171+
- do:
172+
indices.put_index_template:
173+
name: my-template
174+
body:
175+
index_patterns: ["eggplant"]
176+
template:
177+
settings:
178+
number_of_shards: 3
179+
180+
- do:
181+
indices.create:
182+
index: eggplant
183+
184+
- do:
185+
indices.get:
186+
index: eggplant
187+
188+
- match: {eggplant.settings.index.number_of_shards: "3"}

‎server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
import java.util.Locale;
4141
import java.util.regex.Pattern;
4242

43-
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findTemplates;
43+
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
4444

4545
public class MetadataRolloverService {
4646
private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
@@ -161,10 +161,9 @@ static List<AliasAction> rolloverAliasToNewIndex(String oldIndex, String newInde
161161
* the rollover alias will point to multiple indices. This causes indexing requests to be rejected.
162162
* To avoid this, we make sure that there is no duplicated alias in index templates before creating a new index.
163163
*/
164-
static void checkNoDuplicatedAliasInIndexTemplate(
165-
Metadata metadata, String rolloverIndexName, String rolloverRequestAlias,
166-
@Nullable Boolean isHidden) {
167-
final List<IndexTemplateMetadata> matchedTemplates = findTemplates(metadata, rolloverIndexName, isHidden);
164+
static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String rolloverIndexName, String rolloverRequestAlias,
165+
@Nullable Boolean isHidden) {
166+
final List<IndexTemplateMetadata> matchedTemplates = findV1Templates(metadata, rolloverIndexName, isHidden);
168167
for (IndexTemplateMetadata template : matchedTemplates) {
169168
if (template.aliases().containsKey(rolloverRequestAlias)) {
170169
throw new IllegalArgumentException(String.format(Locale.ROOT,

‎server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,12 @@
2626
import org.elasticsearch.cluster.ClusterState;
2727
import org.elasticsearch.cluster.block.ClusterBlockException;
2828
import org.elasticsearch.cluster.block.ClusterBlockLevel;
29-
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
30-
import org.elasticsearch.cluster.metadata.IndexMetadata;
3129
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
30+
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
3231
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
33-
import org.elasticsearch.cluster.metadata.Template;
3432
import org.elasticsearch.cluster.service.ClusterService;
3533
import org.elasticsearch.common.inject.Inject;
3634
import org.elasticsearch.common.io.stream.StreamInput;
37-
import org.elasticsearch.common.settings.Settings;
3835
import org.elasticsearch.tasks.Task;
3936
import org.elasticsearch.threadpool.ThreadPool;
4037
import org.elasticsearch.transport.TransportService;
@@ -75,14 +72,6 @@ protected ClusterBlockException checkBlock(PutIndexTemplateV2Action.Request requ
7572
protected void masterOperation(Task task, final PutIndexTemplateV2Action.Request request, final ClusterState state,
7673
final ActionListener<AcknowledgedResponse> listener) {
7774
IndexTemplateV2 indexTemplate = request.indexTemplate();
78-
Template template = indexTemplate.template();
79-
// Normalize the index settings if necessary
80-
if (template.settings() != null) {
81-
Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
82-
template = new Template(settings.build(), template.mappings(), template.aliases());
83-
indexTemplate = new IndexTemplateV2(indexTemplate.indexPatterns(), template, indexTemplate.composedOf(),
84-
indexTemplate.priority(), indexTemplate.version(), indexTemplate.metadata());
85-
}
8675
indexTemplateService.putIndexTemplateV2(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),
8776
indexTemplate, listener);
8877
}

‎server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
307307
}
308308
} else if (indexRequest.index() != null) {
309309
// the index does not exist yet (and this is a valid request), so match index templates to look for pipelines
310-
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findTemplates(metadata, indexRequest.index(), null);
310+
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
311311
assert (templates != null);
312312
// order of templates are highest order first
313313
for (final IndexTemplateMetadata template : templates) {

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,15 @@ public List<String> indexPatterns() {
114114
return indexPatterns;
115115
}
116116

117+
@Nullable
117118
public Template template() {
118119
return template;
119120
}
120121

121122
public List<String> composedOf() {
123+
if (componentTemplates == null) {
124+
return List.of();
125+
}
122126
return componentTemplates;
123127
}
124128

0 commit comments

Comments
 (0)