Skip to content

Commit 6cd648e

Browse files
committed
Add a template parameter to override auto_create_index value (elastic#61858)
Closes elastic#20640. This PR introduces a new parameter to v2 templates, `allow_auto_create`, which allows templates to override the cluster setting `auto_create_index`. Notes: * `AutoCreateIndex` now looks for a matching v2 template, and if its `allow_auto_create` setting is true, it overrides the usual logic. * `TransportBulkAction` previously used `AutoCreateIndex` to check whether missing indices should be created. We now rely on `AutoCreateAction`, which was already differentiating between creating indices and creating data streams. I've updated `AutoCreateAction` to use `AutoCreateIndex`. Data streams are also influenced by `allow_auto_create`, in that their default auto-create behaviour can be disabled with this setting. * Most of the Java file changes are due to introducing an extra constructor parameter to `ComposableIndexTemplate`. * I've added the new setting to various x-pack templates * I added a YAML test to check that watches can be created even when `auto_create_index` is `false`.
1 parent 9a690cd commit 6cd648e

File tree

46 files changed

+770
-322
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+770
-322
lines changed

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -2040,7 +2040,7 @@ public void testDataStreams() throws Exception {
20402040
CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"@timestamp\":{\"type\":\"date\"}}}");
20412041
Template template = new Template(null, mappings, null);
20422042
ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(Collections.singletonList(dataStreamName), template,
2043-
Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate());
2043+
Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate(), null);
20442044
PutComposableIndexTemplateRequest putComposableIndexTemplateRequest =
20452045
new PutComposableIndexTemplateRequest().name("ds-template").create(true).indexTemplate(indexTemplate);
20462046
AcknowledgedResponse response = execute(putComposableIndexTemplateRequest,
@@ -2119,7 +2119,7 @@ public void testIndexTemplates() throws Exception {
21192119
Template template = new Template(settings, mappings, Collections.singletonMap("alias", alias));
21202120
List<String> pattern = Collections.singletonList("pattern");
21212121
ComposableIndexTemplate indexTemplate =
2122-
new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null);
2122+
new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null);
21232123
PutComposableIndexTemplateRequest putComposableIndexTemplateRequest =
21242124
new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate);
21252125

@@ -2166,7 +2166,7 @@ public void testSimulateIndexTemplate() throws Exception {
21662166
Template template = new Template(settings, mappings, org.elasticsearch.common.collect.Map.of("alias", alias));
21672167
List<String> pattern = org.elasticsearch.common.collect.List.of("pattern");
21682168
ComposableIndexTemplate indexTemplate =
2169-
new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null);
2169+
new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null);
21702170
PutComposableIndexTemplateRequest putComposableIndexTemplateRequest =
21712171
new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate);
21722172

@@ -2178,7 +2178,7 @@ public void testSimulateIndexTemplate() throws Exception {
21782178
AliasMetadata simulationAlias = AliasMetadata.builder("simulation-alias").writeIndex(true).build();
21792179
ComposableIndexTemplate simulationTemplate = new ComposableIndexTemplate(pattern, new Template(null, null,
21802180
org.elasticsearch.common.collect.Map.of("simulation-alias", simulationAlias)), Collections.emptyList(), 2L, 1L,
2181-
new HashMap<>(), null);
2181+
new HashMap<>(), null, null);
21822182
PutComposableIndexTemplateRequest newIndexTemplateReq =
21832183
new PutComposableIndexTemplateRequest().name("used-for-simulation").create(true).indexTemplate(indexTemplate);
21842184
newIndexTemplateReq.indexTemplate(simulationTemplate);

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,6 @@ private static ComposableIndexTemplate randomIndexTemplate() {
8888
if (randomBoolean()) {
8989
dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate();
9090
}
91-
return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate);
91+
return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate, null);
9292
}
9393
}

docs/reference/indices/put-component-template.asciidoc

+19-9
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@
44
<titleabbrev>Put component template</titleabbrev>
55
++++
66

7-
Creates or updates a component template.
8-
Component templates are building blocks for constructing <<indices-templates,index templates>>.
9-
that specify index <<mapping,mappings>>, <<index-modules-settings,settings>>,
10-
and <<indices-aliases,aliases>>.
7+
Creates or updates a component template.
8+
Component templates are building blocks for constructing <<indices-templates,index templates>>.
9+
that specify index <<mapping,mappings>>, <<index-modules-settings,settings>>,
10+
and <<indices-aliases,aliases>>.
1111

1212
[source,console]
1313
--------------------------------------------------
@@ -55,10 +55,10 @@ DELETE _component_template/template_*
5555
[[put-component-template-api-desc]]
5656
==== {api-description-title}
5757

58-
An index template can be composed of multiple component templates.
58+
An index template can be composed of multiple component templates.
5959
To use a component template, specify it in an index template's `composed_of` list.
60-
Component templates are only applied to new data streams and indices
61-
as part of a matching index template.
60+
Component templates are only applied to new data streams and indices
61+
as part of a matching index template.
6262

6363
Settings and mappings specified directly in the index template or the <<indices-create-index, create index>>
6464
request override any settings or mappings specified in a component template.
@@ -112,6 +112,16 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings]
112112
Version number used to manage component templates externally.
113113
This number is not automatically generated or incremented by {es}.
114114

115+
`allow_auto_create`::
116+
(Optional, boolean)
117+
This setting overrides the value of the
118+
<<index-creation,`action.auto_create_index`>> cluster setting. If set to
119+
`true` in a template, then indices can be automatically created using that
120+
template even if auto-creation of indices is disabled via
121+
`actions.auto_create_index`. If set to `false`, then indices or data streams matching the
122+
template must always be explicitly created, and may never be automatically
123+
created.
124+
115125
`_meta`::
116126
(Optional, object)
117127
Optional user metadata about the component template. May have any contents.
@@ -157,7 +167,7 @@ To be applied, a component template must be included in an index template's `com
157167
[[component-templates-version]]
158168
===== Component template versioning
159169

160-
You can use the `version` parameter to add a version number to a component template.
170+
You can use the `version` parameter to add a version number to a component template.
161171
External systems can use these version numbers to simplify template management.
162172

163173
The `version` parameter is optional and not automatically generated or used by {es}.
@@ -182,7 +192,7 @@ To check the `version`, you can use the <<getting-component-templates,get compon
182192
[[component-templates-metadata]]
183193
===== Component template metadata
184194

185-
You can use the `_meta` parameter to add arbitrary metadata to a component template.
195+
You can use the `_meta` parameter to add arbitrary metadata to a component template.
186196
This user-defined object is stored in the cluster state,
187197
so keeping it short is preferrable.
188198

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.http;
21+
22+
import org.elasticsearch.action.support.AutoCreateIndex;
23+
import org.elasticsearch.client.Request;
24+
import org.elasticsearch.client.Response;
25+
import org.elasticsearch.client.ResponseException;
26+
import org.elasticsearch.common.Strings;
27+
import org.elasticsearch.common.io.Streams;
28+
import org.elasticsearch.common.xcontent.XContentBuilder;
29+
import org.elasticsearch.common.xcontent.json.JsonXContent;
30+
import org.elasticsearch.test.rest.ESRestTestCase;
31+
32+
import java.io.IOException;
33+
import java.io.InputStreamReader;
34+
35+
import static java.nio.charset.StandardCharsets.UTF_8;
36+
import static org.hamcrest.Matchers.containsString;
37+
38+
public class AutoCreateIndexIT extends ESRestTestCase {
39+
40+
/**
41+
* Check that setting {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} to <code>false</code>
42+
* disable the automatic creation on indices.
43+
*/
44+
public void testCannotAutoCreateIndexWhenDisabled() throws IOException {
45+
configureAutoCreateIndex(false);
46+
47+
// Attempt to add a document to a non-existing index. Auto-creating the index should fail owing to the setting above.
48+
final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456");
49+
indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }");
50+
final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument);
51+
52+
assertThat(
53+
Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)),
54+
containsString("no such index [recipe_kr] and [action.auto_create_index] is [false]")
55+
);
56+
}
57+
58+
/**
59+
* Check that automatically creating an index is allowed, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING}
60+
* is <code>false</code>, when the index name matches a template and that template has <code>allow_auto_create</code>
61+
* set to <code>true</code>.
62+
*/
63+
public void testCanAutoCreateIndexWhenAllowedByTemplate() throws IOException {
64+
configureAutoCreateIndex(false);
65+
66+
createTemplateWithAllowAutoCreate(true);
67+
68+
// Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name
69+
// matches the template pattern
70+
assertOK(this.indexDocument());
71+
}
72+
73+
/**
74+
* Check that automatically creating an index is disallowed when the index name matches a template and that template has
75+
* <code>allow_auto_create</code> explicitly to <code>false</code>, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING}
76+
* is set to <code>true</code>.
77+
*/
78+
public void testCannotAutoCreateIndexWhenDisallowedByTemplate() throws IOException {
79+
configureAutoCreateIndex(true);
80+
81+
createTemplateWithAllowAutoCreate(false);
82+
83+
// Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name
84+
// matches the template pattern
85+
final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument);
86+
87+
assertThat(
88+
Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)),
89+
containsString("no such index [composable template [recipe*] forbids index auto creation]")
90+
);
91+
}
92+
93+
94+
private void configureAutoCreateIndex(boolean value) throws IOException {
95+
XContentBuilder builder = JsonXContent.contentBuilder()
96+
.startObject()
97+
.startObject("transient")
98+
.field(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), value)
99+
.endObject()
100+
.endObject();
101+
102+
final Request settingsRequest = new Request("PUT", "_cluster/settings");
103+
settingsRequest.setJsonEntity(Strings.toString(builder));
104+
final Response settingsResponse = client().performRequest(settingsRequest);
105+
assertOK(settingsResponse);
106+
}
107+
108+
private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws IOException {
109+
XContentBuilder builder = JsonXContent.contentBuilder()
110+
.startObject()
111+
.array("index_patterns", "recipe*")
112+
.field("allow_auto_create", allowAutoCreate)
113+
.endObject();
114+
115+
final Request createTemplateRequest = new Request("PUT", "_index_template/recipe_template");
116+
createTemplateRequest.setJsonEntity(Strings.toString(builder));
117+
final Response createTemplateResponse = client().performRequest(createTemplateRequest);
118+
assertOK(createTemplateResponse);
119+
}
120+
121+
private Response indexDocument() throws IOException {
122+
final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456");
123+
indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }");
124+
return client().performRequest(indexDocumentRequest);
125+
}
126+
}

server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
2626
import org.elasticsearch.test.ESIntegTestCase.Scope;
2727

28+
import static org.hamcrest.Matchers.equalTo;
29+
2830
@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
2931
public class BulkProcessorClusterSettingsIT extends ESIntegTestCase {
3032
public void testBulkProcessorAutoCreateRestrictions() throws Exception {
@@ -45,7 +47,9 @@ public void testBulkProcessorAutoCreateRestrictions() throws Exception {
4547
assertEquals(3, responses.length);
4648
assertFalse("Operation on existing index should succeed", responses[0].isFailed());
4749
assertTrue("Missing index should have been flagged", responses[1].isFailed());
48-
assertEquals("[wontwork] IndexNotFoundException[no such index [wontwork]]", responses[1].getFailureMessage());
50+
assertThat(
51+
responses[1].getFailureMessage(),
52+
equalTo("[wontwork] IndexNotFoundException: no such index [wontwork] and [action.auto_create_index] is [false]"));
4953
assertFalse("Operation on existing index should succeed", responses[2].isFailed());
5054
}
5155
}

server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ public void testUsageOfDataStreamFails() throws IOException {
102102
Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser));
103103

104104
ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null,
105-
null, new ComposableIndexTemplate.DataStreamTemplate());
105+
null, new ComposableIndexTemplate.DataStreamTemplate(), null);
106106
Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE,
107107
new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet());
108108
Exception actualException = (Exception) e.getCause();

0 commit comments

Comments
 (0)