Skip to content

Commit 8650ca0

Browse files
dakronedakrone
committed
Allocate new indices on "hot" or "content" tier depending on data stream inclusion
This commit changes the default allocation on the "hot" tier to allocating the newly created index to the "hot" tier if it is part of a new or existing data stream, and to the "content" tier if it is not part of a data stream. Overriding any of the `index.routing.allocation.(include|exclude|require).*` settings continues to cause the initial allocation not to be set (no change in behavior). Relates to elastic#60848
1 parent f29c743 commit 8650ca0

File tree

5 files changed

+140
-7
lines changed

5 files changed

+140
-7
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -629,10 +629,11 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
629629
.put(request.settings())
630630
.build();
631631

632+
final boolean isDataStreamIndex = request.dataStreamName() != null;
632633
// Loop through all the explicit index setting providers, adding them to the
633634
// additionalIndexSettings map
634635
for (IndexSettingProvider provider : indexSettingProviders) {
635-
additionalIndexSettings.put(provider.getAdditionalIndexSettings(request.index(), templateAndRequestSettings));
636+
additionalIndexSettings.put(provider.getAdditionalIndexSettings(request.index(), isDataStreamIndex, templateAndRequestSettings));
636637
}
637638

638639
// For all the explicit settings, we go through the template and request level settings

‎server/src/main/java/org/elasticsearch/index/shard/IndexSettingProvider.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public interface IndexSettingProvider {
3030
* Returns explicitly set default index {@link Settings} for the given index. This should not
3131
* return null.
3232
*/
33-
default Settings getAdditionalIndexSettings(String indexName, Settings templateAndRequestSettings) {
33+
default Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings templateAndRequestSettings) {
3434
return Settings.EMPTY;
3535
}
3636
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ public static class DefaultHotAllocationSettingProvider implements IndexSettingP
177177
private static final Logger logger = LogManager.getLogger(DefaultHotAllocationSettingProvider.class);
178178

179179
@Override
180-
public Settings getAdditionalIndexSettings(String indexName, Settings indexSettings) {
180+
public Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings indexSettings) {
181181
Set<String> settings = indexSettings.keySet();
182182
if (settings.contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)) {
183183
// It's okay to put it, it will be removed or overridden by the template/request settings
@@ -186,11 +186,17 @@ public Settings getAdditionalIndexSettings(String indexName, Settings indexSetti
186186
settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + ".")) ||
187187
settings.stream().anyMatch(s -> s.startsWith(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "."))) {
188188
// A different index level require, include, or exclude has been specified, so don't put the setting
189-
logger.debug("index [{}] specifies custom index level routing filtering, skipping hot tier allocation", indexName);
189+
logger.debug("index [{}] specifies custom index level routing filtering, skipping tier allocation", indexName);
190190
return Settings.EMPTY;
191191
} else {
192-
// Otherwise, put the setting in place by default
193-
return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build();
192+
// Otherwise, put the setting in place by default, the "hot"
193+
// tier if the index is part of a data stream, the "content"
194+
// tier if it is not.
195+
if (isDataStreamIndex) {
196+
return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_HOT).build();
197+
} else {
198+
return Settings.builder().put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, DATA_CONTENT).build();
199+
}
194200
}
195201
}
196202
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,42 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
3333
return Collections.singleton(LocalStateCompositeXPackPlugin.class);
3434
}
3535

36-
public void testDefaultAllocateToHot() {
36+
public void testDefaultIndexAllocateToContent() {
3737
startWarmOnlyNode();
3838
startColdOnlyNode();
3939
ensureGreen();
4040

4141
client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
4242

43+
Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
44+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_CONTENT));
45+
46+
// index should be red
47+
assertThat(client().admin().cluster().prepareHealth(index).get().getIndices().get(index).getStatus(),
48+
equalTo(ClusterHealthStatus.RED));
49+
50+
logger.info("--> starting hot node");
51+
if (randomBoolean()) {
52+
startContentOnlyNode();
53+
} else {
54+
startDataNode();
55+
}
56+
57+
logger.info("--> waiting for {} to be yellow", index);
58+
ensureYellow(index);
59+
}
60+
61+
public void testDefaultDataStreamAllocateToHot() {
62+
startWarmOnlyNode();
63+
startColdOnlyNode();
64+
ensureGreen();
65+
66+
ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(index),
67+
null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
68+
client().execute(PutComposableIndexTemplateAction.INSTANCE,
69+
new PutComposableIndexTemplateAction.Request("template").indexTemplate(template)).actionGet();
70+
client().prepareIndex(index).setWaitForActiveShards(0).get();
71+
4372
Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
4473
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
4574

@@ -189,6 +218,20 @@ public void testTemplateOverridesDefaults() {
189218
ensureYellow(index);
190219
}
191220

221+
public void startDataNode() {
222+
Settings nodeSettings = Settings.builder()
223+
.putList("node.roles", Arrays.asList("master", "data", "ingest"))
224+
.build();
225+
internalCluster().startNode(nodeSettings);
226+
}
227+
228+
public void startContentOnlyNode() {
229+
Settings nodeSettings = Settings.builder()
230+
.putList("node.roles", Arrays.asList("master", "data_content", "ingest"))
231+
.build();
232+
internalCluster().startNode(nodeSettings);
233+
}
234+
192235
public void startHotOnlyNode() {
193236
Settings nodeSettings = Settings.builder()
194237
.putList("node.roles", Arrays.asList("master", "data_hot", "ingest"))

‎x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.datastreams;
8+
9+
import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
10+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
11+
import org.elasticsearch.common.settings.Settings;
12+
import org.elasticsearch.plugins.Plugin;
13+
import org.elasticsearch.test.ESIntegTestCase;
14+
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
15+
import org.elasticsearch.xpack.core.DataTier;
16+
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
17+
import org.elasticsearch.xpack.core.action.DeleteDataStreamAction;
18+
19+
import java.util.Arrays;
20+
import java.util.Collection;
21+
import java.util.Collections;
22+
23+
import static org.hamcrest.Matchers.equalTo;
24+
25+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
26+
public class DataTierDataStreamIT extends ESIntegTestCase {
27+
private static final String index = "myindex";
28+
29+
@Override
30+
protected Collection<Class<? extends Plugin>> nodePlugins() {
31+
return Arrays.asList(LocalStateCompositeXPackPlugin.class, DataStreamsPlugin.class);
32+
}
33+
34+
public void testDefaultDataStreamAllocateToHot() {
35+
startHotOnlyNode();
36+
ensureGreen();
37+
38+
ComposableIndexTemplate template = new ComposableIndexTemplate(
39+
Collections.singletonList(index),
40+
null,
41+
null,
42+
null,
43+
null,
44+
null,
45+
new ComposableIndexTemplate.DataStreamTemplate()
46+
);
47+
client().execute(
48+
PutComposableIndexTemplateAction.INSTANCE,
49+
new PutComposableIndexTemplateAction.Request("template").indexTemplate(template)
50+
).actionGet();
51+
client().prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").setWaitForActiveShards(0).get();
52+
53+
Settings idxSettings = client().admin()
54+
.indices()
55+
.prepareGetIndex()
56+
.addIndices(index)
57+
.get()
58+
.getSettings()
59+
.get(".ds-" + index + "-000001");
60+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
61+
62+
logger.info("--> waiting for {} to be yellow", index);
63+
ensureYellow(index);
64+
65+
// Roll over index and ensure the second index also went to the "hot" tier
66+
client().admin().indices().prepareRolloverIndex(index).get();
67+
idxSettings = client().admin()
68+
.indices()
69+
.prepareGetIndex()
70+
.addIndices(index)
71+
.get()
72+
.getSettings()
73+
.get(".ds-" + index + "-000002");
74+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
75+
76+
client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { index }));
77+
}
78+
79+
public void startHotOnlyNode() {
80+
Settings nodeSettings = Settings.builder().putList("node.roles", Arrays.asList("master", "data_hot", "ingest")).build();
81+
internalCluster().startNode(nodeSettings);
82+
}
83+
}

0 commit comments

Comments
 (0)