Skip to content

Commit 4a08928

Browse files
dakronedakrone
authored
[7.x] Add index.routing.allocation.include._tier_preference setting (#62589) (#62667)
This commit adds the `index.routing.allocation.prefer._tier` setting to the `DataTierAllocationDecider`. This special-purpose allocation setting lets a user specify a preference-based list of tiers for an index to be assigned to. For example, if the setting were set to: ``` "index.routing.allocation.prefer._tier": "data_hot,data_warm,data_content" ``` If the cluster contains any nodes with the `data_hot` role, the decider will only allow them to be allocated on the `data_hot` node(s). If there are no `data_hot` nodes, but there are `data_warm` and `data_content` nodes, then the index will be allowed to be allocated on `data_warm` nodes. This allows us to specify an index's preference for tier(s) without causing the index to be unassigned if no nodes of a preferred tier are available. Subsequent work will change the ILM migration to make additional use of this setting. Relates to #60848
1 parent c2e73ba commit 4a08928

File tree

13 files changed

+435
-36
lines changed

13 files changed

+435
-36
lines changed

‎docs/reference/api-conventions.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@ Returns:
396396
"index.creation_date": "1474389951325",
397397
"index.uuid": "n6gzFZTgS664GUfx0Xrpjw",
398398
"index.version.created": ...,
399-
"index.routing.allocation.include._tier" : "data_content",
399+
"index.routing.allocation.include._tier_preference" : "data_content",
400400
"index.provided_name" : "my-index-000001"
401401
}
402402
}
@@ -433,7 +433,7 @@ Returns:
433433
"routing": {
434434
"allocation": {
435435
"include": {
436-
"_tier": "data_content"
436+
"_tier_preference": "data_content"
437437
}
438438
}
439439
},

‎docs/reference/cluster/allocation-explain.asciidoc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ PUT /my-index-000001?master_timeout=1s&timeout=1s
116116
{
117117
"settings": {
118118
"index.routing.allocation.include._name": "non_existent_node",
119-
"index.routing.allocation.include._tier": null
119+
"index.routing.allocation.include._tier_preference": null
120120
}
121121
}
122122

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
250250
// but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
251251
// before timing out
252252
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
253-
.put("index.routing.allocation.include._tier", "")
253+
.put("index.routing.allocation.include._tier_preference", "")
254254
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
255255
createIndex(index, settings.build());
256256
indexDocs(index, 0, 10);
@@ -267,7 +267,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
267267
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
268268
.put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null)
269269
.put("index.routing.allocation.include._id", oldNode)
270-
.putNull("index.routing.allocation.include._tier")
270+
.putNull("index.routing.allocation.include._tier_preference")
271271
);
272272
ensureGreen(index); // wait for the primary to be assigned
273273
ensureNoInitializingShards(); // wait for all other shard activity to finish
@@ -290,7 +290,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
290290
updateIndexSettings(index, Settings.builder()
291291
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
292292
.put("index.routing.allocation.include._id", (String)null)
293-
.putNull("index.routing.allocation.include._tier")
293+
.putNull("index.routing.allocation.include._tier_preference")
294294
);
295295
asyncIndexDocs(index, 60, 45).get();
296296
ensureGreen(index);

‎server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ public boolean match(DiscoveryNode node) {
181181
}
182182
}
183183
}
184-
} else if ("_tier".equals(attr)) {
184+
} else if (attr != null && attr.startsWith("_tier")) {
185185
// Always allow _tier as an attribute, will be handled elsewhere
186186
return true;
187187
} else {

‎x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public void testDoNotAllocateFollowerPrimaryToNodesWithoutRemoteClusterClientRol
5050
final PutFollowAction.Request putFollowRequest = putFollow(leaderIndex, followerIndex);
5151
putFollowRequest.setSettings(Settings.builder()
5252
.put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes))
53-
.putNull("index.routing.allocation.include._tier")
53+
.putNull("index.routing.allocation.include._tier_preference")
5454
.build());
5555
putFollowRequest.waitForActiveShards(ActiveShardCount.ONE);
5656
putFollowRequest.timeout(TimeValue.timeValueSeconds(2));
@@ -84,7 +84,7 @@ public void testAllocateFollowerPrimaryToNodesWithRemoteClusterClientRole() thro
8484
.put("index.routing.rebalance.enable", "none")
8585
.put("index.routing.allocation.include._name",
8686
Stream.concat(dataOnlyNodes.stream(), dataAndRemoteNodes.stream()).collect(Collectors.joining(",")))
87-
.putNull("index.routing.allocation.include._tier")
87+
.putNull("index.routing.allocation.include._tier_preference")
8888
.build());
8989
final PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, putFollowRequest).get();
9090
assertTrue(response.isFollowIndexShardsAcked());
@@ -108,8 +108,8 @@ public void testAllocateFollowerPrimaryToNodesWithRemoteClusterClientRole() thro
108108
followerClient().admin().indices().prepareUpdateSettings(followerIndex)
109109
.setMasterNodeTimeout(TimeValue.MAX_VALUE)
110110
.setSettings(Settings.builder()
111-
.put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes))
112-
.putNull("index.routing.allocation.include._tier"))
111+
.putNull("index.routing.allocation.include._tier_preference")
112+
.put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes)))
113113
.get();
114114
assertBusy(() -> {
115115
final ClusterState state = getFollowerCluster().client().admin().cluster().prepareState().get().getState();

‎x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public void testDefaultIndexAllocateToContent() {
4141
client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
4242

4343
Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
44-
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_CONTENT));
44+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(idxSettings), equalTo(DataTier.DATA_CONTENT));
4545

4646
// index should be red
4747
assertThat(client().admin().cluster().prepareHealth(index).get().getIndices().get(index).getStatus(),
@@ -51,7 +51,7 @@ public void testDefaultIndexAllocateToContent() {
5151
logger.info("--> starting content node");
5252
startContentOnlyNode();
5353
} else {
54-
logger.info("--> starting hot node");
54+
logger.info("--> starting data node");
5555
startDataNode();
5656
}
5757

@@ -65,7 +65,7 @@ public void testOverrideDefaultAllocation() {
6565
ensureGreen();
6666

6767
String setting = randomBoolean() ? DataTierAllocationDecider.INDEX_ROUTING_REQUIRE :
68-
DataTierAllocationDecider.INDEX_ROUTING_INCLUDE;
68+
DataTierAllocationDecider.INDEX_ROUTING_PREFER;
6969

7070
client().admin().indices().prepareCreate(index)
7171
.setWaitForActiveShards(0)
@@ -89,31 +89,31 @@ public void testRequestSettingOverridesAllocation() {
8989
client().admin().indices().prepareCreate(index)
9090
.setWaitForActiveShards(0)
9191
.setSettings(Settings.builder()
92-
.putNull(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE))
92+
.putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER))
9393
.get();
9494

9595
Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
96-
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(""));
96+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(idxSettings), equalTo(""));
9797
// Even the key shouldn't exist if it has been nulled out
98-
assertFalse(idxSettings.keySet().toString(), idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE));
98+
assertFalse(idxSettings.keySet().toString(), idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_PREFER));
9999

100100
// index should be yellow
101101
logger.info("--> waiting for {} to be yellow", index);
102102
ensureYellow(index);
103103

104104
client().admin().indices().prepareDelete(index).get();
105105

106-
// Now test it overriding the "require" setting, in which case the include should be skipped
106+
// Now test it overriding the "require" setting, in which case the preference should be skipped
107107
client().admin().indices().prepareCreate(index)
108108
.setWaitForActiveShards(0)
109109
.setSettings(Settings.builder()
110110
.put(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE, DataTier.DATA_COLD))
111111
.get();
112112

113113
idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
114-
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(""));
114+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(idxSettings), equalTo(""));
115115
// The key should not be put in place since it was overridden
116-
assertFalse(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE));
116+
assertFalse(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_PREFER));
117117
assertThat(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE_SETTING.get(idxSettings), equalTo(DataTier.DATA_COLD));
118118

119119
// index should be yellow
@@ -134,7 +134,7 @@ public void testShrinkStaysOnTier() {
134134
.setSettings(Settings.builder()
135135
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
136136
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
137-
.put(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE, "data_warm"))
137+
.put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, "data_warm"))
138138
.get();
139139

140140
client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.READ_ONLY, index).get();
@@ -150,7 +150,7 @@ public void testShrinkStaysOnTier() {
150150
Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index + "-shrunk")
151151
.get().getSettings().get(index + "-shrunk");
152152
// It should inherit the setting of its originator
153-
assertThat(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING.get(idxSettings), equalTo(DataTier.DATA_WARM));
153+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(idxSettings), equalTo(DataTier.DATA_WARM));
154154

155155
// Required or else the test cleanup fails because it can't delete the indices
156156
client().admin().indices().prepareUpdateSettings(index, index + "-shrunk")
@@ -172,15 +172,15 @@ public void testTemplateOverridesDefaults() {
172172
client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
173173

174174
Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
175-
assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE), equalTo(false));
175+
assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_PREFER), equalTo(false));
176176

177177
// index should be yellow
178178
ensureYellow(index);
179179

180180
client().admin().indices().prepareDelete(index).get();
181181

182182
t = new Template(Settings.builder()
183-
.putNull(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE)
183+
.putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER)
184184
.build(), null, null);
185185
ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null);
186186
client().execute(PutComposableIndexTemplateAction.INSTANCE,
@@ -189,7 +189,7 @@ public void testTemplateOverridesDefaults() {
189189
client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get();
190190

191191
idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index);
192-
assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_INCLUDE), equalTo(false));
192+
assertThat(idxSettings.keySet().contains(DataTierAllocationDecider.INDEX_ROUTING_PREFER), equalTo(false));
193193

194194
ensureYellow(index);
195195
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java

Lines changed: 74 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66

77
package org.elasticsearch.xpack.cluster.routing.allocation;
88

9+
import com.carrotsearch.hppc.cursors.ObjectCursor;
910
import org.elasticsearch.cluster.metadata.IndexMetadata;
1011
import org.elasticsearch.cluster.node.DiscoveryNode;
1112
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
13+
import org.elasticsearch.cluster.node.DiscoveryNodes;
1214
import org.elasticsearch.cluster.routing.RoutingNode;
1315
import org.elasticsearch.cluster.routing.ShardRouting;
1416
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -21,6 +23,7 @@
2123
import org.elasticsearch.xpack.core.DataTier;
2224

2325
import java.util.Arrays;
26+
import java.util.Optional;
2427
import java.util.Set;
2528
import java.util.stream.Collectors;
2629

@@ -38,6 +41,7 @@ public class DataTierAllocationDecider extends AllocationDecider {
3841
public static final String CLUSTER_ROUTING_EXCLUDE = "cluster.routing.allocation.exclude._tier";
3942
public static final String INDEX_ROUTING_REQUIRE = "index.routing.allocation.require._tier";
4043
public static final String INDEX_ROUTING_INCLUDE = "index.routing.allocation.include._tier";
44+
public static final String INDEX_ROUTING_PREFER = "index.routing.allocation.include._tier_preference";
4145
public static final String INDEX_ROUTING_EXCLUDE = "index.routing.allocation.exclude._tier";
4246

4347
public static final Setting<String> CLUSTER_ROUTING_REQUIRE_SETTING = Setting.simpleString(CLUSTER_ROUTING_REQUIRE,
@@ -52,6 +56,8 @@ public class DataTierAllocationDecider extends AllocationDecider {
5256
DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
5357
public static final Setting<String> INDEX_ROUTING_EXCLUDE_SETTING = Setting.simpleString(INDEX_ROUTING_EXCLUDE,
5458
DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
59+
public static final Setting<String> INDEX_ROUTING_PREFER_SETTING = Setting.simpleString(INDEX_ROUTING_PREFER,
60+
DataTierAllocationDecider::validateTierSetting, Setting.Property.Dynamic, Setting.Property.IndexScope);
5561

5662
private static void validateTierSetting(String setting) {
5763
if (Strings.hasText(setting)) {
@@ -101,7 +107,12 @@ public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNod
101107
return decision;
102108
}
103109

104-
return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require tier filters");
110+
decision = shouldIndexPreferTier(indexMetadata, node, allocation);
111+
if (decision != null) {
112+
return decision;
113+
}
114+
115+
return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require/prefer tier filters");
105116
}
106117

107118
private Decision shouldFilter(ShardRouting shardRouting, DiscoveryNode node, RoutingAllocation allocation) {
@@ -115,7 +126,12 @@ private Decision shouldFilter(ShardRouting shardRouting, DiscoveryNode node, Rou
115126
return decision;
116127
}
117128

118-
return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require tier filters");
129+
decision = shouldIndexPreferTier(allocation.metadata().getIndexSafe(shardRouting.index()), node, allocation);
130+
if (decision != null) {
131+
return decision;
132+
}
133+
134+
return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require/prefer tier filters");
119135
}
120136

121137
private Decision shouldFilter(IndexMetadata indexMd, DiscoveryNode node, RoutingAllocation allocation) {
@@ -129,7 +145,37 @@ private Decision shouldFilter(IndexMetadata indexMd, DiscoveryNode node, Routing
129145
return decision;
130146
}
131147

132-
return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require tier filters");
148+
decision = shouldIndexPreferTier(indexMd, node, allocation);
149+
if (decision != null) {
150+
return decision;
151+
}
152+
153+
return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require/prefer tier filters");
154+
}
155+
156+
private Decision shouldIndexPreferTier(IndexMetadata indexMetadata, DiscoveryNode node, RoutingAllocation allocation) {
157+
Settings indexSettings = indexMetadata.getSettings();
158+
String tierPreference = INDEX_ROUTING_PREFER_SETTING.get(indexSettings);
159+
160+
if (Strings.hasText(tierPreference)) {
161+
Optional<String> tier = preferredAvailableTier(tierPreference, allocation.nodes());
162+
if (tier.isPresent()) {
163+
String tierName = tier.get();
164+
// The OpType doesn't actually matter here, because we have
165+
// selected only a single tier as our "preferred" tier
166+
if (allocationAllowed(OpType.AND, tierName, node)) {
167+
return allocation.decision(Decision.YES, NAME,
168+
"index has a preference for tiers [%s] and node has tier [%s]", tierPreference, tierName);
169+
} else {
170+
return allocation.decision(Decision.NO, NAME,
171+
"index has a preference for tiers [%s] and node does not meet the required [%s] tier", tierPreference, tierName);
172+
}
173+
} else {
174+
return allocation.decision(Decision.NO, NAME, "index has a preference for tiers [%s], " +
175+
"but no nodes for any of those tiers are available in the cluster", tierPreference);
176+
}
177+
}
178+
return null;
133179
}
134180

135181
private Decision shouldIndexFilter(IndexMetadata indexMd, DiscoveryNode node, RoutingAllocation allocation) {
@@ -186,6 +232,31 @@ private enum OpType {
186232
OR
187233
}
188234

235+
/**
236+
* Given a string of comma-separated prioritized tiers (highest priority
237+
* first) and an allocation, find the highest priority tier for which nodes
238+
* exist. If no nodes for any of the tiers are available, returns an empty
239+
* {@code Optional<String>}.
240+
*/
241+
static Optional<String> preferredAvailableTier(String prioritizedTiers, DiscoveryNodes nodes) {
242+
String[] tiers = Strings.tokenizeToStringArray(prioritizedTiers, ",");
243+
return Arrays.stream(tiers).filter(tier -> tierNodesPresent(tier, nodes)).findFirst();
244+
}
245+
246+
static boolean tierNodesPresent(String singleTier, DiscoveryNodes nodes) {
247+
assert singleTier.equals(DiscoveryNodeRole.DATA_ROLE.roleName()) || DataTier.validTierName(singleTier) :
248+
"tier " + singleTier + " is an invalid tier name";
249+
for (ObjectCursor<DiscoveryNode> node : nodes.getNodes().values()) {
250+
if (node.value.getRoles().stream()
251+
.map(DiscoveryNodeRole::roleName)
252+
.anyMatch(s -> s.equals(DiscoveryNodeRole.DATA_ROLE.roleName()) || s.equals(singleTier))) {
253+
return true;
254+
}
255+
}
256+
return false;
257+
}
258+
259+
189260
private static boolean allocationAllowed(OpType opType, String tierSetting, DiscoveryNode node) {
190261
String[] values = Strings.tokenizeToStringArray(tierSetting, ",");
191262
for (String value : values) {

0 commit comments

Comments
 (0)