Skip to content

Commit ee93a05

Browse files
henningandersenhenningandersen
authored
Autoscale frozen tier into existence (#73435) (#74412)
This commit adds two related changes: * ILM WaitForDataTierStep * Autoscaling frozen_existence decider The first part ensures that we wait mounting an index until a node that can hold the index is available, avoiding a failed restore and red cluster state. This is in particular important for the frozen phase, but is done generically in the searchable snapshot action. The second part triggers on indices in the ILM frozen phase to scale the tier into existence by requiring a minimal amount of memory and storage. Closes #72771
1 parent facfe5c commit ee93a05

File tree

41 files changed

+841
-137
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+841
-137
lines changed

‎docs/reference/autoscaling/autoscaling-deciders.asciidoc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ Estimates required storage capacity as a percentage of the total data set of
2020
partially mounted indices.
2121
Available for policies governing frozen data nodes.
2222

23+
<<autoscaling-frozen-existence-decider,Frozen existence decider>>::
24+
Estimates a minimum require frozen memory and storage capacity when any index is
25+
in the frozen <<index-lifecycle-management,ILM>> phase.
26+
2327
<<autoscaling-machine-learning-decider,Machine learning decider>>::
2428
Estimates required memory capacity based on machine learning jobs.
2529
Available for policies governing machine learning nodes.
@@ -31,5 +35,6 @@ include::deciders/reactive-storage-decider.asciidoc[]
3135
include::deciders/proactive-storage-decider.asciidoc[]
3236
include::deciders/frozen-shards-decider.asciidoc[]
3337
include::deciders/frozen-storage-decider.asciidoc[]
38+
include::deciders/frozen-existence-decider.asciidoc[]
3439
include::deciders/machine-learning-decider.asciidoc[]
3540
include::deciders/fixed-decider.asciidoc[]

‎docs/reference/autoscaling/deciders/frozen-existence-decider.asciidoc

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[role="xpack"]
2+
[[autoscaling-frozen-existence-decider]]
3+
=== Frozen existence decider
4+
5+
The frozen existence decider (`frozen_existence`) ensures that once the first
6+
index enters the frozen ILM phase, the frozen tier is scaled into existence.
7+
8+
The frozen existence decider is enabled for all policies governing frozen data
9+
nodes and has no configuration options.

‎x-pack/plugin/autoscaling/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ dependencies {
1616
testImplementation(testArtifact(project(xpackModule('core'))))
1717
testImplementation project(path: xpackModule('data-streams'))
1818
testImplementation project(path: xpackModule('searchable-snapshots'))
19+
testImplementation project(path: xpackModule('ilm'))
1920
}
2021

2122
addQaCheckDependencies()

‎x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
1111
import org.elasticsearch.cluster.node.DiscoveryNode;
12+
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
1213
import org.elasticsearch.common.Strings;
1314
import org.elasticsearch.common.settings.Settings;
1415
import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -29,7 +30,6 @@
2930
import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction;
3031
import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
3132
import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
32-
import org.junit.Before;
3333

3434
import java.util.Collection;
3535
import java.util.Locale;
@@ -84,18 +84,19 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
8484
Settings.Builder builder = Settings.builder()
8585
.put(super.nodeSettings(nodeOrdinal, otherSettings))
8686
.put(SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
87-
if (DiscoveryNode.canContainData(otherSettings)) {
87+
if (DiscoveryNode.hasRole(otherSettings, DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE)) {
8888
builder.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(10, ByteSizeUnit.MB));
8989
}
9090
return builder.build();
9191
}
9292

93-
@Before
94-
public void setupPolicyAndMountedIndex() throws Exception {
93+
protected void setupRepoAndPolicy() {
9594
createRepository(fsRepoName, "fs");
9695
putAutoscalingPolicy();
97-
assertAcked(prepareCreate(indexName, Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true)));
96+
}
9897

98+
protected void createAndMountIndex() throws InterruptedException, java.util.concurrent.ExecutionException {
99+
assertAcked(prepareCreate(indexName, Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true)));
99100
indexRandom(
100101
randomBoolean(),
101102
IntStream.range(0, 10).mapToObj(i -> client().prepareIndex(indexName, "_doc").setSource()).collect(Collectors.toList())

‎x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.autoscaling.existence;
9+
10+
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
11+
import org.elasticsearch.cluster.health.ClusterHealthStatus;
12+
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
13+
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.core.TimeValue;
15+
import org.elasticsearch.plugins.Plugin;
16+
import org.elasticsearch.snapshots.SnapshotInfo;
17+
import org.elasticsearch.test.ESIntegTestCase;
18+
import org.elasticsearch.test.NodeRoles;
19+
import org.elasticsearch.xpack.autoscaling.AbstractFrozenAutoscalingIntegTestCase;
20+
import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCapacity;
21+
import org.elasticsearch.xpack.core.ilm.ExplainLifecycleRequest;
22+
import org.elasticsearch.xpack.core.ilm.ExplainLifecycleResponse;
23+
import org.elasticsearch.xpack.core.ilm.IndexLifecycleExplainResponse;
24+
import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
25+
import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
26+
import org.elasticsearch.xpack.core.ilm.Phase;
27+
import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction;
28+
import org.elasticsearch.xpack.core.ilm.WaitForDataTierStep;
29+
import org.elasticsearch.xpack.core.ilm.action.ExplainLifecycleAction;
30+
import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
31+
32+
import java.util.Collection;
33+
import java.util.Collections;
34+
35+
import static java.util.Collections.singletonMap;
36+
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
37+
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
38+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
39+
import static org.hamcrest.Matchers.anyOf;
40+
import static org.hamcrest.Matchers.arrayContaining;
41+
import static org.hamcrest.Matchers.equalTo;
42+
import static org.hamcrest.Matchers.not;
43+
44+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
45+
public class FrozenExistenceDeciderIT extends AbstractFrozenAutoscalingIntegTestCase {
46+
47+
private static final String INDEX_NAME = "index";
48+
private static final String PARTIAL_INDEX_NAME = "partial-index";
49+
50+
@Override
51+
protected String deciderName() {
52+
return FrozenExistenceDeciderService.NAME;
53+
}
54+
55+
@Override
56+
protected Settings.Builder addDeciderSettings(Settings.Builder builder) {
57+
return builder;
58+
}
59+
60+
@Override
61+
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
62+
Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings));
63+
settings.put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s");
64+
settings.put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false);
65+
settings.put(LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
66+
return settings.build();
67+
}
68+
69+
@Override
70+
protected Collection<Class<? extends Plugin>> nodePlugins() {
71+
return org.elasticsearch.core.List.of(LocalStateAutoscalingAndSearchableSnapshotsAndIndexLifecycle.class);
72+
}
73+
74+
public void testZeroToOne() throws Exception {
75+
internalCluster().startMasterOnlyNode();
76+
setupRepoAndPolicy();
77+
logger.info("starting 2 content data nodes");
78+
internalCluster().startNode(NodeRoles.onlyRole(DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE));
79+
internalCluster().startNode(NodeRoles.onlyRole(DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE));
80+
// create an ignored snapshot to initialize the latest-N file.
81+
final SnapshotInfo snapshotInfo = createFullSnapshot(fsRepoName, snapshotName);
82+
83+
Phase hotPhase = new Phase("hot", TimeValue.ZERO, Collections.emptyMap());
84+
Phase frozenPhase = new Phase(
85+
"frozen",
86+
TimeValue.ZERO,
87+
singletonMap(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(fsRepoName, randomBoolean()))
88+
);
89+
LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(
90+
"policy",
91+
org.elasticsearch.core.Map.of("hot", hotPhase, "frozen", frozenPhase)
92+
);
93+
PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
94+
assertAcked(client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get());
95+
96+
Settings settings = Settings.builder()
97+
.put(indexSettings())
98+
.put(SETTING_NUMBER_OF_SHARDS, 1)
99+
.put(SETTING_NUMBER_OF_REPLICAS, 1)
100+
.put(LifecycleSettings.LIFECYCLE_NAME, "policy")
101+
.build();
102+
CreateIndexResponse res = client().admin().indices().prepareCreate(INDEX_NAME).setSettings(settings).get();
103+
assertTrue(res.isAcknowledged());
104+
logger.info("created index");
105+
106+
assertBusy(() -> { assertMinimumCapacity(capacity().results().get("frozen").requiredCapacity().total()); });
107+
assertMinimumCapacity(capacity().results().get("frozen").requiredCapacity().node());
108+
109+
assertThat(
110+
client().admin().cluster().prepareHealth().get().getStatus(),
111+
anyOf(equalTo(ClusterHealthStatus.YELLOW), equalTo(ClusterHealthStatus.GREEN))
112+
);
113+
114+
assertBusy(() -> {
115+
ExplainLifecycleResponse response = client().execute(
116+
ExplainLifecycleAction.INSTANCE,
117+
new ExplainLifecycleRequest().indices(INDEX_NAME)
118+
).actionGet();
119+
IndexLifecycleExplainResponse indexResponse = response.getIndexResponses().get(INDEX_NAME);
120+
assertNotNull(indexResponse);
121+
assertThat(indexResponse.getStep(), equalTo(WaitForDataTierStep.NAME));
122+
});
123+
124+
// verify that SearchableSnapshotAction uses WaitForDataTierStep and that it waits.
125+
assertThat(indices(), not(arrayContaining(PARTIAL_INDEX_NAME)));
126+
127+
logger.info("starting dedicated frozen node");
128+
internalCluster().startNode(NodeRoles.onlyRole(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE));
129+
130+
assertBusy(() -> {
131+
String[] indices = indices();
132+
assertThat(indices, arrayContaining(PARTIAL_INDEX_NAME));
133+
assertThat(indices, not(arrayContaining(INDEX_NAME)));
134+
});
135+
ensureGreen();
136+
}
137+
138+
private String[] indices() {
139+
return client().admin().indices().prepareGetIndex().addIndices("index").get().indices();
140+
}
141+
142+
private void assertMinimumCapacity(AutoscalingCapacity.AutoscalingResources resources) {
143+
assertThat(resources.memory(), equalTo(FrozenExistenceDeciderService.MINIMUM_FROZEN_MEMORY));
144+
assertThat(resources.storage(), equalTo(FrozenExistenceDeciderService.MINIMUM_FROZEN_STORAGE));
145+
}
146+
}

‎x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/LocalStateAutoscalingAndSearchableSnapshotsAndIndexLifecycle.java

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.autoscaling.existence;
9+
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.license.XPackLicenseState;
12+
import org.elasticsearch.xpack.autoscaling.shards.LocalStateAutoscalingAndSearchableSnapshots;
13+
import org.elasticsearch.xpack.ilm.IndexLifecycle;
14+
15+
/**
16+
* We need a local state plugin including both searchable snapshots and ilm in order to verify the frozen 0-1 case works through ilm.
17+
* The local state plugin is necessary to avoid touching the "static SetOnce" licenseState field in XPackPlugin.
18+
*/
19+
public class LocalStateAutoscalingAndSearchableSnapshotsAndIndexLifecycle extends LocalStateAutoscalingAndSearchableSnapshots {
20+
21+
public LocalStateAutoscalingAndSearchableSnapshotsAndIndexLifecycle(final Settings settings) {
22+
super(settings);
23+
plugins.add(new IndexLifecycle(settings) {
24+
@Override
25+
protected XPackLicenseState getLicenseState() {
26+
return LocalStateAutoscalingAndSearchableSnapshotsAndIndexLifecycle.this.getLicenseState();
27+
}
28+
});
29+
}
30+
}

‎x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderIT.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ protected int numberOfShards() {
1818
return 1;
1919
}
2020

21-
public void testScale() {
21+
public void testScale() throws Exception {
22+
setupRepoAndPolicy();
23+
createAndMountIndex();
24+
2225
assertThat(
2326
capacity().results().get("frozen").requiredCapacity().total().memory(),
2427
equalTo(FrozenShardsDeciderService.DEFAULT_MEMORY_PER_SHARD)

‎x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderIT.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@
2020

2121
public class FrozenStorageDeciderIT extends AbstractFrozenAutoscalingIntegTestCase {
2222

23-
public void testScale() {
23+
public void testScale() throws Exception {
24+
setupRepoAndPolicy();
25+
createAndMountIndex();
26+
2427
IndicesStatsResponse statsResponse = client().admin()
2528
.indices()
2629
.stats(new IndicesStatsRequest().indices(restoredIndexName))

‎x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService;
5050
import org.elasticsearch.xpack.autoscaling.capacity.FixedAutoscalingDeciderService;
5151
import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryInfoService;
52+
import org.elasticsearch.xpack.autoscaling.existence.FrozenExistenceDeciderService;
5253
import org.elasticsearch.xpack.autoscaling.rest.RestDeleteAutoscalingPolicyHandler;
5354
import org.elasticsearch.xpack.autoscaling.rest.RestGetAutoscalingCapacityHandler;
5455
import org.elasticsearch.xpack.autoscaling.rest.RestGetAutoscalingPolicyHandler;
@@ -176,6 +177,11 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
176177
AutoscalingDeciderResult.Reason.class,
177178
FrozenStorageDeciderService.NAME,
178179
FrozenStorageDeciderService.FrozenReason::new
180+
),
181+
new NamedWriteableRegistry.Entry(
182+
AutoscalingDeciderResult.Reason.class,
183+
FrozenExistenceDeciderService.NAME,
184+
FrozenExistenceDeciderService.FrozenExistenceReason::new
179185
)
180186
);
181187
}
@@ -208,7 +214,8 @@ public Collection<AutoscalingDeciderService> deciders() {
208214
allocationDeciders.get()
209215
),
210216
new FrozenShardsDeciderService(),
211-
new FrozenStorageDeciderService()
217+
new FrozenStorageDeciderService(),
218+
new FrozenExistenceDeciderService()
212219
);
213220
}
214221

0 commit comments

Comments
 (0)