Skip to content

Commit 28aaf9f

Browse files
gwbrowngwbrown
committed
Ensure ILM policies run safely on leader indices (elastic#38140)
Adds a Step to the Shrink and Delete actions which prevents those actions from running on a leader index - all follower indices must first unfollow the leader index before these actions can run. This prevents the loss of history before follower indices are ready, which might otherwise result in the loss of data.
1 parent 7df7c7f commit 28aaf9f

File tree

8 files changed

+565
-82
lines changed

8 files changed

+565
-82
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteAction.java

+10-3
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
1616

1717
import java.io.IOException;
18-
import java.util.Collections;
18+
import java.util.Arrays;
1919
import java.util.List;
2020

2121
/**
@@ -59,13 +59,20 @@ public boolean isSafeAction() {
5959

6060
@Override
6161
public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
62+
Step.StepKey waitForNoFollowerStepKey = new Step.StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
6263
Step.StepKey deleteStepKey = new Step.StepKey(phase, NAME, DeleteStep.NAME);
63-
return Collections.singletonList(new DeleteStep(deleteStepKey, nextStepKey, client));
64+
65+
WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, deleteStepKey, client);
66+
DeleteStep deleteStep = new DeleteStep(deleteStepKey, nextStepKey, client);
67+
return Arrays.asList(waitForNoFollowersStep, deleteStep);
6468
}
6569

6670
@Override
6771
public List<StepKey> toStepKeys(String phase) {
68-
return Collections.singletonList(new Step.StepKey(phase, NAME, DeleteStep.NAME));
72+
Step.StepKey waitForNoFollowerStepKey = new Step.StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
73+
Step.StepKey deleteStepKey = new Step.StepKey(phase, NAME, DeleteStep.NAME);
74+
75+
return Arrays.asList(waitForNoFollowerStepKey, deleteStepKey);
6976
}
7077

7178
@Override

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java

+8-5
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
8686
Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build();
8787

8888
StepKey branchingKey = new StepKey(phase, NAME, BranchingStep.NAME);
89+
StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
8990
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
9091
StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
9192
StepKey allocationRoutedKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
@@ -95,8 +96,9 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
9596
StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
9697
StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
9798

98-
BranchingStep conditionalSkipShrinkStep = new BranchingStep(branchingKey, readOnlyKey, nextStepKey,
99+
BranchingStep conditionalSkipShrinkStep = new BranchingStep(branchingKey, waitForNoFollowerStepKey, nextStepKey,
99100
(index, clusterState) -> clusterState.getMetaData().index(index).getNumberOfShards() == numberOfShards);
101+
WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, readOnlyKey, client);
100102
UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, setSingleNodeKey, client, readOnlySettings);
101103
SetSingleNodeAllocateStep setSingleNodeStep = new SetSingleNodeAllocateStep(setSingleNodeKey, allocationRoutedKey, client);
102104
CheckShrinkReadyStep checkShrinkReadyStep = new CheckShrinkReadyStep(allocationRoutedKey, shrinkKey);
@@ -105,13 +107,14 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
105107
CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep(copyMetadataKey, aliasKey, SHRUNKEN_INDEX_PREFIX);
106108
ShrinkSetAliasStep aliasSwapAndDelete = new ShrinkSetAliasStep(aliasKey, isShrunkIndexKey, client, SHRUNKEN_INDEX_PREFIX);
107109
ShrunkenIndexCheckStep waitOnShrinkTakeover = new ShrunkenIndexCheckStep(isShrunkIndexKey, nextStepKey, SHRUNKEN_INDEX_PREFIX);
108-
return Arrays.asList(conditionalSkipShrinkStep, readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated,
109-
copyMetadata, aliasSwapAndDelete, waitOnShrinkTakeover);
110+
return Arrays.asList(conditionalSkipShrinkStep, waitForNoFollowersStep, readOnlyStep, setSingleNodeStep, checkShrinkReadyStep,
111+
shrink, allocated, copyMetadata, aliasSwapAndDelete, waitOnShrinkTakeover);
110112
}
111113

112114
@Override
113115
public List<StepKey> toStepKeys(String phase) {
114116
StepKey conditionalSkipKey = new StepKey(phase, NAME, BranchingStep.NAME);
117+
StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
115118
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
116119
StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
117120
StepKey checkShrinkReadyKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
@@ -120,8 +123,8 @@ public List<StepKey> toStepKeys(String phase) {
120123
StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
121124
StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
122125
StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
123-
return Arrays.asList(conditionalSkipKey, readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey,
124-
copyMetadataKey, aliasKey, isShrunkIndexKey);
126+
return Arrays.asList(conditionalSkipKey, waitForNoFollowerStepKey, readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey,
127+
enoughShardsKey, copyMetadataKey, aliasKey, isShrunkIndexKey);
125128
}
126129

127130
@Override

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForNoFollowersStep.java

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.core.indexlifecycle;
8+
9+
import org.apache.logging.log4j.LogManager;
10+
import org.apache.logging.log4j.Logger;
11+
import org.elasticsearch.action.ActionListener;
12+
import org.elasticsearch.action.admin.indices.stats.IndexStats;
13+
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
14+
import org.elasticsearch.action.admin.indices.stats.ShardStats;
15+
import org.elasticsearch.client.Client;
16+
import org.elasticsearch.cluster.metadata.IndexMetaData;
17+
import org.elasticsearch.common.ParseField;
18+
import org.elasticsearch.common.xcontent.ToXContentObject;
19+
import org.elasticsearch.common.xcontent.XContentBuilder;
20+
21+
import java.io.IOException;
22+
import java.util.Arrays;
23+
import java.util.Objects;
24+
25+
/**
26+
* A step that waits until the index it's used on is no longer a leader index.
27+
* This is necessary as there are some actions which are not safe to perform on
28+
* a leader index, such as those which delete the index, including Shrink and
29+
* Delete.
30+
*/
31+
public class WaitForNoFollowersStep extends AsyncWaitStep {
32+
33+
private static final Logger logger = LogManager.getLogger(WaitForNoFollowersStep.class);
34+
35+
static final String NAME = "wait-for-shard-history-leases";
36+
static final String CCR_LEASE_KEY = "ccr";
37+
38+
WaitForNoFollowersStep(StepKey key, StepKey nextStepKey, Client client) {
39+
super(key, nextStepKey, client);
40+
}
41+
42+
@Override
43+
public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) {
44+
IndicesStatsRequest request = new IndicesStatsRequest();
45+
request.clear();
46+
String indexName = indexMetaData.getIndex().getName();
47+
request.indices(indexName);
48+
getClient().admin().indices().stats(request, ActionListener.wrap((response) -> {
49+
IndexStats indexStats = response.getIndex(indexName);
50+
if (indexStats == null) {
51+
// Index was probably deleted
52+
logger.debug("got null shard stats for index {}, proceeding on the assumption it has been deleted",
53+
indexMetaData.getIndex());
54+
listener.onResponse(true, null);
55+
return;
56+
}
57+
58+
boolean isCurrentlyLeaderIndex = Arrays.stream(indexStats.getShards())
59+
.map(ShardStats::getRetentionLeaseStats)
60+
.flatMap(retentionLeaseStats -> retentionLeaseStats.retentionLeases().leases().stream())
61+
.anyMatch(lease -> CCR_LEASE_KEY.equals(lease.source()));
62+
63+
if (isCurrentlyLeaderIndex) {
64+
listener.onResponse(false, new Info());
65+
} else {
66+
listener.onResponse(true, null);
67+
}
68+
}, listener::onFailure));
69+
}
70+
71+
static final class Info implements ToXContentObject {
72+
73+
static final ParseField MESSAGE_FIELD = new ParseField("message");
74+
75+
private static final String message = "this index is a leader index; waiting for all following indices to cease " +
76+
"following before proceeding";
77+
78+
Info() { }
79+
80+
String getMessage() {
81+
return message;
82+
}
83+
84+
@Override
85+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
86+
builder.startObject();
87+
builder.field(MESSAGE_FIELD.getPreferredName(), message);
88+
builder.endObject();
89+
return builder;
90+
}
91+
92+
@Override
93+
public boolean equals(Object o) {
94+
if (this == o) {
95+
return true;
96+
}
97+
if (o == null || getClass() != o.getClass()) {
98+
return false;
99+
}
100+
return true;
101+
}
102+
103+
@Override
104+
public int hashCode() {
105+
return Objects.hash(getMessage());
106+
}
107+
}
108+
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteActionTests.java

+8-4
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,14 @@ public void testToSteps() {
3636
randomAlphaOfLengthBetween(1, 10));
3737
List<Step> steps = action.toSteps(null, phase, nextStepKey);
3838
assertNotNull(steps);
39-
assertEquals(1, steps.size());
40-
StepKey expectedFirstStepKey = new StepKey(phase, DeleteAction.NAME, DeleteStep.NAME);
41-
DeleteStep firstStep = (DeleteStep) steps.get(0);
39+
assertEquals(2, steps.size());
40+
StepKey expectedFirstStepKey = new StepKey(phase, DeleteAction.NAME, WaitForNoFollowersStep.NAME);
41+
StepKey expectedSecondStepKey = new StepKey(phase, DeleteAction.NAME, DeleteStep.NAME);
42+
WaitForNoFollowersStep firstStep = (WaitForNoFollowersStep) steps.get(0);
43+
DeleteStep secondStep = (DeleteStep) steps.get(1);
4244
assertEquals(expectedFirstStepKey, firstStep.getKey());
43-
assertEquals(nextStepKey, firstStep.getNextStepKey());
45+
assertEquals(expectedSecondStepKey, firstStep.getNextStepKey());
46+
assertEquals(expectedSecondStepKey, secondStep.getKey());
47+
assertEquals(nextStepKey, secondStep.getNextStepKey());
4448
}
4549
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java

+29-24
Original file line numberDiff line numberDiff line change
@@ -126,60 +126,65 @@ public void testToSteps() {
126126
StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
127127
randomAlphaOfLengthBetween(1, 10));
128128
List<Step> steps = action.toSteps(null, phase, nextStepKey);
129-
assertThat(steps.size(), equalTo(9));
129+
assertThat(steps.size(), equalTo(10));
130130
StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, BranchingStep.NAME);
131-
StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME);
132-
StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME);
133-
StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME);
134-
StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME);
135-
StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME);
136-
StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME);
137-
StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME);
138-
StepKey expectedNinthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME);
131+
StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, WaitForNoFollowersStep.NAME);
132+
StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME);
133+
StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME);
134+
StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME);
135+
StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME);
136+
StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME);
137+
StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME);
138+
StepKey expectedNinthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME);
139+
StepKey expectedTenthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME);
139140

140141
assertTrue(steps.get(0) instanceof BranchingStep);
141142
assertThat(steps.get(0).getKey(), equalTo(expectedFirstKey));
142143
expectThrows(IllegalStateException.class, () -> steps.get(0).getNextStepKey());
143144
assertThat(((BranchingStep) steps.get(0)).getNextStepKeyOnFalse(), equalTo(expectedSecondKey));
144145
assertThat(((BranchingStep) steps.get(0)).getNextStepKeyOnTrue(), equalTo(nextStepKey));
145146

146-
assertTrue(steps.get(1) instanceof UpdateSettingsStep);
147+
assertTrue(steps.get(1) instanceof WaitForNoFollowersStep);
147148
assertThat(steps.get(1).getKey(), equalTo(expectedSecondKey));
148149
assertThat(steps.get(1).getNextStepKey(), equalTo(expectedThirdKey));
149-
assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(1)).getSettings()));
150150

151-
assertTrue(steps.get(2) instanceof SetSingleNodeAllocateStep);
151+
assertTrue(steps.get(2) instanceof UpdateSettingsStep);
152152
assertThat(steps.get(2).getKey(), equalTo(expectedThirdKey));
153153
assertThat(steps.get(2).getNextStepKey(), equalTo(expectedFourthKey));
154+
assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(2)).getSettings()));
154155

155-
assertTrue(steps.get(3) instanceof CheckShrinkReadyStep);
156+
assertTrue(steps.get(3) instanceof SetSingleNodeAllocateStep);
156157
assertThat(steps.get(3).getKey(), equalTo(expectedFourthKey));
157158
assertThat(steps.get(3).getNextStepKey(), equalTo(expectedFifthKey));
158159

159-
assertTrue(steps.get(4) instanceof ShrinkStep);
160+
assertTrue(steps.get(4) instanceof CheckShrinkReadyStep);
160161
assertThat(steps.get(4).getKey(), equalTo(expectedFifthKey));
161162
assertThat(steps.get(4).getNextStepKey(), equalTo(expectedSixthKey));
162-
assertThat(((ShrinkStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
163163

164-
assertTrue(steps.get(5) instanceof ShrunkShardsAllocatedStep);
164+
assertTrue(steps.get(5) instanceof ShrinkStep);
165165
assertThat(steps.get(5).getKey(), equalTo(expectedSixthKey));
166166
assertThat(steps.get(5).getNextStepKey(), equalTo(expectedSeventhKey));
167-
assertThat(((ShrunkShardsAllocatedStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
167+
assertThat(((ShrinkStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
168168

169-
assertTrue(steps.get(6) instanceof CopyExecutionStateStep);
169+
assertTrue(steps.get(6) instanceof ShrunkShardsAllocatedStep);
170170
assertThat(steps.get(6).getKey(), equalTo(expectedSeventhKey));
171171
assertThat(steps.get(6).getNextStepKey(), equalTo(expectedEighthKey));
172-
assertThat(((CopyExecutionStateStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
172+
assertThat(((ShrunkShardsAllocatedStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
173173

174-
assertTrue(steps.get(7) instanceof ShrinkSetAliasStep);
174+
assertTrue(steps.get(7) instanceof CopyExecutionStateStep);
175175
assertThat(steps.get(7).getKey(), equalTo(expectedEighthKey));
176176
assertThat(steps.get(7).getNextStepKey(), equalTo(expectedNinthKey));
177-
assertThat(((ShrinkSetAliasStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
177+
assertThat(((CopyExecutionStateStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
178178

179-
assertTrue(steps.get(8) instanceof ShrunkenIndexCheckStep);
179+
assertTrue(steps.get(8) instanceof ShrinkSetAliasStep);
180180
assertThat(steps.get(8).getKey(), equalTo(expectedNinthKey));
181-
assertThat(steps.get(8).getNextStepKey(), equalTo(nextStepKey));
182-
assertThat(((ShrunkenIndexCheckStep) steps.get(8)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
181+
assertThat(steps.get(8).getNextStepKey(), equalTo(expectedTenthKey));
182+
assertThat(((ShrinkSetAliasStep) steps.get(8)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
183+
184+
assertTrue(steps.get(9) instanceof ShrunkenIndexCheckStep);
185+
assertThat(steps.get(9).getKey(), equalTo(expectedTenthKey));
186+
assertThat(steps.get(9).getNextStepKey(), equalTo(nextStepKey));
187+
assertThat(((ShrunkenIndexCheckStep) steps.get(9)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
183188
}
184189

185190
@Override

0 commit comments

Comments
 (0)