Skip to content

Commit bf96bca

Browse files
tlrxtlrx
authored
Fix IndexFoldersDeletionListenerIT (#66329)
The tests testListenersInvokedWhenIndexIsRelocated and testListenersInvokedWhenIndexIsDangling fail on CI with errors like: java.lang.AssertionError: Expecting no shards deleted on node node_t4 at __randomizedtesting.SeedInfo.seed(...) at org.junit.Assert.fail(Assert.java:88) at org.junit.Assert.assertTrue(Assert.java:41) at org.elasticsearch.plugins.IndexFoldersDeletionListenerIT. testListenersInvokedWhenIndexIsDangling (IndexFoldersDeletionListenerIT.java:189) This commit tries to make those tests more stable by starting all nodes upfront, by waiting for no rebalancing/ relocations before checking the previous assertions and by extending a bit the time of some assertBusy() as on CI it can take more than 10 sec for pending deletes to be processed.
1 parent 0a1cec2 commit bf96bca

File tree

1 file changed

+73
-33
lines changed

1 file changed

+73
-33
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java

Lines changed: 73 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.elasticsearch.cluster.routing.ShardRouting;
2626
import org.elasticsearch.cluster.routing.ShardRoutingState;
2727
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
28+
import org.elasticsearch.common.Priority;
2829
import org.elasticsearch.common.settings.Settings;
2930
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
3031
import org.elasticsearch.env.Environment;
@@ -45,6 +46,7 @@
4546
import java.util.Locale;
4647
import java.util.Map;
4748
import java.util.Set;
49+
import java.util.concurrent.TimeUnit;
4850
import java.util.stream.Collectors;
4951

5052
import static org.elasticsearch.env.NodeEnvironment.INDICES_FOLDER;
@@ -55,7 +57,7 @@
5557
import static org.hamcrest.Matchers.hasSize;
5658
import static org.hamcrest.Matchers.notNullValue;
5759

58-
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
60+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
5961
public class IndexFoldersDeletionListenerIT extends ESIntegTestCase {
6062

6163
@Override
@@ -66,26 +68,37 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
6668
}
6769

6870
public void testListenersInvokedWhenIndexIsDeleted() throws Exception {
71+
final String masterNode = internalCluster().startMasterOnlyNode();
72+
internalCluster().startDataOnlyNodes(2);
73+
ensureStableCluster(2 + 1, masterNode);
74+
6975
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
70-
createIndex(indexName);
76+
createIndex(indexName, Settings.builder()
77+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2 * between(1, 2))
78+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1))
79+
.build());
7180

7281
final NumShards numShards = getNumShards(indexName);
73-
ensureClusterSizeConsistency(); // wait for a stable cluster
74-
ensureGreen(indexName); // wait for no relocation
75-
76-
final ClusterState clusterState = clusterService().state();
82+
assertFalse(client().admin().cluster().prepareHealth()
83+
.setIndices(indexName)
84+
.setWaitForGreenStatus()
85+
.setWaitForEvents(Priority.LANGUID)
86+
.setWaitForNoRelocatingShards(true)
87+
.setWaitForNoInitializingShards(true)
88+
.get()
89+
.isTimedOut());
90+
91+
final ClusterState clusterState = internalCluster().clusterService(masterNode).state();
7792
final Index index = clusterState.metadata().index(indexName).getIndex();
7893
final Map<String, List<ShardRouting>> shardsByNodes = shardRoutingsByNodes(clusterState, index);
7994
assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards));
8095

8196
for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
82-
final String nodeName = shardsByNode.getKey();
83-
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
84-
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
85-
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
97+
assertNoDeletions(shardsByNode.getKey());
8698
}
8799

88100
assertAcked(client().admin().indices().prepareDelete(indexName));
101+
assertPendingDeletesProcessed();
89102

90103
assertBusy(() -> {
91104
for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
@@ -105,30 +118,37 @@ public void testListenersInvokedWhenIndexIsDeleted() throws Exception {
105118
deletedShards.contains(shardId));
106119
}
107120
}
108-
});
121+
}, 30L, TimeUnit.SECONDS);
109122
}
110123

111124
public void testListenersInvokedWhenIndexIsRelocated() throws Exception {
112-
internalCluster().ensureAtLeastNumDataNodes(4);
125+
final String masterNode = internalCluster().startMasterOnlyNode();
126+
internalCluster().startDataOnlyNodes(4);
127+
ensureStableCluster(4 + 1, masterNode);
128+
113129
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
114130
createIndex(indexName, Settings.builder()
115-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(4, 10))
131+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4 * between(1, 2))
116132
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1))
117133
.build());
118134

119135
final NumShards numShards = getNumShards(indexName);
120-
ensureGreen(indexName);
121-
122-
final ClusterState clusterState = clusterService().state();
136+
assertFalse(client().admin().cluster().prepareHealth()
137+
.setIndices(indexName)
138+
.setWaitForGreenStatus()
139+
.setWaitForEvents(Priority.LANGUID)
140+
.setWaitForNoRelocatingShards(true)
141+
.setWaitForNoInitializingShards(true)
142+
.get()
143+
.isTimedOut());
144+
145+
final ClusterState clusterState = internalCluster().clusterService(masterNode).state();
123146
final Index index = clusterState.metadata().index(indexName).getIndex();
124147
final Map<String, List<ShardRouting>> shardsByNodes = shardRoutingsByNodes(clusterState, index);
125148
assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards));
126149

127150
for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
128-
final String nodeName = shardsByNode.getKey();
129-
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
130-
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
131-
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
151+
assertNoDeletions(shardsByNode.getKey());
132152
}
133153

134154
final List<String> excludedNodes = randomSubsetOf(2, shardsByNodes.keySet());
@@ -158,48 +178,58 @@ public void testListenersInvokedWhenIndexIsRelocated() throws Exception {
158178
deletedShards.contains(shardId));
159179
}
160180
} else {
161-
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
162-
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
181+
assertNoDeletions(nodeName);
163182
}
164183
}
165-
});
184+
}, 30L, TimeUnit.SECONDS);
166185
}
167186

168187
public void testListenersInvokedWhenIndexIsDangling() throws Exception {
169-
internalCluster().ensureAtLeastNumDataNodes(4);
188+
final String masterNode = internalCluster().startMasterOnlyNode();
189+
internalCluster().startDataOnlyNodes(4);
190+
ensureStableCluster(4 + 1, masterNode);
191+
170192
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
171193
createIndex(indexName, Settings.builder()
172-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(4, 10))
194+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4 * between(1, 2))
173195
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1))
174196
.build());
175197

176198
final NumShards numShards = getNumShards(indexName);
177-
ensureGreen(indexName);
178-
179-
final ClusterState clusterState = clusterService().state();
199+
assertFalse(client().admin().cluster().prepareHealth()
200+
.setIndices(indexName)
201+
.setWaitForGreenStatus()
202+
.setWaitForEvents(Priority.LANGUID)
203+
.setWaitForNoRelocatingShards(true)
204+
.setWaitForNoInitializingShards(true)
205+
.get()
206+
.isTimedOut());
207+
208+
final ClusterState clusterState = internalCluster().clusterService(masterNode).state();
180209
final Index index = clusterState.metadata().index(indexName).getIndex();
181210
final Map<String, List<ShardRouting>> shardsByNodes = shardRoutingsByNodes(clusterState, index);
182211
assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards));
183212

184213
for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
185-
final String nodeName = shardsByNode.getKey();
186-
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
187-
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
188-
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
214+
assertNoDeletions(shardsByNode.getKey());
189215
}
190216

191217
final String stoppedNode = randomFrom(shardsByNodes.keySet());
192218
final Settings stoppedNodeDataPathSettings = internalCluster().dataPathSettings(stoppedNode);
193219
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(stoppedNode));
220+
ensureStableCluster(3 + 1, masterNode);
194221

195222
assertAcked(client().admin().indices().prepareDelete(indexName));
196223

197224
final String restartedNode = internalCluster().startNode(stoppedNodeDataPathSettings);
225+
ensureStableCluster(4 + 1, masterNode);
226+
assertPendingDeletesProcessed();
227+
198228
assertBusy(() -> {
199229
final IndexFoldersDeletionListenerPlugin plugin = plugin(restartedNode);
200230
assertTrue("Listener should have been notified of deletion of index " + index + " on node " + restartedNode,
201231
plugin.deletedIndices.contains(index));
202-
});
232+
}, 30L, TimeUnit.SECONDS);
203233
}
204234

205235
public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception {
@@ -228,6 +258,7 @@ public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception {
228258

229259
logger.debug("--> stopping data node [{}], the data left on disk will be injected as left-overs in a newer data node", dataNode);
230260
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode));
261+
ensureStableCluster(1, masterNode);
231262

232263
logger.debug("--> deleting leftover indices");
233264
assertAcked(client().admin().indices().prepareDelete("index-*"));
@@ -264,6 +295,7 @@ public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception {
264295
dataPaths.stream().map(p -> p.toAbsolutePath().toString()).collect(Collectors.toList()))
265296
.putNull(Environment.PATH_SHARED_DATA_SETTING.getKey())
266297
.build());
298+
ensureStableCluster(1 + 1, masterNode);
267299

268300
final IndexFoldersDeletionListenerPlugin plugin = plugin(dataNode);
269301
assertTrue("Expecting no shards deleted on node " + dataNode, plugin.deletedShards.isEmpty());
@@ -328,4 +360,12 @@ private static void assertPendingDeletesProcessed() throws Exception {
328360
services.forEach(indicesService -> assertFalse(indicesService.hasUncompletedPendingDeletes()));
329361
});
330362
}
363+
364+
private static void assertNoDeletions(String nodeName) {
365+
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
366+
assertTrue("Expecting no indices deleted on node [" + nodeName + "] but got: " + plugin.deletedIndices,
367+
plugin.deletedIndices.isEmpty());
368+
assertTrue("Expecting no shards deleted on node [" + nodeName + "] but got: " + plugin.deletedShards,
369+
plugin.deletedShards.isEmpty());
370+
}
331371
}

0 commit comments

Comments
 (0)