Skip to content

Commit 4886628

Browse files
tlrxtlrx
authored
Fix IndexFoldersDeletionListenerIT.testListenersInvokedWhenIndexIsRelocated (#66329) (#66511)
The tests testListenersInvokedWhenIndexIsRelocated and testListenersInvokedWhenIndexIsDangling fail on CI with errors like: java.lang.AssertionError: Expecting no shards deleted on node node_t4 at __randomizedtesting.SeedInfo.seed(...) at org.junit.Assert.fail(Assert.java:88) at org.junit.Assert.assertTrue(Assert.java:41) at org.elasticsearch.plugins.IndexFoldersDeletionListenerIT. testListenersInvokedWhenIndexIsDangling (IndexFoldersDeletionListenerIT.java:189) This commit tries to make those tests more stable by starting all nodes upfront, by waiting for no rebalancing/ relocations before checking the previous assertions and by extending a bit the time of some assertBusy() as on CI it can take more than 10 sec for pending deletes to be processed.
1 parent 8513510 commit 4886628

File tree

2 files changed

+83
-40
lines changed

2 files changed

+83
-40
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java

Lines changed: 73 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.elasticsearch.cluster.routing.ShardRouting;
2626
import org.elasticsearch.cluster.routing.ShardRoutingState;
2727
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
28+
import org.elasticsearch.common.Priority;
2829
import org.elasticsearch.common.settings.Settings;
2930
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
3031
import org.elasticsearch.env.Environment;
@@ -45,6 +46,7 @@
4546
import java.util.Locale;
4647
import java.util.Map;
4748
import java.util.Set;
49+
import java.util.concurrent.TimeUnit;
4850
import java.util.stream.Collectors;
4951

5052
import static java.util.Collections.singletonList;
@@ -56,7 +58,7 @@
5658
import static org.hamcrest.Matchers.hasSize;
5759
import static org.hamcrest.Matchers.notNullValue;
5860

59-
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
61+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
6062
public class IndexFoldersDeletionListenerIT extends ESIntegTestCase {
6163

6264
@Override
@@ -67,26 +69,37 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
6769
}
6870

6971
public void testListenersInvokedWhenIndexIsDeleted() throws Exception {
72+
final String masterNode = internalCluster().startMasterOnlyNode();
73+
internalCluster().startDataOnlyNodes(2);
74+
ensureStableCluster(2 + 1, masterNode);
75+
7076
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
71-
createIndex(indexName);
77+
createIndex(indexName, Settings.builder()
78+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2 * between(1, 2))
79+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1))
80+
.build());
7281

7382
final NumShards numShards = getNumShards(indexName);
74-
ensureClusterSizeConsistency(); // wait for a stable cluster
75-
ensureGreen(indexName); // wait for no relocation
76-
77-
final ClusterState clusterState = clusterService().state();
83+
assertFalse(client().admin().cluster().prepareHealth()
84+
.setIndices(indexName)
85+
.setWaitForGreenStatus()
86+
.setWaitForEvents(Priority.LANGUID)
87+
.setWaitForNoRelocatingShards(true)
88+
.setWaitForNoInitializingShards(true)
89+
.get()
90+
.isTimedOut());
91+
92+
final ClusterState clusterState = internalCluster().clusterService(masterNode).state();
7893
final Index index = clusterState.metadata().index(indexName).getIndex();
7994
final Map<String, List<ShardRouting>> shardsByNodes = shardRoutingsByNodes(clusterState, index);
8095
assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards));
8196

8297
for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
83-
final String nodeName = shardsByNode.getKey();
84-
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
85-
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
86-
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
98+
assertNoDeletions(shardsByNode.getKey());
8799
}
88100

89101
assertAcked(client().admin().indices().prepareDelete(indexName));
102+
assertPendingDeletesProcessed();
90103

91104
assertBusy(() -> {
92105
for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
@@ -106,30 +119,37 @@ public void testListenersInvokedWhenIndexIsDeleted() throws Exception {
106119
deletedShards.contains(shardId));
107120
}
108121
}
109-
});
122+
}, 30L, TimeUnit.SECONDS);
110123
}
111124

112125
public void testListenersInvokedWhenIndexIsRelocated() throws Exception {
113-
internalCluster().ensureAtLeastNumDataNodes(4);
126+
final String masterNode = internalCluster().startMasterOnlyNode();
127+
internalCluster().startDataOnlyNodes(4);
128+
ensureStableCluster(4 + 1, masterNode);
129+
114130
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
115131
createIndex(indexName, Settings.builder()
116-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(4, 10))
132+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4 * between(1, 2))
117133
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1))
118134
.build());
119135

120136
final NumShards numShards = getNumShards(indexName);
121-
ensureGreen(indexName);
122-
123-
final ClusterState clusterState = clusterService().state();
137+
assertFalse(client().admin().cluster().prepareHealth()
138+
.setIndices(indexName)
139+
.setWaitForGreenStatus()
140+
.setWaitForEvents(Priority.LANGUID)
141+
.setWaitForNoRelocatingShards(true)
142+
.setWaitForNoInitializingShards(true)
143+
.get()
144+
.isTimedOut());
145+
146+
final ClusterState clusterState = internalCluster().clusterService(masterNode).state();
124147
final Index index = clusterState.metadata().index(indexName).getIndex();
125148
final Map<String, List<ShardRouting>> shardsByNodes = shardRoutingsByNodes(clusterState, index);
126149
assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards));
127150

128151
for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
129-
final String nodeName = shardsByNode.getKey();
130-
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
131-
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
132-
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
152+
assertNoDeletions(shardsByNode.getKey());
133153
}
134154

135155
final List<String> excludedNodes = randomSubsetOf(2, shardsByNodes.keySet());
@@ -159,48 +179,58 @@ public void testListenersInvokedWhenIndexIsRelocated() throws Exception {
159179
deletedShards.contains(shardId));
160180
}
161181
} else {
162-
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
163-
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
182+
assertNoDeletions(nodeName);
164183
}
165184
}
166-
});
185+
}, 30L, TimeUnit.SECONDS);
167186
}
168187

169188
public void testListenersInvokedWhenIndexIsDangling() throws Exception {
170-
internalCluster().ensureAtLeastNumDataNodes(4);
189+
final String masterNode = internalCluster().startMasterOnlyNode();
190+
internalCluster().startDataOnlyNodes(4);
191+
ensureStableCluster(4 + 1, masterNode);
192+
171193
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
172194
createIndex(indexName, Settings.builder()
173-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(4, 10))
195+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4 * between(1, 2))
174196
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 1))
175197
.build());
176198

177199
final NumShards numShards = getNumShards(indexName);
178-
ensureGreen(indexName);
179-
180-
final ClusterState clusterState = clusterService().state();
200+
assertFalse(client().admin().cluster().prepareHealth()
201+
.setIndices(indexName)
202+
.setWaitForGreenStatus()
203+
.setWaitForEvents(Priority.LANGUID)
204+
.setWaitForNoRelocatingShards(true)
205+
.setWaitForNoInitializingShards(true)
206+
.get()
207+
.isTimedOut());
208+
209+
final ClusterState clusterState = internalCluster().clusterService(masterNode).state();
181210
final Index index = clusterState.metadata().index(indexName).getIndex();
182211
final Map<String, List<ShardRouting>> shardsByNodes = shardRoutingsByNodes(clusterState, index);
183212
assertThat(shardsByNodes.values().stream().mapToInt(List::size).sum(), equalTo(numShards.totalNumShards));
184213

185214
for (Map.Entry<String, List<ShardRouting>> shardsByNode : shardsByNodes.entrySet()) {
186-
final String nodeName = shardsByNode.getKey();
187-
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
188-
assertTrue("Expecting no indices deleted on node " + nodeName, plugin.deletedIndices.isEmpty());
189-
assertTrue("Expecting no shards deleted on node " + nodeName, plugin.deletedShards.isEmpty());
215+
assertNoDeletions(shardsByNode.getKey());
190216
}
191217

192218
final String stoppedNode = randomFrom(shardsByNodes.keySet());
193219
final Settings stoppedNodeDataPathSettings = internalCluster().dataPathSettings(stoppedNode);
194220
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(stoppedNode));
221+
ensureStableCluster(3 + 1, masterNode);
195222

196223
assertAcked(client().admin().indices().prepareDelete(indexName));
197224

198225
final String restartedNode = internalCluster().startNode(stoppedNodeDataPathSettings);
226+
ensureStableCluster(4 + 1, masterNode);
227+
assertPendingDeletesProcessed();
228+
199229
assertBusy(() -> {
200230
final IndexFoldersDeletionListenerPlugin plugin = plugin(restartedNode);
201231
assertTrue("Listener should have been notified of deletion of index " + index + " on node " + restartedNode,
202232
plugin.deletedIndices.contains(index));
203-
});
233+
}, 30L, TimeUnit.SECONDS);
204234
}
205235

206236
public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception {
@@ -229,6 +259,7 @@ public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception {
229259

230260
logger.debug("--> stopping data node [{}], the data left on disk will be injected as left-overs in a newer data node", dataNode);
231261
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode));
262+
ensureStableCluster(1, masterNode);
232263

233264
logger.debug("--> deleting leftover indices");
234265
assertAcked(client().admin().indices().prepareDelete("index-*"));
@@ -266,6 +297,7 @@ public void testListenersInvokedWhenIndexHasLeftOverShard() throws Exception {
266297
dataPaths.stream().map(p -> p.toAbsolutePath().toString()).collect(Collectors.toList()))
267298
.putNull(Environment.PATH_SHARED_DATA_SETTING.getKey())
268299
.build());
300+
ensureStableCluster(1 + 1, masterNode);
269301

270302
final IndexFoldersDeletionListenerPlugin plugin = plugin(dataNode);
271303
assertTrue("Expecting no shards deleted on node " + dataNode, plugin.deletedShards.isEmpty());
@@ -330,4 +362,12 @@ private static void assertPendingDeletesProcessed() throws Exception {
330362
services.forEach(indicesService -> assertFalse(indicesService.hasUncompletedPendingDeletes()));
331363
});
332364
}
365+
366+
private static void assertNoDeletions(String nodeName) {
367+
final IndexFoldersDeletionListenerPlugin plugin = plugin(nodeName);
368+
assertTrue("Expecting no indices deleted on node [" + nodeName + "] but got: " + plugin.deletedIndices,
369+
plugin.deletedIndices.isEmpty());
370+
assertTrue("Expecting no shards deleted on node [" + nodeName + "] but got: " + plugin.deletedShards,
371+
plugin.deletedShards.isEmpty());
372+
}
333373
}

‎x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/SearchableSnapshotsPersistentCacheIntegTests.java

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@
77
package org.elasticsearch.xpack.searchablesnapshots.cache;
88

99
import org.apache.lucene.document.Document;
10-
import org.apache.lucene.util.LuceneTestCase;
1110
import org.elasticsearch.cluster.metadata.IndexMetadata;
1211
import org.elasticsearch.cluster.node.DiscoveryNode;
1312
import org.elasticsearch.cluster.node.DiscoveryNodes;
1413
import org.elasticsearch.common.settings.Settings;
1514
import org.elasticsearch.common.unit.ByteSizeUnit;
1615
import org.elasticsearch.common.unit.ByteSizeValue;
16+
import org.elasticsearch.common.unit.TimeValue;
1717
import org.elasticsearch.index.Index;
1818
import org.elasticsearch.index.IndexService;
1919
import org.elasticsearch.index.shard.ShardPath;
@@ -39,7 +39,6 @@
3939
import static org.hamcrest.Matchers.greaterThan;
4040
import static org.hamcrest.Matchers.notNullValue;
4141

42-
@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/66278")
4342
public class SearchableSnapshotsPersistentCacheIntegTests extends BaseSearchableSnapshotsIntegTestCase {
4443

4544
@Override
@@ -48,6 +47,8 @@ protected Settings nodeSettings(int nodeOrdinal) {
4847
.put(super.nodeSettings(nodeOrdinal))
4948
// ensure the cache is definitely used
5049
.put(CacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(1L, ByteSizeUnit.GB))
50+
// to make cache synchronization predictable
51+
.put(CacheService.SNAPSHOT_CACHE_SYNC_INTERVAL_SETTING.getKey(), TimeValue.timeValueHours(1L))
5152
.build();
5253
}
5354

@@ -141,19 +142,21 @@ public Settings onNodeStopped(String nodeName) {
141142
}
142143
});
143144

144-
persistentCache = internalCluster().getInstance(CacheService.class, dataNode).getPersistentCache();
145-
assertThat(persistentCache.getNumDocs(), equalTo((long) cacheFiles.size()));
145+
cacheService = internalCluster().getInstance(CacheService.class, dataNode);
146+
persistentCache = cacheService.getPersistentCache();
146147
ensureGreen(restoredIndexName);
147148

148149
cacheFiles.forEach(cacheFile -> assertTrue(cacheFile + " should have survived node restart", Files.exists(cacheFile)));
150+
assertThat("Cache files should be repopulated in cache", persistentCache.getNumDocs(), equalTo((long) cacheFiles.size()));
149151

150152
assertAcked(client().admin().indices().prepareDelete(restoredIndexName));
151153

152-
assertBusy(() -> cacheFiles.forEach(cacheFile -> assertFalse(cacheFile + " should have been cleaned up", Files.exists(cacheFile))));
153-
cacheService = internalCluster().getInstance(CacheService.class, dataNode);
154+
assertBusy(() -> {
155+
cacheFiles.forEach(cacheFile -> assertFalse(cacheFile + " should have been cleaned up", Files.exists(cacheFile)));
156+
assertTrue(internalCluster().getInstance(CacheService.class, dataNode).getPersistentCache().hasDeletions());
157+
});
154158
cacheService.synchronizeCache();
155159

156-
persistentCache = cacheService.getPersistentCache();
157160
assertThat(persistentCache.getNumDocs(), equalTo(0L));
158161
}
159162
}

0 commit comments

Comments
 (0)