Skip to content

Commit 85b0b54

Browse files
ywelschywelsch
committed
Fix refresh behavior in MockDiskUsagesIT (#57926)
Ensures that InternalClusterInfoService's internally cached stats are refreshed whenever the shard size or disk usage function (to mock out disk usage) are overridden. Closes #57888
1 parent 6fc8317 commit 85b0b54

File tree

2 files changed

+34
-26
lines changed

2 files changed

+34
-26
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java

Lines changed: 22 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,10 @@ public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
9292
clusterInfoService.onMaster();
9393

9494
// prevent any effects from in-flight recoveries, since we are only simulating a 100-byte disk
95-
clusterInfoService.shardSizeFunction = shardRouting -> 0L;
95+
clusterInfoService.setShardSizeFunctionAndRefresh(shardRouting -> 0L);
9696

9797
// start with all nodes below the watermark
98-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100));
98+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100)));
9999

100100
final boolean watermarkBytes = randomBoolean(); // we have to consistently use bytes or percentage for the disk watermark settings
101101
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
@@ -115,8 +115,8 @@ public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
115115
});
116116

117117
// move node2 above high watermark
118-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100,
119-
discoveryNode.getId().equals(nodeIds.get(2)) ? between(0, 9) : between(10, 100));
118+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100,
119+
discoveryNode.getId().equals(nodeIds.get(2)) ? between(0, 9) : between(10, 100)));
120120

121121
logger.info("--> waiting for shards to relocate off node [{}]", nodeIds.get(2));
122122

@@ -128,7 +128,7 @@ public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
128128
});
129129

130130
// move all nodes below watermark again
131-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100));
131+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100)));
132132

133133
logger.info("--> waiting for shards to rebalance back onto node [{}]", nodeIds.get(2));
134134

@@ -154,10 +154,10 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception {
154154
clusterInfoService.onMaster();
155155

156156
// prevent any effects from in-flight recoveries, since we are only simulating a 100-byte disk
157-
clusterInfoService.shardSizeFunction = shardRouting -> 0L;
157+
clusterInfoService.setShardSizeFunctionAndRefresh(shardRouting -> 0L);
158158

159159
// start with all nodes below the low watermark
160-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(15, 100));
160+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(15, 100)));
161161

162162
final boolean watermarkBytes = randomBoolean(); // we have to consistently use bytes or percentage for the disk watermark settings
163163
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
@@ -184,8 +184,8 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception {
184184

185185
// Move all nodes above the low watermark so no shard movement can occur, and at least one node above the flood stage watermark so
186186
// the index is blocked
187-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100,
188-
discoveryNode.getId().equals(nodeIds.get(2)) ? between(0, 4) : between(0, 9));
187+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100,
188+
discoveryNode.getId().equals(nodeIds.get(2)) ? between(0, 4) : between(0, 9)));
189189

190190
assertBusy(() -> assertBlocked(
191191
client().prepareIndex().setIndex("test").setType("doc").setId("1").setSource("foo", "bar"),
@@ -201,7 +201,7 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception {
201201
logger.info("--> index is confirmed read-only, releasing disk space");
202202

203203
// Move all nodes below the high watermark so that the index is unblocked
204-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100));
204+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100)));
205205

206206
// Attempt to create a new document until DiskUsageMonitor unblocks the index
207207
assertBusy(() -> {
@@ -230,10 +230,10 @@ public void testOnlyMovesEnoughShardsToDropBelowHighWatermark() throws Exception
230230
});
231231

232232
// shards are 1 byte large
233-
clusterInfoService.shardSizeFunction = shardRouting -> 1L;
233+
clusterInfoService.setShardSizeFunctionAndRefresh(shardRouting -> 1L);
234234

235235
// start with all nodes below the watermark
236-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 1000L, 1000L);
236+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 1000L, 1000L));
237237

238238
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
239239
.put(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%")
@@ -260,10 +260,10 @@ public void testOnlyMovesEnoughShardsToDropBelowHighWatermark() throws Exception
260260
.put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
261261

262262
// node2 suddenly has 99 bytes free, less than 10%, but moving one shard is enough to bring it up to 100 bytes free:
263-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 1000L,
263+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 1000L,
264264
discoveryNode.getId().equals(nodeIds.get(2))
265265
? 101L - masterAppliedClusterState.get().getRoutingNodes().node(nodeIds.get(2)).numberOfOwningShards()
266-
: 1000L);
266+
: 1000L));
267267

268268
clusterInfoService.refresh();
269269

@@ -302,13 +302,13 @@ public void testDoesNotExceedLowWatermarkWhenRebalancing() throws Exception {
302302
.put(CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "100%")));
303303

304304
// shards are 1 byte large
305-
clusterInfoService.shardSizeFunction = shardRouting -> 1L;
305+
clusterInfoService.setShardSizeFunctionAndRefresh(shardRouting -> 1L);
306306

307307
// node 2 only has space for one shard
308-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 1000L,
308+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 1000L,
309309
discoveryNode.getId().equals(nodeIds.get(2))
310310
? 150L - masterAppliedClusterState.get().getRoutingNodes().node(nodeIds.get(2)).numberOfOwningShards()
311-
: 1000L);
311+
: 1000L));
312312

313313
assertAcked(prepareCreate("test").setSettings(Settings.builder()
314314
.put("number_of_shards", 6)
@@ -352,10 +352,10 @@ public void testMovesShardsOffSpecificDataPathAboveWatermark() throws Exception
352352
final MockInternalClusterInfoService clusterInfoService = getMockInternalClusterInfoService();
353353

354354
// prevent any effects from in-flight recoveries, since we are only simulating a 100-byte disk
355-
clusterInfoService.shardSizeFunction = shardRouting -> 0L;
355+
clusterInfoService.setShardSizeFunctionAndRefresh(shardRouting -> 0L);
356356

357357
// start with all paths below the watermark
358-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100));
358+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100, between(10, 100)));
359359

360360
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
361361
.put(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%")
@@ -381,15 +381,13 @@ public void testMovesShardsOffSpecificDataPathAboveWatermark() throws Exception
381381
&& shardStats.getDataPath().startsWith(pathOverWatermark.toString()) == false).count();
382382
logger.info("--> shards on good path: [{}]", shardsOnGoodPath);
383383

384-
// one of the paths on node0 suddenly exceeds the high watermark
385-
clusterInfoService.diskUsageFunction = (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100L,
386-
fsInfoPath.getPath().startsWith(pathOverWatermark.toString()) ? between(0, 9) : between(10, 100));
387-
388384
// disable rebalancing, or else we might move shards back onto the over-full path since we're not faking that
389385
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
390386
.put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
391387

392-
clusterInfoService.refresh();
388+
// one of the paths on node0 suddenly exceeds the high watermark
389+
clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, 100L,
390+
fsInfoPath.getPath().startsWith(pathOverWatermark.toString()) ? between(0, 9) : between(10, 100)));
393391

394392
logger.info("--> waiting for shards to relocate off path [{}]", pathOverWatermark);
395393

‎test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,15 +42,25 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService {
4242
public static class TestPlugin extends Plugin {}
4343

4444
@Nullable // if no fakery should take place
45-
public volatile Function<ShardRouting, Long> shardSizeFunction;
45+
private volatile Function<ShardRouting, Long> shardSizeFunction;
4646

4747
@Nullable // if no fakery should take place
48-
public volatile BiFunction<DiscoveryNode, FsInfo.Path, FsInfo.Path> diskUsageFunction;
48+
private volatile BiFunction<DiscoveryNode, FsInfo.Path, FsInfo.Path> diskUsageFunction;
4949

5050
public MockInternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client) {
5151
super(settings, clusterService, threadPool, client);
5252
}
5353

54+
public void setDiskUsageFunctionAndRefresh(BiFunction<DiscoveryNode, FsInfo.Path, FsInfo.Path> diskUsageFunction) {
55+
this.diskUsageFunction = diskUsageFunction;
56+
refresh();
57+
}
58+
59+
public void setShardSizeFunctionAndRefresh(Function<ShardRouting, Long> shardSizeFunction) {
60+
this.shardSizeFunction = shardSizeFunction;
61+
refresh();
62+
}
63+
5464
@Override
5565
public ClusterInfo getClusterInfo() {
5666
final ClusterInfo clusterInfo = super.getClusterInfo();

0 commit comments

Comments
 (0)