From b74875dd2d24fa56513e79ae4e2ea3de670a17f4 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 10 Sep 2019 12:24:35 -0400 Subject: [PATCH 1/2] Increase timeout for relocation tests --- .../recovery/IndexPrimaryRelocationIT.java | 20 ++++++++++++++----- .../elasticsearch/recovery/RelocationIT.java | 5 ++--- .../elasticsearch/test/ESIntegTestCase.java | 9 +++++++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java index cab3165f52daf..b71acc9dfe617 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java @@ -19,9 +19,9 @@ package org.elasticsearch.indices.recovery; -import org.apache.lucene.util.Constants; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.cluster.ClusterState; @@ -29,14 +29,15 @@ import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; -import static org.hamcrest.Matchers.equalTo; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class IndexPrimaryRelocationIT extends ESIntegTestCase { @@ -44,7 +45,6 @@ public class IndexPrimaryRelocationIT extends ESIntegTestCase { private static final int RELOCATION_COUNT = 15; public void testPrimaryRelocationWhileIndexing() throws Exception { - assumeFalse("https://github.com/elastic/elasticsearch/issues/46526", Constants.MAC_OS_X); internalCluster().ensureAtLeastNumDataNodes(randomIntBetween(2, 3)); client().admin().indices().prepareCreate("test") .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) @@ -56,7 +56,7 @@ public void testPrimaryRelocationWhileIndexing() throws Exception { Thread indexingThread = new Thread() { @Override public void run() { - while (finished.get() == false) { + while (finished.get() == false && numAutoGenDocs.get() < 10_000) { IndexResponse indexResponse = client().prepareIndex("test", "type", "id").setSource("field", "value").get(); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); DeleteResponse deleteResponse = client().prepareDelete("test", "type", "id").get(); @@ -82,8 +82,18 @@ public void run() { .add(new MoveAllocationCommand("test", 0, relocationSource.getId(), relocationTarget.getId())) .execute().actionGet(); ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth() + .setTimeout(TimeValue.timeValueSeconds(60)) .setWaitForEvents(Priority.LANGUID).setWaitForNoRelocatingShards(true).execute().actionGet(); - assertThat(clusterHealthResponse.isTimedOut(), equalTo(false)); + if (clusterHealthResponse.isTimedOut()) { + final String hotThreads = client().admin().cluster().prepareNodesHotThreads().setIgnoreIdleThreads(false).get().getNodes() + .stream().map(NodeHotThreads::getHotThreads).collect(Collectors.joining("\n")); + final ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); + logger.info("timed out for waiting for relocation iteration [{}] \ncluster state {} \nhot threads {}", + i, clusterState, hotThreads); + finished.set(true); + indexingThread.join(); + throw new AssertionError("timed out waiting for relocation iteration [" + i + "] "); + } logger.info("--> [iteration {}] relocation complete", i); relocationSource = relocationTarget; // indexing process aborted early, no need for more relocations as test has already failed diff --git a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 369daef08d1c3..a448736ed55ad 100644 --- a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -78,7 +78,6 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -446,7 +445,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO } } - public void testIndexAndRelocateConcurrently() throws ExecutionException, InterruptedException { + public void testIndexAndRelocateConcurrently() throws Exception { int halfNodes = randomIntBetween(1, 3); Settings[] nodeSettings = Stream.concat( Stream.generate(() -> Settings.builder().put("node.attr.color", "blue").build()).limit(halfNodes), @@ -494,7 +493,7 @@ public void testIndexAndRelocateConcurrently() throws ExecutionException, Interr numDocs *= 2; logger.info(" --> waiting for relocation to complete"); - ensureGreen("test"); // move all shards to the new nodes (it waits on relocation) + ensureGreen(TimeValue.timeValueSeconds(60), "test"); // move all shards to the new nodes (it waits on relocation) final int numIters = randomIntBetween(10, 20); for (int i = 0; i < numIters; i++) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 58ebc1be607ce..995d769d4d68f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -33,6 +33,7 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -166,6 +167,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.BooleanSupplier; import java.util.function.Function; +import java.util.stream.Collectors; import static org.elasticsearch.client.Requests.syncedFlushRequest; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; @@ -880,10 +882,13 @@ private ClusterHealthStatus ensureColor(ClusterHealthStatus clusterHealthStatus, ClusterHealthResponse actionGet = client().admin().cluster().health(healthRequest).actionGet(); if (actionGet.isTimedOut()) { - logger.info("{} timed out, cluster state:\n{}\n{}", + final String hotThreads = client().admin().cluster().prepareNodesHotThreads().setIgnoreIdleThreads(false).get().getNodes() + .stream().map(NodeHotThreads::getHotThreads).collect(Collectors.joining("\n")); + logger.info("{} timed out, cluster state:\n{}\n{}\n{}", method, client().admin().cluster().prepareState().get().getState(), - client().admin().cluster().preparePendingClusterTasks().get()); + client().admin().cluster().preparePendingClusterTasks().get(), + hotThreads); fail("timed out waiting for " + color + " state"); } assertThat("Expected at least " + clusterHealthStatus + " but got " + actionGet.getStatus(), From 601401f87bc46113542e82e2d9cc367f77236573 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 11 Sep 2019 08:31:13 -0400 Subject: [PATCH 2/2] =?UTF-8?q?Tanguy=E2=80=99s=20feedback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/org/elasticsearch/test/ESIntegTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 995d769d4d68f..1ee607aaa3be4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -884,7 +884,7 @@ private ClusterHealthStatus ensureColor(ClusterHealthStatus clusterHealthStatus, if (actionGet.isTimedOut()) { final String hotThreads = client().admin().cluster().prepareNodesHotThreads().setIgnoreIdleThreads(false).get().getNodes() .stream().map(NodeHotThreads::getHotThreads).collect(Collectors.joining("\n")); - logger.info("{} timed out, cluster state:\n{}\n{}\n{}", + logger.info("{} timed out, cluster state:\n{}\npending tasks:\n{}\nhot threads:\n{}\n", method, client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get(),