Closed
Description
Possibly related to #39509
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+7.x+internalClusterTest/2350/console
REPRODUCE WITH: ./gradlew :x-pack:plugin:ccr:internalClusterTest \
-Dtests.seed=377894C8C7910121 \
-Dtests.class=org.elasticsearch.xpack.ccr.CcrRetentionLeaseIT \
-Dtests.method="testRetentionLeasesAreNotBeingRenewedAfterRecoveryCompletes" \
-Dtests.security.manager=true \
-Dtests.locale=ja-JP-u-ca-japanese-x-lvariant-JP \
-Dtests.timezone=GMT0 \
-Dcompiler.java=11 \
-Druntime.java=8
REPRODUCE WITH: ./gradlew :x-pack:plugin:ccr:internalClusterTest \
-Dtests.seed=377894C8C7910121 \
-Dtests.class=org.elasticsearch.xpack.ccr.CcrRetentionLeaseIT \
-Dtests.method="testUnfollowFailsToRemoveRetentionLeases" \
-Dtests.security.manager=true \
-Dtests.locale=ja-JP-u-ca-japanese-x-lvariant-JP \
-Dtests.timezone=GMT0 \
-Dcompiler.java=11 \
-Druntime.java=8
REPRODUCE WITH: ./gradlew :x-pack:plugin:ccr:internalClusterTest \
-Dtests.seed=377894C8C7910121 \
-Dtests.class=org.elasticsearch.xpack.ccr.CcrRetentionLeaseIT \
-Dtests.method="testRetentionLeaseRenewedWhileFollowing" \
-Dtests.security.manager=true \
-Dtests.locale=ja-JP-u-ca-japanese-x-lvariant-JP \
-Dtests.timezone=GMT0 \
-Dcompiler.java=11 \
-Druntime.java=8
REPRODUCE WITH: ./gradlew :x-pack:plugin:ccr:internalClusterTest \
-Dtests.seed=377894C8C7910121 \
-Dtests.class=org.elasticsearch.xpack.ccr.CcrRetentionLeaseIT \
-Dtests.method="testRetentionLeaseIsAddedIfItDisappearsWhileFollowing" \
-Dtests.security.manager=true \
-Dtests.locale=ja-JP-u-ca-japanese-x-lvariant-JP \
-Dtests.timezone=GMT0 \
-Dcompiler.java=11 \
-Druntime.java=8
A number of errors like:
java.lang.AssertionError: CreateIndexResponse failed - index creation acked but not all shards were started
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked(ElasticsearchAssertions.java:125)
at org.elasticsearch.xpack.ccr.CcrRetentionLeaseIT.testRetentionLeaseIsAddedIfItDisappearsWhileFollowing(CcrRetentionLeaseIT.java:759)
java.lang.AssertionError: CreateIndexResponse failed - index creation acked but not all shards were started
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked(ElasticsearchAssertions.java:125)
at org.elasticsearch.xpack.ccr.CcrRetentionLeaseIT.testUnfollowRemovesRetentionLeases(CcrRetentionLeaseIT.java:379)
java.lang.AssertionError: CreateIndexResponse failed - index creation acked but not all shards were started
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked(ElasticsearchAssertions.java:125)
at org.elasticsearch.xpack.ccr.CcrRetentionLeaseIT.testForgetFollower(CcrRetentionLeaseIT.java:931)
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=47, name=elasticsearch[leaderd4][transport_worker][T#2], state=RUNNABLE, group=TGRP-CcrRetentionLeaseIT]
Caused by: java.lang.AssertionError: shard [leader][0], node[xAJZZ5hpRLCJklizBkx0uw], [P], recovery_source[new shard recovery], s[INITIALIZING], a[id=MYU37kVrQOiBS8JDjMLVJA], unassigned_info[[reason=INDEX_CREATED], at[2019-03-15T02:17:39.163Z], delayed=false, allocation_status[no_attempt]] is not a primary shard in primary mode
at __randomizedtesting.SeedInfo.seed([377894C8C7910121]:0)
at org.elasticsearch.index.shard.IndexShard.assertPrimaryMode(IndexShard.java:1576)
at org.elasticsearch.index.shard.IndexShard.renewRetentionLease(IndexShard.java:1974)
at org.elasticsearch.index.seqno.RetentionLeaseActions$Renew$TransportAction.doRetentionLeaseAction(RetentionLeaseActions.java:217)
at org.elasticsearch.index.seqno.RetentionLeaseActions$Renew$TransportAction.doRetentionLeaseAction(RetentionLeaseActions.java:193)
at org.elasticsearch.index.seqno.RetentionLeaseActions$TransportRetentionLeaseAction$1.onResponse(RetentionLeaseActions.java:103)
at org.elasticsearch.index.seqno.RetentionLeaseActions$TransportRetentionLeaseAction$1.onResponse(RetentionLeaseActions.java:98)
at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:269)
at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:236)
at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2503)
at org.elasticsearch.index.seqno.RetentionLeaseActions$TransportRetentionLeaseAction.asyncShardOperation(RetentionLeaseActions.java:97)
at org.elasticsearch.index.seqno.RetentionLeaseActions$TransportRetentionLeaseAction.asyncShardOperation(RetentionLeaseActions.java:59)
at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$ShardTransportHandler.messageReceived(TransportSingleShardAction.java:299)
at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$ShardTransportHandler.messageReceived(TransportSingleShardAction.java:292)
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1073)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192)
at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1032)
at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:918)
at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753)
at org.elasticsearch.transport.TcpTransport.consumeNetworkReads(TcpTransport.java:780)
at org.elasticsearch.transport.nio.MockNioTransport$MockTcpReadWriteHandler.consumeReads(MockNioTransport.java:241)
at org.elasticsearch.nio.SocketChannelContext.handleReadBytes(SocketChannelContext.java:213)
at org.elasticsearch.nio.BytesChannelContext.read(BytesChannelContext.java:47)
at org.elasticsearch.nio.EventHandler.handleRead(EventHandler.java:119)
at org.elasticsearch.transport.nio.TestEventHandler.handleRead(TestEventHandler.java:121)
at org.elasticsearch.nio.NioSelector.handleRead(NioSelector.java:397)
at org.elasticsearch.nio.NioSelector.processKey(NioSelector.java:246)
at org.elasticsearch.nio.NioSelector.singleLoop(NioSelector.java:172)
at org.elasticsearch.nio.NioSelector.runLoop(NioSelector.java:129)
at java.lang.Thread.run(Thread.java:748)