From 0acc36c9e9f4efd83311924fa2f871fc315354fd Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 4 Dec 2018 17:38:03 +0100 Subject: [PATCH 1/8] Hide un-recovered state and enable some tests requiring storage/recovery --- .../ingest/common/IngestRestartIT.java | 7 ---- .../cluster/coordination/Coordinator.java | 3 +- .../gateway/ClusterStateUpdaters.java | 20 +++++++++- .../admin/cluster/stats/ClusterStatsIT.java | 8 ++-- .../admin/indices/create/CreateIndexIT.java | 9 ----- .../gateway/ClusterStateUpdatersTests.java | 39 +++++++++++++++++++ .../gateway/GatewayIndexStateIT.java | 12 ++---- .../gateway/QuorumGatewayIT.java | 8 ---- .../gateway/RecoverAfterNodesIT.java | 19 ++++----- .../index/store/CorruptedFileIT.java | 2 - .../elasticsearch/indices/flush/FlushIT.java | 9 ----- .../PersistentTasksExecutorFullRestartIT.java | 9 ----- .../decider/EnableAssignmentDeciderIT.java | 8 ---- .../license/LicenseServiceClusterTests.java | 1 - .../integration/MlDistributedFailureIT.java | 7 ---- 15 files changed, 74 insertions(+), 87 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java index 414877a14e053..c308a44cb85fc 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java @@ -46,13 +46,6 @@ @ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.TEST) public class IngestRestartIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // no state persistence yet - .build(); - } - @Override protected Collection> nodePlugins() { return Arrays.asList(IngestCommonPlugin.class, CustomScriptPlugin.class); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index c0d8d054740c6..d68dc5924f7e4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -81,6 +81,7 @@ import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentSet; import static org.elasticsearch.discovery.DiscoverySettings.NO_MASTER_BLOCK_WRITES; +import static org.elasticsearch.gateway.ClusterStateUpdaters.hideStateIfNotRecovered; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; public class Coordinator extends AbstractLifecycleComponent implements Discovery { @@ -209,7 +210,7 @@ private void handleApplyCommit(ApplyCommitRequest applyCommitRequest, ActionList logger.trace("handleApplyCommit: applying commit {}", applyCommitRequest); coordinationState.get().handleCommit(applyCommitRequest); - final ClusterState committedState = coordinationState.get().getLastAcceptedState(); + final ClusterState committedState = hideStateIfNotRecovered(coordinationState.get().getLastAcceptedState()); applierState = mode == Mode.CANDIDATE ? clusterStateWithNoMasterBlock(committedState) : committedState; if (applyCommitRequest.getSourceNode().equals(getLocalNode())) { // master node applies the committed state at the end of the publication process, not here. diff --git a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java index ae18cf7207900..1b879421f87da 100644 --- a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java +++ b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java @@ -36,7 +36,9 @@ import java.util.Map; -class ClusterStateUpdaters { +import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; + +public class ClusterStateUpdaters { private static final Logger logger = LogManager.getLogger(ClusterStateUpdaters.class); static ClusterState setLocalNode(final ClusterState clusterState, DiscoveryNode localNode) { @@ -149,4 +151,20 @@ static ClusterState mixCurrentStateAndRecoveredState(final ClusterState currentS .build(); } + public static ClusterState hideStateIfNotRecovered(ClusterState state) { + if (state.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) { + ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(state.blocks()); + blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK); + blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK); + for (IndexMetaData indexMetaData: state.metaData()) { + blocks.removeIndexBlocks(indexMetaData.getIndex().getName()); + } + return ClusterState.builder(state) + .metaData(MetaData.EMPTY_META_DATA) + .blocks(blocks.build()) + .build(); + } + return state; + } + } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java index 7375cfafee6f5..427aaa45cd678 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -34,7 +34,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.discovery.TestZenDiscovery; import org.hamcrest.Matchers; import java.io.IOException; @@ -212,15 +211,14 @@ public void testAllocatedProcessors() throws Exception { } public void testClusterStatusWhenStateNotRecovered() throws Exception { - internalCluster().startMasterOnlyNode(Settings.builder().put("gateway.recover_after_nodes", 2) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false).build()); + internalCluster().startMasterOnlyNode(Settings.builder().put("gateway.recover_after_nodes", 2).build()); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get(); assertThat(response.getStatus(), equalTo(ClusterHealthStatus.RED)); if (randomBoolean()) { - internalCluster().startMasterOnlyNode(Settings.builder().put(TestZenDiscovery.USE_ZEN2.getKey(), false).build()); + internalCluster().startMasterOnlyNode(); } else { - internalCluster().startDataOnlyNode(Settings.builder().put(TestZenDiscovery.USE_ZEN2.getKey(), false).build()); + internalCluster().startDataOnlyNode(); } // wait for the cluster status to settle ensureGreen(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 318f14040cb65..88a2cdf6c0675 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -42,7 +42,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.test.discovery.TestZenDiscovery; import java.util.HashMap; import java.util.HashSet; @@ -67,14 +66,6 @@ @ClusterScope(scope = Scope.TEST) public class CreateIndexIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - // testIndexWithUnknownSetting and testRestartIndexCreationAfterFullClusterRestart fail with Zen2 - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) - .build(); - } - public void testCreationDateGivenFails() { try { prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 4L)).get(); diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java index 91c45681b4696..15131ebca60dd 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java @@ -45,6 +45,7 @@ import static org.elasticsearch.cluster.metadata.MetaData.CLUSTER_READ_ONLY_BLOCK; import static org.elasticsearch.gateway.ClusterStateUpdaters.closeBadIndices; +import static org.elasticsearch.gateway.ClusterStateUpdaters.hideStateIfNotRecovered; import static org.elasticsearch.gateway.ClusterStateUpdaters.mixCurrentStateAndRecoveredState; import static org.elasticsearch.gateway.ClusterStateUpdaters.recoverClusterBlocks; import static org.elasticsearch.gateway.ClusterStateUpdaters.removeStateNotRecoveredBlock; @@ -271,4 +272,42 @@ public void testSetLocalNode() { assertThat(updatedState.nodes().getSize(), is(1)); } + public void testDoNotHideStateIfRecovered() { + final IndexMetaData indexMetaData = createIndexMetaData("test", Settings.EMPTY); + final MetaData metaData = MetaData.builder() + .persistentSettings(Settings.builder().put("test", "test").build()) + .put(indexMetaData, false) + .build(); + final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metaData(metaData) + .build(); + assertMetaDataEquals(initialState, hideStateIfNotRecovered(initialState)); + } + + public void testHideStateIfNotRecovered() { + final IndexMetaData indexMetaData = createIndexMetaData("test", + Settings.builder().put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), true).build()); + final MetaData metaData = MetaData.builder() + .persistentSettings(Settings.builder().put(MetaData.SETTING_READ_ONLY_SETTING.getKey(), true).build()) + .transientSettings(Settings.builder().put(MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true).build()) + .put(indexMetaData, false) + .build(); + final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metaData(metaData) + .blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) + .build(); + final DiscoveryNode localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(), + Sets.newHashSet(DiscoveryNode.Role.MASTER), Version.CURRENT); + final ClusterState updatedState = Function.identity() + .andThen(state -> setLocalNode(state, localNode)) + .andThen(ClusterStateUpdaters::recoverClusterBlocks) + .apply(initialState); + final ClusterState hiddenState = hideStateIfNotRecovered(updatedState); + assertTrue(MetaData.isGlobalStateEquals(hiddenState.metaData(), MetaData.EMPTY_META_DATA)); + assertTrue(hiddenState.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)); + assertFalse(hiddenState.blocks().hasGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)); + assertFalse(hiddenState.blocks().hasGlobalBlock(MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK)); + assertFalse(hiddenState.blocks().hasIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_READ_ONLY_BLOCK)); + } + } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 112dca8f8c2d3..c0ccd0845bbd6 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -66,14 +66,6 @@ public class GatewayIndexStateIT extends ESIntegTestCase { private final Logger logger = LogManager.getLogger(GatewayIndexStateIT.class); - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - // testRecoverBrokenIndexMetadata, testRecoverMissingAnalyzer, testDanglingIndices and testArchiveBrokenClusterSettings fail - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) - .build(); - } - public void testMappingMetaDataParsed() throws Exception { logger.info("--> starting 1 nodes"); internalCluster().startNode(); @@ -285,7 +277,9 @@ public void testTwoNodesSingleDoc() throws Exception { public void testDanglingIndices() throws Exception { logger.info("--> starting two nodes"); - final String node_1 = internalCluster().startNodes(2).get(0); + final String node_1 = internalCluster().startNodes(2, + //TODO fails wih Zen2 + Settings.builder().put(TestZenDiscovery.USE_ZEN2.getKey(), false).build()).get(0); logger.info("--> indexing a simple document"); client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); diff --git a/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java index d3577ee60bcc4..6483c040a713a 100644 --- a/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/QuorumGatewayIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster.RestartCallback; -import org.elasticsearch.test.discovery.TestZenDiscovery; import java.util.concurrent.TimeUnit; @@ -40,13 +39,6 @@ @ClusterScope(numDataNodes = 0, scope = Scope.TEST) public class QuorumGatewayIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // no state persistence yet - .build(); - } - @Override protected int numberOfReplicas() { return 2; diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index fbd4cc8b6256b..94aa403c3d3e9 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.coordination.ClusterBootstrapService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.zen.ElectMasterService; @@ -29,7 +30,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.discovery.TestZenDiscovery; import java.util.Set; @@ -40,13 +40,6 @@ public class RecoverAfterNodesIT extends ESIntegTestCase { private static final TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10); - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // recover_after no implemented in Zen2 yet - .build(); - } - public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException { long start = System.currentTimeMillis(); Set blocks; @@ -67,7 +60,9 @@ public Client startNode(Settings.Builder settings, int minMasterNodes) { public void testRecoverAfterNodes() throws Exception { logger.info("--> start node (1)"); - Client clientNode1 = startNode(Settings.builder().put("gateway.recover_after_nodes", 3), 1); + Client clientNode1 = startNode(Settings.builder() + .put("gateway.recover_after_nodes", 3) + .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1), 1); assertThat(clientNode1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -94,7 +89,8 @@ public void testRecoverAfterMasterNodes() throws Exception { logger.info("--> start master_node (1)"); Client master1 = startNode(Settings.builder() .put("gateway.recover_after_master_nodes", 2).put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true) + .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1), 1); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); @@ -140,7 +136,8 @@ public void testRecoverAfterDataNodes() throws Exception { Client master1 = startNode(Settings.builder() .put("gateway.recover_after_data_nodes", 2) .put(Node.NODE_DATA_SETTING.getKey(), false) - .put(Node.NODE_MASTER_SETTING.getKey(), true), 1); + .put(Node.NODE_MASTER_SETTING.getKey(), true) + .put(ClusterBootstrapService.INITIAL_MASTER_NODE_COUNT_SETTING.getKey(), 1), 1); assertThat(master1.admin().cluster().prepareState().setLocal(true).execute().actionGet() .getState().blocks().global(ClusterBlockLevel.METADATA_WRITE), hasItem(GatewayService.STATE_NOT_RECOVERED_BLOCK)); diff --git a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java index d6db76c315a37..9fc83cd39de06 100644 --- a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -68,7 +68,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.MockIndexEventListener; -import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.test.store.MockFSIndexStore; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; @@ -117,7 +116,6 @@ protected Settings nodeSettings(int nodeOrdinal) { // speed up recoveries .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), 5) .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 5) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // no state persistence yet .build(); } diff --git a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java index 90767313d8104..774c50097d832 100644 --- a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java @@ -56,7 +56,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.discovery.TestZenDiscovery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.emptyIterable; @@ -66,14 +65,6 @@ public class FlushIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - // uses fullClusterRestart - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) - .build(); - } - public void testWaitIfOngoing() throws InterruptedException { createIndex("test"); ensureGreen("test"); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java index 01902dddde70a..195e14f65c004 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java @@ -20,13 +20,11 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams; import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.ArrayList; @@ -41,13 +39,6 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 1) public class PersistentTasksExecutorFullRestartIT extends ESIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // no state persistence yet - .build(); - } - @Override protected Collection> nodePlugins() { return Collections.singletonList(TestPersistentTasksPlugin.class); diff --git a/server/src/test/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java b/server/src/test/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java index 1d8d77509ab80..aeb4d9b3a9bfb 100644 --- a/server/src/test/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java +++ b/server/src/test/java/org/elasticsearch/persistent/decider/EnableAssignmentDeciderIT.java @@ -32,7 +32,6 @@ import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.discovery.TestZenDiscovery; import java.util.Collection; import java.util.concurrent.CountDownLatch; @@ -56,13 +55,6 @@ protected Collection> transportClientPlugins() { return nodePlugins(); } - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // state recovery not completed in Zen2 - .build(); - } - @Override protected boolean ignoreExternalCluster() { return true; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java index 17819614ea42d..87946012bd67e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java @@ -43,7 +43,6 @@ private Settings.Builder nodeSettingsBuilder(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("node.data", true) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // no state persistence .put("resource.reload.interval.high", "500ms"); // for license mode file watcher } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index 9ca8677953471..f7407165e7046 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -47,13 +47,6 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(TestZenDiscovery.USE_ZEN2.getKey(), false) // no state persistence yet - .build(); - } - public void testFailOver() throws Exception { internalCluster().ensureAtLeastNumDataNodes(3); ensureStableClusterOnAllNodes(3); From 7666a00b14f022af02831ca2f9bb4aee43e9dac5 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 4 Dec 2018 18:05:50 +0100 Subject: [PATCH 2/8] testIndexDeletionWhenNodeRejoins fails on zen2 --- .../java/org/elasticsearch/gateway/GatewayIndexStateIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index c0ccd0845bbd6..78393f94d7958 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -332,7 +332,8 @@ public void testIndexDeletionWhenNodeRejoins() throws Exception { final List nodes; logger.info("--> starting a cluster with " + numNodes + " nodes"); nodes = internalCluster().startNodes(numNodes, - Settings.builder().put(IndexGraveyard.SETTING_MAX_TOMBSTONES.getKey(), randomIntBetween(10, 100)).build()); + Settings.builder().put(IndexGraveyard.SETTING_MAX_TOMBSTONES.getKey(), randomIntBetween(10, 100)) + .put(TestZenDiscovery.USE_ZEN2.getKey(), false).build()); logger.info("--> create an index"); createIndex(indexName); From b5084645e8f741ffdf6edab7d3c25c01880c5e94 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 4 Dec 2018 18:16:31 +0100 Subject: [PATCH 3/8] testIndexDeletionWhenNodeRejoins add TODO --- .../test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 78393f94d7958..e7d6c966988fb 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -333,6 +333,7 @@ public void testIndexDeletionWhenNodeRejoins() throws Exception { logger.info("--> starting a cluster with " + numNodes + " nodes"); nodes = internalCluster().startNodes(numNodes, Settings.builder().put(IndexGraveyard.SETTING_MAX_TOMBSTONES.getKey(), randomIntBetween(10, 100)) + //TODO fails with Zen2 .put(TestZenDiscovery.USE_ZEN2.getKey(), false).build()); logger.info("--> create an index"); createIndex(indexName); From 7d26c64d77d2ce615c328f4c231fca82a97d0aaf Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 5 Dec 2018 12:56:09 +0100 Subject: [PATCH 4/8] Fix testNotWaitForQuorumCopies --- .../cluster/routing/PrimaryAllocationIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java index 16cd74a103ed2..44393d2e4580f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java @@ -341,7 +341,7 @@ public boolean clearData(String nodeName) { public void testNotWaitForQuorumCopies() throws Exception { logger.info("--> starting 3 nodes"); - internalCluster().startNodes(3, Settings.builder().put(TestZenDiscovery.USE_ZEN2.getKey(), false).build()); // needs state recovery + List nodes = internalCluster().startNodes(3); logger.info("--> creating index with 1 primary and 2 replicas"); assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() .put("index.number_of_shards", randomIntBetween(1, 3)).put("index.number_of_replicas", 2)).get()); @@ -349,9 +349,9 @@ public void testNotWaitForQuorumCopies() throws Exception { client().prepareIndex("test", "type1").setSource(jsonBuilder() .startObject().field("field", "value1").endObject()).get(); logger.info("--> removing 2 nodes from cluster"); - internalCluster().stopRandomDataNode(); - internalCluster().stopRandomDataNode(); - internalCluster().fullRestart(); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1))); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(2))); + internalCluster().restartRandomDataNode(); logger.info("--> checking that index still gets allocated with only 1 shard copy being available"); ensureYellow("test"); assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 1L); From 0591d54cd60de120fff11a3ef3f69947b17ff6f2 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 5 Dec 2018 15:09:13 +0100 Subject: [PATCH 5/8] Remove unused import --- .../java/org/elasticsearch/ingest/common/IngestRestartIT.java | 1 - .../xpack/ml/integration/MlDistributedFailureIT.java | 1 - 2 files changed, 2 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java index c308a44cb85fc..8c3976d2b175c 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestRestartIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.test.discovery.TestZenDiscovery; import java.util.Arrays; import java.util.Collection; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index f7407165e7046..2e14289da705e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -20,7 +20,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask; -import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; From 254aa2fba23aa23140d54f79908dd848627e834c Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 5 Dec 2018 15:12:41 +0100 Subject: [PATCH 6/8] randomization --- .../elasticsearch/cluster/routing/PrimaryAllocationIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java index 44393d2e4580f..e2777616f426d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java @@ -349,8 +349,8 @@ public void testNotWaitForQuorumCopies() throws Exception { client().prepareIndex("test", "type1").setSource(jsonBuilder() .startObject().field("field", "value1").endObject()).get(); logger.info("--> removing 2 nodes from cluster"); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1))); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(2))); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1), nodes.get(2))); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1), nodes.get(2))); internalCluster().restartRandomDataNode(); logger.info("--> checking that index still gets allocated with only 1 shard copy being available"); ensureYellow("test"); From 66e90e7c51b38b41d47a61e6a4bc0ff535d838dc Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 5 Dec 2018 15:27:17 +0100 Subject: [PATCH 7/8] Remove unused import --- .../org/elasticsearch/license/LicenseServiceClusterTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java index 87946012bd67e..00d1c47cdedaa 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseServiceClusterTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.XPackPlugin; From 9c2bfed2d81490dbb2ef09cab3e374dd137bf948 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 5 Dec 2018 16:21:01 +0100 Subject: [PATCH 8/8] Do not hide coordinationMetaData and clusterUUID --- .../gateway/ClusterStateUpdaters.java | 9 +++++++-- .../gateway/ClusterStateUpdatersTests.java | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java index 1b879421f87da..b9dbae83b7993 100644 --- a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java +++ b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java @@ -153,14 +153,19 @@ static ClusterState mixCurrentStateAndRecoveredState(final ClusterState currentS public static ClusterState hideStateIfNotRecovered(ClusterState state) { if (state.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) { - ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(state.blocks()); + final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(state.blocks()); blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK); blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK); for (IndexMetaData indexMetaData: state.metaData()) { blocks.removeIndexBlocks(indexMetaData.getIndex().getName()); } + final MetaData metaData = MetaData.builder() + .clusterUUID(state.metaData().clusterUUID()) + .coordinationMetaData(state.metaData().coordinationMetaData()) + .build(); + return ClusterState.builder(state) - .metaData(MetaData.EMPTY_META_DATA) + .metaData(metaData) .blocks(blocks.build()) .build(); } diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java index 15131ebca60dd..217caf10890d1 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.coordination.CoordinationMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -36,6 +37,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.Set; import java.util.function.BiConsumer; @@ -287,9 +289,18 @@ public void testDoNotHideStateIfRecovered() { public void testHideStateIfNotRecovered() { final IndexMetaData indexMetaData = createIndexMetaData("test", Settings.builder().put(IndexMetaData.INDEX_READ_ONLY_SETTING.getKey(), true).build()); + final String clusterUUID = UUIDs.randomBase64UUID(); + final CoordinationMetaData coordinationMetaData = new CoordinationMetaData(randomLong(), + new CoordinationMetaData.VotingConfiguration(Sets.newHashSet(generateRandomStringArray(5, 5, false))), + new CoordinationMetaData.VotingConfiguration(Sets.newHashSet(generateRandomStringArray(5, 5, false))), + Arrays.stream(generateRandomStringArray(5, 5, false)) + .map(id -> new CoordinationMetaData.VotingTombstone(id, id)) + .collect(Collectors.toSet())); final MetaData metaData = MetaData.builder() .persistentSettings(Settings.builder().put(MetaData.SETTING_READ_ONLY_SETTING.getKey(), true).build()) .transientSettings(Settings.builder().put(MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true).build()) + .clusterUUID(clusterUUID) + .coordinationMetaData(coordinationMetaData) .put(indexMetaData, false) .build(); final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE) @@ -302,8 +313,12 @@ public void testHideStateIfNotRecovered() { .andThen(state -> setLocalNode(state, localNode)) .andThen(ClusterStateUpdaters::recoverClusterBlocks) .apply(initialState); + final ClusterState hiddenState = hideStateIfNotRecovered(updatedState); - assertTrue(MetaData.isGlobalStateEquals(hiddenState.metaData(), MetaData.EMPTY_META_DATA)); + + assertTrue(MetaData.isGlobalStateEquals(hiddenState.metaData(), + MetaData.builder().coordinationMetaData(coordinationMetaData).clusterUUID(clusterUUID).build())); + assertThat(hiddenState.metaData().indices().size(), is(0)); assertTrue(hiddenState.blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)); assertFalse(hiddenState.blocks().hasGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK)); assertFalse(hiddenState.blocks().hasGlobalBlock(MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK));