diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 1ed7fb04407f1..ded77ce4287d6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -649,7 +649,7 @@ private static IndexService validateActiveShardCountAndCreateIndexService(String "]: cannot be greater than number of shard copies [" + (tmpImd.getNumberOfReplicas() + 1) + "]"); } - return indicesService.createIndex(tmpImd, Collections.emptyList()); + return indicesService.createIndex(tmpImd, Collections.emptyList(), false); } private void validate(CreateIndexClusterStateUpdateRequest request, ClusterState state) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java index 5efd4b6eae8bc..c6149682a203a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java @@ -140,7 +140,7 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable new ParameterizedMessage("failed to write dangling indices state for index {}", index()), e); + } + } + + // method is synchronized so that IndexService can't be closed while we're deleting dangling indices information + public synchronized void deleteDanglingIndicesInfo() { + if (closed.get()) { + return; + } + try { + MetaDataStateFormat.deleteMetaState(nodeEnv.indexPaths(index())); + } catch (IOException e) { + logger.warn(() -> new ParameterizedMessage("failed to delete dangling indices state for index {}", index()), e); + } + } public String indexUUID() { return indexSettings.getUUID(); @@ -669,24 +695,30 @@ public IndexMetaData getMetaData() { return indexSettings.getIndexMetaData(); } + private final CopyOnWriteArrayList> metaDataListeners = new CopyOnWriteArrayList<>(); + + public void addMetaDataListener(Consumer listener) { + metaDataListeners.add(listener); + } + @Override public synchronized void updateMetaData(final IndexMetaData currentIndexMetaData, final IndexMetaData newIndexMetaData) { - final boolean updateIndexMetaData = indexSettings.updateIndexMetaData(newIndexMetaData); + final boolean updateIndexSettings = indexSettings.updateIndexMetaData(newIndexMetaData); if (Assertions.ENABLED && currentIndexMetaData != null) { final long currentSettingsVersion = currentIndexMetaData.getSettingsVersion(); final long newSettingsVersion = newIndexMetaData.getSettingsVersion(); if (currentSettingsVersion == newSettingsVersion) { - assert updateIndexMetaData == false; + assert updateIndexSettings == false; } else { - assert updateIndexMetaData; + assert updateIndexSettings; assert currentSettingsVersion < newSettingsVersion : "expected current settings version [" + currentSettingsVersion + "] " + "to be less than new settings version [" + newSettingsVersion + "]"; } } - if (updateIndexMetaData) { + if (updateIndexSettings) { for (final IndexShard shard : this.shards.values()) { try { shard.onSettingsChanged(); @@ -722,6 +754,8 @@ public boolean isForceExecution() { } updateFsyncTaskIfNecessary(); } + + metaDataListeners.forEach(c -> c.accept(newIndexMetaData)); } private void updateFsyncTaskIfNecessary() { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index acef5d9626ede..62be919485669 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -41,6 +41,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; @@ -65,8 +66,12 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.AbstractRefCounted; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.common.util.iterable.Iterables; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; @@ -116,6 +121,7 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.node.Node; import org.elasticsearch.plugins.IndexStorePlugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; @@ -158,6 +164,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; +import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory; import static org.elasticsearch.index.IndexService.IndexCreationContext.CREATE_INDEX; import static org.elasticsearch.index.IndexService.IndexCreationContext.META_DATA_VERIFICATION; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -172,6 +179,11 @@ public class IndicesService extends AbstractLifecycleComponent public static final Setting INDICES_ID_FIELD_DATA_ENABLED_SETTING = Setting.boolSetting("indices.id_field_data.enabled", false, Property.Dynamic, Property.NodeScope); + public static final Setting WRITE_DANGLING_INDICES_INFO_SETTING = Setting.boolSetting( + "gateway.write_dangling_indices_info", + true, + Setting.Property.NodeScope + ); /** * The node's settings. @@ -209,6 +221,12 @@ public class IndicesService extends AbstractLifecycleComponent private final CountDownLatch closeLatch = new CountDownLatch(1); private volatile boolean idFieldDataEnabled; + @Nullable + private final EsThreadPoolExecutor danglingIndicesThreadPoolExecutor; + private final Set danglingIndicesToWrite = Sets.newConcurrentHashSet(); + private final boolean nodeWriteDanglingIndicesInfo; + + @Override protected void doStart() { // Start thread that will manage cleaning the field data cache periodically @@ -289,12 +307,25 @@ protected void closeInternal() { } } }; + + final String nodeName = Objects.requireNonNull(Node.NODE_NAME_SETTING.get(settings)); + nodeWriteDanglingIndicesInfo = WRITE_DANGLING_INDICES_INFO_SETTING.get(settings); + danglingIndicesThreadPoolExecutor = nodeWriteDanglingIndicesInfo ? EsExecutors.newScaling( + nodeName + "/" + DANGLING_INDICES_UPDATE_THREAD_NAME, + 1, 1, + 0, TimeUnit.MILLISECONDS, + daemonThreadFactory(nodeName, DANGLING_INDICES_UPDATE_THREAD_NAME), + threadPool.getThreadContext()) : null; } + private static final String DANGLING_INDICES_UPDATE_THREAD_NAME = "DanglingIndices#updateTask"; + @Override protected void doStop() { + ThreadPool.terminate(danglingIndicesThreadPoolExecutor, 10, TimeUnit.SECONDS); + ExecutorService indicesStopExecutor = - Executors.newFixedThreadPool(5, EsExecutors.daemonThreadFactory(settings, "indices_shutdown")); + Executors.newFixedThreadPool(5, daemonThreadFactory(settings, "indices_shutdown")); // Copy indices because we modify it asynchronously in the body of the loop final Set indices = this.indices.values().stream().map(s -> s.index()).collect(Collectors.toSet()); @@ -455,6 +486,7 @@ public boolean hasIndex(Index index) { public IndexService indexService(Index index) { return indices.get(index.getUUID()); } + /** * Returns an IndexService for the specified index if exists otherwise a {@link IndexNotFoundException} is thrown. */ @@ -478,7 +510,8 @@ public IndexService indexServiceSafe(Index index) { */ @Override public synchronized IndexService createIndex( - final IndexMetaData indexMetaData, final List builtInListeners) throws IOException { + final IndexMetaData indexMetaData, final List builtInListeners, + final boolean writeDanglingIndices) throws IOException { ensureChangesAllowed(); if (indexMetaData.getIndexUUID().equals(IndexMetaData.INDEX_UUID_NA_VALUE)) { throw new IllegalArgumentException("index must have a real UUID found value: [" + indexMetaData.getIndexUUID() + "]"); @@ -514,8 +547,18 @@ public void onStoreClosed(ShardId shardId) { indexingMemoryController); boolean success = false; try { + if (writeDanglingIndices && nodeWriteDanglingIndicesInfo) { + indexService.addMetaDataListener(imd -> updateDanglingIndicesInfo(index)); + } indexService.getIndexEventListener().afterIndexCreated(indexService); indices = Maps.copyMapWithAddedEntry(indices, index.getUUID(), indexService); + if (writeDanglingIndices) { + if (nodeWriteDanglingIndicesInfo) { + updateDanglingIndicesInfo(index); + } else { + indexService.deleteDanglingIndicesInfo(); + } + } success = true; return indexService; } finally { @@ -1487,4 +1530,51 @@ public static Optional checkShardLimit(int newShards, ClusterState state } return Optional.empty(); } + + private void updateDanglingIndicesInfo(Index index) { + assert DiscoveryNode.isDataNode(settings) : "dangling indices information should only be persisted on data nodes"; + assert nodeWriteDanglingIndicesInfo : "writing dangling indices info is not enabled"; + assert danglingIndicesThreadPoolExecutor != null : "executor for dangling indices info is not available"; + if (danglingIndicesToWrite.add(index)) { + logger.trace("triggered dangling indices update for {}", index); + final long triggeredTimeMillis = threadPool.relativeTimeInMillis(); + try { + danglingIndicesThreadPoolExecutor.execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + logger.warn(() -> new ParameterizedMessage("failed to write dangling indices state for index {}", index), e); + } + + @Override + protected void doRun() { + final boolean exists = danglingIndicesToWrite.remove(index); + assert exists : "removed non-existing item for " + index; + final IndexService indexService = indices.get(index.getUUID()); + if (indexService != null) { + final long executedTimeMillis = threadPool.relativeTimeInMillis(); + logger.trace("writing out dangling indices state for index {}, triggered {} ago", index, + TimeValue.timeValueMillis(Math.min(0L, executedTimeMillis - triggeredTimeMillis))); + indexService.writeDanglingIndicesInfo(); + final long completedTimeMillis = threadPool.relativeTimeInMillis(); + logger.trace("writing out of dangling indices state for index {} completed after {}", index, + TimeValue.timeValueMillis(Math.min(0L, completedTimeMillis - executedTimeMillis))); + } else { + logger.trace("omit writing dangling indices state for index {} as index is deallocated on this node", index); + } + } + }); + } catch (EsRejectedExecutionException e) { + // ignore cases where we are shutting down..., there is really nothing interesting to be done here... + assert danglingIndicesThreadPoolExecutor.isShutdown(); + } + } else { + logger.trace("dangling indices update already pending for {}", index); + } + } + + // visible for testing + public boolean allPendingDanglingIndicesWritten() { + return nodeWriteDanglingIndicesInfo == false || + (danglingIndicesToWrite.isEmpty() && danglingIndicesThreadPoolExecutor.getActiveCount() == 0); + } } diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index b72a56b4a0543..f070daab9b1cd 100644 --- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -499,7 +499,7 @@ private void createIndices(final ClusterState state) { AllocatedIndex indexService = null; try { - indexService = indicesService.createIndex(indexMetaData, buildInIndexListener); + indexService = indicesService.createIndex(indexMetaData, buildInIndexListener, true); if (indexService.updateMapping(null, indexMetaData) && sendRefreshMapping) { nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(), new NodeMappingRefreshAction.NodeMappingRefreshRequest(indexMetaData.getIndex().getName(), @@ -859,10 +859,12 @@ public interface AllocatedIndices> * @param indexMetaData the index metadata to create the index for * @param builtInIndexListener a list of built-in lifecycle {@link IndexEventListener} that should should be used along side with * the per-index listeners + * @param writeDanglingIndices whether dangling indices information should be written * @throws ResourceAlreadyExistsException if the index already exists. */ U createIndex(IndexMetaData indexMetaData, - List builtInIndexListener) throws IOException; + List builtInIndexListener, + boolean writeDanglingIndices) throws IOException; /** * Verify that the contents on disk for the given index is deleted; if not, delete the contents. diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java index a487c2301c458..71e73a796bb34 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.env.NodeMetaData; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.gateway.PersistedClusterStateService; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; @@ -329,6 +330,8 @@ public void testAllMasterEligibleNodesFailedDanglingIndexImport() throws Excepti logger.info("--> index 1 doc and ensure index is green"); client().prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); ensureGreen("test"); + assertBusy(() -> internalCluster().getInstances(IndicesService.class).forEach( + indicesService -> assertTrue(indicesService.allPendingDanglingIndicesWritten()))); logger.info("--> verify 1 doc in the index"); assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L); @@ -353,15 +356,14 @@ public boolean clearData(String nodeName) { internalCluster().startDataOnlyNode(dataNodeDataPathSettings); ensureStableCluster(2); - // TODO: @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48701") // dangling indices -// logger.info("--> verify that the dangling index exists and has green status"); -// assertBusy(() -> { -// assertThat(indexExists("test"), equalTo(true)); -// }); -// ensureGreen("test"); -// -// logger.info("--> verify the doc is there"); -// assertThat(client().prepareGet("test", "1").execute().actionGet().isExists(), equalTo(true)); + logger.info("--> verify that the dangling index exists and has green status"); + assertBusy(() -> { + assertThat(indexExists("test"), equalTo(true)); + }); + ensureGreen("test"); + + logger.info("--> verify the doc is there"); + assertThat(client().prepareGet("test", "1").execute().actionGet().isExists(), equalTo(true)); } public void testNoInitialBootstrapAfterDetach() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java index 16595e3749ae7..baebd32dffcb0 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; @@ -47,11 +48,13 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) public class NodeEnvironmentIT extends ESIntegTestCase { - public void testStartFailureOnDataForNonDataNode() { + public void testStartFailureOnDataForNonDataNode() throws Exception { final String indexName = "test-fail-on-data"; logger.info("--> starting one node"); - String node = internalCluster().startNode(); + final boolean writeDanglingIndices = randomBoolean(); + String node = internalCluster().startNode(Settings.builder() + .put(IndicesService.WRITE_DANGLING_INDICES_INFO_SETTING.getKey(), writeDanglingIndices).build()); Settings dataPathSettings = internalCluster().dataPathSettings(node); logger.info("--> creating index"); @@ -60,6 +63,10 @@ public void testStartFailureOnDataForNonDataNode() { .put("index.number_of_replicas", 0) ).get(); final String indexUUID = resolveIndex(indexName).getUUID(); + if (writeDanglingIndices) { + assertBusy(() -> internalCluster().getInstances(IndicesService.class).forEach( + indicesService -> assertTrue(indicesService.allPendingDanglingIndicesWritten()))); + } logger.info("--> restarting the node with node.data=false and node.master=false"); IllegalStateException ex = expectThrows(IllegalStateException.class, @@ -74,10 +81,19 @@ public Settings onNodeStopped(String nodeName) { .build(); } })); - assertThat(ex.getMessage(), - startsWith("Node is started with " - + Node.NODE_DATA_SETTING.getKey() - + "=false, but has shard data")); + if (writeDanglingIndices) { + assertThat(ex.getMessage(), + startsWith("Node is started with " + + Node.NODE_DATA_SETTING.getKey() + + "=false and " + + Node.NODE_MASTER_SETTING.getKey() + + "=false, but has index metadata")); + } else { + assertThat(ex.getMessage(), + startsWith("Node is started with " + + Node.NODE_DATA_SETTING.getKey() + + "=false, but has shard data")); + } logger.info("--> start the node again with node.data=true and node.master=true"); internalCluster().startNode(dataPathSettings); diff --git a/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandIT.java b/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandIT.java index 003957f3199e9..300191d3f2f3f 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandIT.java +++ b/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matcher; @@ -36,7 +37,8 @@ public void testRepurpose() throws Exception { logger.info("--> starting two nodes"); final String masterNode = internalCluster().startMasterOnlyNode(); - final String dataNode = internalCluster().startDataOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode( + Settings.builder().put(IndicesService.WRITE_DANGLING_INDICES_INFO_SETTING.getKey(), false).build()); logger.info("--> creating index"); prepareCreate(indexName, Settings.builder() diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index 069d96d7ddd33..97b29327b9d3b 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -202,7 +202,11 @@ public void testLoadFullStateAndUpdateAndClean() throws IOException { assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); assertThat(loadedMetaData.index("test1"), equalTo(index)); - metaStateService.deleteAll(); + if (randomBoolean()) { + metaStateService.unreferenceAll(); + } else { + metaStateService.deleteAll(); + } manifestAndMetaData = metaStateService.loadFullState(); assertTrue(manifestAndMetaData.v1().isEmpty()); metaData = manifestAndMetaData.v2(); diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index a196b55edce48..ee55545fe34ea 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -538,7 +538,6 @@ public void assertSyncIdsNotNull() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48701") public void testStartedShardFoundIfStateNotYetProcessed() throws Exception { // nodes may need to report the shards they processed the initial recovered cluster state from the master final String nodeName = internalCluster().startNode(); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java index 436126930e3d1..b3fdd53ee8f15 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java @@ -122,7 +122,7 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem }; indicesService.removeIndex(idx, DELETED, "simon says"); try { - IndexService index = indicesService.createIndex(metaData, Arrays.asList(countingListener)); + IndexService index = indicesService.createIndex(metaData, Arrays.asList(countingListener), false); assertEquals(3, counter.get()); idx = index.index(); ShardRouting newRouting = shardRouting; diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index 2f748bb4bdfa0..ea41f14a53af3 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -549,7 +549,7 @@ public void testGetEngineFactory() throws IOException { .numberOfShards(1) .numberOfReplicas(0) .build(); - final IndexService indexService = indicesService.createIndex(indexMetaData, Collections.emptyList()); + final IndexService indexService = indicesService.createIndex(indexMetaData, Collections.emptyList(), false); if (value != null && value) { assertThat(indexService.getEngineFactory(), instanceOf(FooEnginePlugin.FooEngineFactory.class)); } else { @@ -575,7 +575,7 @@ public void testConflictingEngineFactories() { final IndicesService indicesService = getIndicesService(); final IllegalStateException e = - expectThrows(IllegalStateException.class, () -> indicesService.createIndex(indexMetaData, Collections.emptyList())); + expectThrows(IllegalStateException.class, () -> indicesService.createIndex(indexMetaData, Collections.emptyList(), false)); final String pattern = ".*multiple engine factories provided for \\[foobar/.*\\]: \\[.*FooEngineFactory\\],\\[.*BarEngineFactory\\].*"; assertThat(e, hasToString(new RegexMatcher(pattern))); diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java index 80cf443e5007e..d7b94e26eff9e 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java @@ -195,7 +195,8 @@ protected class MockIndicesService implements AllocatedIndices buildInIndexListener) throws IOException { + List buildInIndexListener, + boolean writeDanglingIndices) throws IOException { MockIndexService indexService = new MockIndexService(new IndexSettings(indexMetaData, Settings.EMPTY)); indices = Maps.copyMapWithAddedEntry(indices, indexMetaData.getIndexUUID(), indexService); return indexService; diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 74b94ad3a16b4..901409739c1d6 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -110,6 +110,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assert.assertThat; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyList; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doAnswer; @@ -163,7 +164,7 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th // MetaDataCreateIndexService creates indices using its IndicesService instance to check mappings -> fake it here try { @SuppressWarnings("unchecked") final List listeners = anyList(); - when(indicesService.createIndex(any(IndexMetaData.class), listeners)) + when(indicesService.createIndex(any(IndexMetaData.class), listeners, anyBoolean())) .then(invocationOnMock -> { IndexService indexService = mock(IndexService.class); IndexMetaData indexMetaData = (IndexMetaData)invocationOnMock.getArguments()[0]; diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java index e642297b96a46..cbb8be70197d7 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java @@ -19,8 +19,8 @@ package org.elasticsearch.indices.recovery; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.InternalTestCluster; @@ -30,17 +30,18 @@ import static org.elasticsearch.cluster.metadata.IndexGraveyard.SETTING_MAX_TOMBSTONES; import static org.elasticsearch.gateway.DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48701") // TODO add dangling indices support @ClusterScope(numDataNodes = 0, scope = ESIntegTestCase.Scope.TEST) public class DanglingIndicesIT extends ESIntegTestCase { private static final String INDEX_NAME = "test-idx-1"; - private Settings buildSettings(boolean importDanglingIndices) { + private Settings buildSettings(boolean writeDanglingIndices, boolean importDanglingIndices) { return Settings.builder() // Don't keep any indices in the graveyard, so that when we delete an index, // it's definitely considered to be dangling. .put(SETTING_MAX_TOMBSTONES.getKey(), 0) + .put(IndicesService.WRITE_DANGLING_INDICES_INFO_SETTING.getKey(), writeDanglingIndices) .put(AUTO_IMPORT_DANGLING_INDICES_SETTING.getKey(), importDanglingIndices) .build(); } @@ -50,10 +51,21 @@ private Settings buildSettings(boolean importDanglingIndices) { * the cluster, so long as the recovery setting is enabled. */ public void testDanglingIndicesAreRecoveredWhenSettingIsEnabled() throws Exception { - final Settings settings = buildSettings(true); + final Settings settings = buildSettings(true, true); internalCluster().startNodes(3, settings); createIndex(INDEX_NAME, Settings.builder().put("number_of_replicas", 2).build()); + ensureGreen(INDEX_NAME); + assertBusy(() -> internalCluster().getInstances(IndicesService.class).forEach( + indicesService -> assertTrue(indicesService.allPendingDanglingIndicesWritten()))); + + boolean refreshIntervalChanged = randomBoolean(); + if (refreshIntervalChanged) { + client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings( + Settings.builder().put("index.refresh_interval", "42s").build()).get(); + assertBusy(() -> internalCluster().getInstances(IndicesService.class).forEach( + indicesService -> assertTrue(indicesService.allPendingDanglingIndicesWritten()))); + } // Restart node, deleting the index in its absence, so that there is a dangling index to recover internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() { @@ -66,6 +78,10 @@ public Settings onNodeStopped(String nodeName) throws Exception { }); assertBusy(() -> assertTrue("Expected dangling index " + INDEX_NAME + " to be recovered", indexExists(INDEX_NAME))); + if (refreshIntervalChanged) { + assertThat(client().admin().indices().prepareGetSettings(INDEX_NAME).get().getSetting(INDEX_NAME, "index.refresh_interval"), + equalTo("42s")); + } } /** @@ -73,9 +89,41 @@ public Settings onNodeStopped(String nodeName) throws Exception { * the cluster when the recovery setting is disabled. */ public void testDanglingIndicesAreNotRecoveredWhenSettingIsDisabled() throws Exception { - internalCluster().startNodes(3, buildSettings(false)); + internalCluster().startNodes(3, buildSettings(true, false)); createIndex(INDEX_NAME, Settings.builder().put("number_of_replicas", 2).build()); + ensureGreen(INDEX_NAME); + assertBusy(() -> internalCluster().getInstances(IndicesService.class).forEach( + indicesService -> assertTrue(indicesService.allPendingDanglingIndicesWritten()))); + + // Restart node, deleting the index in its absence, so that there is a dangling index to recover + internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() { + + @Override + public Settings onNodeStopped(String nodeName) throws Exception { + assertAcked(client().admin().indices().prepareDelete(INDEX_NAME)); + return super.onNodeStopped(nodeName); + } + }); + + // Since index recovery is async, we can't prove index recovery will never occur, just that it doesn't occur within some reasonable + // amount of time + assertFalse( + "Did not expect dangling index " + INDEX_NAME + " to be recovered", + waitUntil(() -> indexExists(INDEX_NAME), 1, TimeUnit.SECONDS) + ); + } + + /** + * Check that when dangling indices are not written, then they cannot be recovered into the cluster. + */ + public void testDanglingIndicesAreNotRecoveredWhenNotWritten() throws Exception { + internalCluster().startNodes(3, buildSettings(false, true)); + + createIndex(INDEX_NAME, Settings.builder().put("number_of_replicas", 2).build()); + ensureGreen(INDEX_NAME); + internalCluster().getInstances(IndicesService.class).forEach( + indicesService -> assertTrue(indicesService.allPendingDanglingIndicesWritten())); // Restart node, deleting the index in its absence, so that there is a dangling index to recover internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {