Skip to content

Commit 6fc3590

Browse files
committed
Adapt CloseFollowerIndexIT for replicated closed indices (#38767)
Now the test `CloseFollowerIndexIT` has been added in #38702, it needs to be adapted for replicated closed indices. The test closes the follower index which is lagging behind the leader index. When it's closed, no sanity checks are executed because it's a follower index (this is a consequence of #38702). But with replicated closed indices, the index is reinitialized as a closed index with a `NoOpEngine` and such engines make strong assertions on the values of the maximum sequence number and the global checkpoint. Since the values do not match, the shards cannot be created and fail and the cluster health turns RED. This commit adapts the `CloseFollowerIndexIT` test so that it wraps the default `UncaughtExceptionHandler` with a handler that tolerates any exception thrown by `ReadOnlyEngine.assertMaxSeqNoEqualsToGlobalCheckpoint()`. Replacing the default uncaught exception handler requires specific permissions, and instead of creating another gradle project it duplicates the `internalClusterTest` task to make it work without security manager for this specific test only. Relates to #33888
1 parent dfe8463 commit 6fc3590

File tree

2 files changed

+60
-7
lines changed

2 files changed

+60
-7
lines changed

x-pack/plugin/ccr/build.gradle

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,32 @@ integTest.enabled = false
1818
compileJava.options.compilerArgs << "-Xlint:-try"
1919
compileTestJava.options.compilerArgs << "-Xlint:-try"
2020

21+
// Integration Test classes that cannot run with the security manager
22+
String[] noSecurityManagerITClasses = [ "**/CloseFollowerIndexIT.class" ]
23+
2124
// Instead we create a separate task to run the
2225
// tests based on ESIntegTestCase
2326
task internalClusterTest(type: RandomizedTestingTask,
2427
group: JavaBasePlugin.VERIFICATION_GROUP,
2528
description: 'Java fantasy integration tests',
2629
dependsOn: unitTest.dependsOn) {
2730
include '**/*IT.class'
31+
exclude noSecurityManagerITClasses
2832
systemProperty 'es.set.netty.runtime.available.processors', 'false'
2933
}
3034
check.dependsOn internalClusterTest
3135
internalClusterTest.mustRunAfter test
3236

37+
task internalClusterTestNoSecurityManager(type: RandomizedTestingTask,
38+
group: JavaBasePlugin.VERIFICATION_GROUP,
39+
description: 'Java fantasy integration tests with no security manager',
40+
dependsOn: unitTest.dependsOn) {
41+
include noSecurityManagerITClasses
42+
systemProperty 'es.set.netty.runtime.available.processors', 'false'
43+
systemProperty 'tests.security.manager', 'false'
44+
}
45+
internalClusterTest.dependsOn internalClusterTestNoSecurityManager
46+
3347
// add all sub-projects of the qa sub-project
3448
gradle.projectsEvaluated {
3549
project.subprojects

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,21 @@
1111
import org.elasticsearch.action.support.ActiveShardCount;
1212
import org.elasticsearch.action.support.master.AcknowledgedResponse;
1313
import org.elasticsearch.cluster.ClusterState;
14-
import org.elasticsearch.cluster.block.ClusterBlock;
14+
import org.elasticsearch.cluster.health.ClusterHealthStatus;
15+
import org.elasticsearch.cluster.metadata.IndexMetaData;
1516
import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
1617
import org.elasticsearch.common.unit.ByteSizeValue;
1718
import org.elasticsearch.common.unit.TimeValue;
1819
import org.elasticsearch.common.xcontent.XContentType;
1920
import org.elasticsearch.index.IndexSettings;
21+
import org.elasticsearch.index.engine.ReadOnlyEngine;
2022
import org.elasticsearch.xpack.CcrIntegTestCase;
2123
import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
24+
import org.junit.After;
25+
import org.junit.Before;
2226

23-
import java.util.ArrayList;
24-
import java.util.List;
27+
import java.security.AccessController;
28+
import java.security.PrivilegedAction;
2529
import java.util.concurrent.atomic.AtomicBoolean;
2630

2731
import static java.util.Collections.singletonMap;
@@ -31,7 +35,36 @@
3135

3236
public class CloseFollowerIndexIT extends CcrIntegTestCase {
3337

34-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/38767")
38+
private Thread.UncaughtExceptionHandler uncaughtExceptionHandler;
39+
40+
@Before
41+
public void wrapUncaughtExceptionHandler() {
42+
uncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler();
43+
AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
44+
Thread.setDefaultUncaughtExceptionHandler((t, e) -> {
45+
if (t.getThreadGroup().getName().contains(getTestClass().getSimpleName())) {
46+
for (StackTraceElement element : e.getStackTrace()) {
47+
if (element.getClassName().equals(ReadOnlyEngine.class.getName())) {
48+
if (element.getMethodName().equals("assertMaxSeqNoEqualsToGlobalCheckpoint")) {
49+
return;
50+
}
51+
}
52+
}
53+
}
54+
uncaughtExceptionHandler.uncaughtException(t, e);
55+
});
56+
return null;
57+
});
58+
}
59+
60+
@After
61+
public void restoreUncaughtExceptionHandler() {
62+
AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
63+
Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler);
64+
return null;
65+
});
66+
}
67+
3568
public void testCloseAndReopenFollowerIndex() throws Exception {
3669
final String leaderIndexSettings = getIndexSettings(1, 1, singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true"));
3770
assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON));
@@ -67,16 +100,22 @@ public void testCloseAndReopenFollowerIndex() throws Exception {
67100
assertThat(response.isAcknowledged(), is(true));
68101

69102
ClusterState clusterState = followerClient().admin().cluster().prepareState().get().getState();
70-
List<ClusterBlock> blocks = new ArrayList<>(clusterState.getBlocks().indices().get("index2"));
71-
assertThat(blocks.size(), equalTo(1));
72-
assertThat(blocks.get(0).id(), equalTo(MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID));
103+
assertThat(clusterState.metaData().index("index2").getState(), is(IndexMetaData.State.CLOSE));
104+
assertThat(clusterState.getBlocks().hasIndexBlock("index2", MetaDataIndexStateService.INDEX_CLOSED_BLOCK), is(true));
105+
assertThat(followerClient().admin().cluster().prepareHealth("index2").get().getStatus(), equalTo(ClusterHealthStatus.RED));
73106

74107
isRunning.set(false);
75108
for (Thread thread : threads) {
76109
thread.join();
77110
}
111+
78112
assertAcked(followerClient().admin().indices().open(new OpenIndexRequest("index2")).get());
79113

114+
clusterState = followerClient().admin().cluster().prepareState().get().getState();
115+
assertThat(clusterState.metaData().index("index2").getState(), is(IndexMetaData.State.OPEN));
116+
assertThat(clusterState.getBlocks().hasIndexBlockWithId("index2", MetaDataIndexStateService.INDEX_CLOSED_BLOCK_ID), is(false));
117+
ensureFollowerGreen("index2");
118+
80119
refresh(leaderClient(), "index1");
81120
SearchRequest leaderSearchRequest = new SearchRequest("index1");
82121
leaderSearchRequest.source().trackTotalHits(true);

0 commit comments

Comments
 (0)