Skip to content

Commit 10bcb94

Browse files
martijnvgmartijnvg
authored
Migrate muted auto follow rolling upgrade test and unmute this test (#38900)
The rest of `CCRIT` is now no longer relevant, because the remaining test tests the same of the index following test in the rolling upgrade multi cluster module. Added `tests.upgrade_from_version` version to test. It is not needed in this branch, but is in 6.7 branch. Closes #37231
1 parent dacb0df commit 10bcb94

File tree

4 files changed

+164
-283
lines changed

4 files changed

+164
-283
lines changed

‎x-pack/qa/rolling-upgrade-multi-cluster/build.gradle

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ for (Version version : bwcVersions.wireCompatible) {
4141

4242
Task leaderClusterTestRunner = tasks.getByName("${taskPrefix}#leader#clusterTestRunner")
4343
leaderClusterTestRunner.configure {
44+
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
4445
systemProperty 'tests.rest.upgrade_state', 'none'
4546
systemProperty 'tests.rest.cluster_name', 'leader'
4647

@@ -71,6 +72,7 @@ for (Version version : bwcVersions.wireCompatible) {
7172

7273
Task followerClusterTestRunner = tasks.getByName("${taskPrefix}#follower#clusterTestRunner")
7374
followerClusterTestRunner.configure {
75+
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
7476
systemProperty 'tests.rest.upgrade_state', 'none'
7577
systemProperty 'tests.rest.cluster_name', 'follower'
7678

@@ -115,6 +117,7 @@ for (Version version : bwcVersions.wireCompatible) {
115117

116118
Task followerOneThirdUpgradedTestRunner = tasks.getByName("${taskPrefix}#follower#oneThirdUpgradedTestRunner")
117119
followerOneThirdUpgradedTestRunner.configure {
120+
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
118121
systemProperty 'tests.rest.upgrade_state', 'one_third'
119122
systemProperty 'tests.rest.cluster_name', 'follower'
120123

@@ -135,6 +138,7 @@ for (Version version : bwcVersions.wireCompatible) {
135138

136139
Task followerTwoThirdsUpgradedTestRunner = tasks.getByName("${taskPrefix}#follower#twoThirdsUpgradedTestRunner")
137140
followerTwoThirdsUpgradedTestRunner.configure {
141+
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
138142
systemProperty 'tests.rest.upgrade_state', 'two_third'
139143
systemProperty 'tests.rest.cluster_name', 'follower'
140144

@@ -155,6 +159,7 @@ for (Version version : bwcVersions.wireCompatible) {
155159

156160
Task followerUpgradedClusterTestRunner = tasks.getByName("${taskPrefix}#follower#upgradedClusterTestRunner")
157161
followerUpgradedClusterTestRunner.configure {
162+
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
158163
systemProperty 'tests.rest.upgrade_state', 'all'
159164
systemProperty 'tests.rest.cluster_name', 'follower'
160165

@@ -181,6 +186,7 @@ for (Version version : bwcVersions.wireCompatible) {
181186

182187
Task leaderOneThirdUpgradedTestRunner = tasks.getByName("${taskPrefix}#leader#oneThirdUpgradedTestRunner")
183188
leaderOneThirdUpgradedTestRunner.configure {
189+
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
184190
systemProperty 'tests.rest.upgrade_state', 'one_third'
185191
systemProperty 'tests.rest.cluster_name', 'leader'
186192

@@ -201,6 +207,7 @@ for (Version version : bwcVersions.wireCompatible) {
201207

202208
Task leaderTwoThirdsUpgradedTestRunner = tasks.getByName("${taskPrefix}#leader#twoThirdsUpgradedTestRunner")
203209
leaderTwoThirdsUpgradedTestRunner.configure {
210+
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
204211
systemProperty 'tests.rest.upgrade_state', 'two_third'
205212
systemProperty 'tests.rest.cluster_name', 'leader'
206213

@@ -221,6 +228,7 @@ for (Version version : bwcVersions.wireCompatible) {
221228

222229
Task leaderUpgradedClusterTestRunner = tasks.getByName("${taskPrefix}#leader#upgradedClusterTestRunner")
223230
leaderUpgradedClusterTestRunner.configure {
231+
systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
224232
systemProperty 'tests.rest.upgrade_state', 'all'
225233
systemProperty 'tests.rest.cluster_name', 'leader'
226234

‎x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/AbstractMultiClusterUpgradeTestCase.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import org.apache.http.HttpHost;
99
import org.apache.http.util.EntityUtils;
10+
import org.elasticsearch.Version;
1011
import org.elasticsearch.client.Request;
1112
import org.elasticsearch.client.Response;
1213
import org.elasticsearch.client.RestClient;
@@ -71,6 +72,9 @@ public static ClusterName parse(String value) {
7172

7273
protected final ClusterName clusterName = ClusterName.parse(System.getProperty("tests.rest.cluster_name"));
7374

75+
protected static final Version UPGRADE_FROM_VERSION =
76+
Version.fromString(System.getProperty("tests.upgrade_from_version"));
77+
7478
private static RestClient leaderClient;
7579
private static RestClient followerClient;
7680
private static boolean initialized = false;

‎x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.elasticsearch.client.RestClient;
1111
import org.elasticsearch.common.Strings;
1212
import org.elasticsearch.common.settings.Settings;
13+
import org.elasticsearch.common.xcontent.ObjectPath;
1314
import org.elasticsearch.common.xcontent.support.XContentMapValues;
1415

1516
import java.io.IOException;
@@ -88,6 +89,128 @@ public void testIndexFollowing() throws Exception {
8889
}
8990
}
9091

92+
public void testAutoFollowing() throws Exception {
93+
final Settings indexSettings = Settings.builder()
94+
.put("index.soft_deletes.enabled", true)
95+
.put("index.number_of_shards", 1)
96+
.build();
97+
98+
String leaderIndex1 = "logs-20200101";
99+
String leaderIndex2 = "logs-20200102";
100+
String leaderIndex3 = "logs-20200103";
101+
102+
if (clusterName == ClusterName.LEADER) {
103+
switch (upgradeState) {
104+
case NONE:
105+
case ONE_THIRD:
106+
case TWO_THIRD:
107+
break;
108+
case ALL:
109+
index(leaderClient(), leaderIndex1, 64);
110+
assertBusy(() -> {
111+
String followerIndex = "copy-" + leaderIndex1;
112+
assertTotalHitCount(followerIndex, 320, followerClient());
113+
});
114+
index(leaderClient(), leaderIndex2, 64);
115+
assertBusy(() -> {
116+
String followerIndex = "copy-" + leaderIndex2;
117+
assertTotalHitCount(followerIndex, 256, followerClient());
118+
});
119+
index(leaderClient(), leaderIndex3, 64);
120+
assertBusy(() -> {
121+
String followerIndex = "copy-" + leaderIndex3;
122+
assertTotalHitCount(followerIndex, 192, followerClient());
123+
});
124+
125+
deleteAutoFollowPattern(followerClient(), "test_pattern");
126+
stopIndexFollowing(followerClient(), "copy-" + leaderIndex1);
127+
stopIndexFollowing(followerClient(), "copy-" + leaderIndex2);
128+
stopIndexFollowing(followerClient(), "copy-" + leaderIndex3);
129+
break;
130+
default:
131+
throw new AssertionError("unexpected upgrade_state [" + upgradeState + "]");
132+
}
133+
} else if (clusterName == ClusterName.FOLLOWER) {
134+
switch (upgradeState) {
135+
case NONE:
136+
putAutoFollowPattern(followerClient(), "test_pattern", "leader", "logs-*");
137+
createIndex(leaderIndex1, indexSettings);
138+
index(leaderClient(), leaderIndex1, 64);
139+
assertBusy(() -> {
140+
String followerIndex = "copy-" + leaderIndex1;
141+
assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(1));
142+
assertTotalHitCount(followerIndex, 64, followerClient());
143+
});
144+
break;
145+
case ONE_THIRD:
146+
index(leaderClient(), leaderIndex1, 64);
147+
assertBusy(() -> {
148+
String followerIndex = "copy-" + leaderIndex1;
149+
assertTotalHitCount(followerIndex, 128, followerClient());
150+
});
151+
// Auto follow stats are kept in-memory on master elected node
152+
// and if this node get updated then auto follow stats are reset
153+
{
154+
int previousNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices();
155+
createIndex(leaderIndex2, indexSettings);
156+
index(leaderClient(), leaderIndex2, 64);
157+
assertBusy(() -> {
158+
String followerIndex = "copy-" + leaderIndex2;
159+
assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(previousNumberOfSuccessfulFollowedIndices + 1));
160+
assertTotalHitCount(followerIndex, 64, followerClient());
161+
});
162+
}
163+
break;
164+
case TWO_THIRD:
165+
index(leaderClient(), leaderIndex1, 64);
166+
assertBusy(() -> {
167+
String followerIndex = "copy-" + leaderIndex1;
168+
assertTotalHitCount(followerIndex, 192, followerClient());
169+
});
170+
index(leaderClient(), leaderIndex2, 64);
171+
assertBusy(() -> {
172+
String followerIndex = "copy-" + leaderIndex2;
173+
assertTotalHitCount(followerIndex, 128, followerClient());
174+
});
175+
176+
// Auto follow stats are kept in-memory on master elected node
177+
// and if this node get updated then auto follow stats are reset
178+
{
179+
int previousNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices();
180+
createIndex(leaderIndex3, indexSettings);
181+
index(leaderClient(), leaderIndex3, 64);
182+
assertBusy(() -> {
183+
String followerIndex = "copy-" + leaderIndex3;
184+
assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(previousNumberOfSuccessfulFollowedIndices + 1));
185+
assertTotalHitCount(followerIndex, 64, followerClient());
186+
});
187+
}
188+
break;
189+
case ALL:
190+
index(leaderClient(), leaderIndex1, 64);
191+
assertBusy(() -> {
192+
String followerIndex = "copy-" + leaderIndex1;
193+
assertTotalHitCount(followerIndex, 256, followerClient());
194+
});
195+
index(leaderClient(), leaderIndex2, 64);
196+
assertBusy(() -> {
197+
String followerIndex = "copy-" + leaderIndex2;
198+
assertTotalHitCount(followerIndex, 192, followerClient());
199+
});
200+
index(leaderClient(), leaderIndex3, 64);
201+
assertBusy(() -> {
202+
String followerIndex = "copy-" + leaderIndex3;
203+
assertTotalHitCount(followerIndex, 128, followerClient());
204+
});
205+
break;
206+
default:
207+
throw new UnsupportedOperationException("unexpected upgrade state [" + upgradeState + "]");
208+
}
209+
} else {
210+
throw new AssertionError("unexpected cluster_name [" + clusterName + "]");
211+
}
212+
}
213+
91214
public void testCannotFollowLeaderInUpgradedCluster() throws Exception {
92215
assumeTrue("Tests only runs with upgrade_state [all]", upgradeState == UpgradeState.ALL);
93216

@@ -134,6 +257,29 @@ private static void followIndex(RestClient client, String leaderCluster, String
134257
assertOK(client.performRequest(request));
135258
}
136259

260+
private static void putAutoFollowPattern(RestClient client, String name, String remoteCluster, String pattern) throws IOException {
261+
Request request = new Request("PUT", "/_ccr/auto_follow/" + name);
262+
request.setJsonEntity("{\"leader_index_patterns\": [\"" + pattern + "\"], \"remote_cluster\": \"" + remoteCluster + "\"," +
263+
"\"follow_index_pattern\": \"copy-{{leader_index}}\", \"read_poll_timeout\": \"10ms\"}");
264+
assertOK(client.performRequest(request));
265+
}
266+
267+
private static void deleteAutoFollowPattern(RestClient client, String patternName) throws IOException {
268+
Request request = new Request("DELETE", "/_ccr/auto_follow/" + patternName);
269+
assertOK(client.performRequest(request));
270+
}
271+
272+
private int getNumberOfSuccessfulFollowedIndices() throws IOException {
273+
Request statsRequest = new Request("GET", "/_ccr/stats");
274+
Map<?, ?> response = toMap(client().performRequest(statsRequest));
275+
Integer actualSuccessfulFollowedIndices = ObjectPath.eval("auto_follow_stats.number_of_successful_follow_indices", response);
276+
if (actualSuccessfulFollowedIndices != null) {
277+
return actualSuccessfulFollowedIndices;
278+
} else {
279+
return -1;
280+
}
281+
}
282+
137283
private static void index(RestClient client, String index, int numDocs) throws IOException {
138284
for (int i = 0; i < numDocs; i++) {
139285
final Request request = new Request("POST", "/" + index + "/_doc/");
@@ -162,4 +308,10 @@ private static void verifyTotalHitCount(final String index,
162308
assertThat(totalHits, equalTo(expectedTotalHits));
163309
}
164310

311+
private static void stopIndexFollowing(RestClient client, String followerIndex) throws IOException {
312+
assertOK(client.performRequest(new Request("POST", "/" + followerIndex + "/_ccr/pause_follow")));
313+
assertOK(client.performRequest(new Request("POST", "/" + followerIndex + "/_close")));
314+
assertOK(client.performRequest(new Request("POST", "/" + followerIndex + "/_ccr/unfollow")));
315+
}
316+
165317
}

0 commit comments

Comments
 (0)