Skip to content

Commit 139bbc3

Browse files
nik9000nik9000
authored
Rollup: Consolidate rollup cleanup for http tests (#34342)
This moves the rollup cleanup code for http tests from the high level rest client into the test framework and then entirely removes the rollup cleanup code for http tests that lived in x-pack. This is nice because it consolidates the cleanup into one spot, automatically invokes the cleanup without the test having to know that it is "about rollup", and should allow us to run the rollup docs tests. Part of #34530
1 parent fb579d2 commit 139bbc3

File tree

10 files changed

+98
-191
lines changed

10 files changed

+98
-191
lines changed

‎client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,7 @@ public void testPutAndGetRollupJob() throws Exception {
208208
}
209209
});
210210

211-
// TODO when we move cleaning rollup into ESTestCase we can randomly choose the _all version of this request
212-
GetRollupJobRequest getRollupJobRequest = new GetRollupJobRequest(id);
211+
GetRollupJobRequest getRollupJobRequest = randomBoolean() ? new GetRollupJobRequest() : new GetRollupJobRequest(id);
213212
GetRollupJobResponse getResponse = execute(getRollupJobRequest, rollupClient::getRollupJob, rollupClient::getRollupJobAsync);
214213
assertThat(getResponse.getJobs(), hasSize(1));
215214
JobWrapper job = getResponse.getJobs().get(0);

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java

Lines changed: 0 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,7 @@
2727
import org.elasticsearch.action.index.IndexRequest;
2828
import org.elasticsearch.action.support.WriteRequest;
2929
import org.elasticsearch.client.ESRestHighLevelClientTestCase;
30-
import org.elasticsearch.client.Request;
3130
import org.elasticsearch.client.RequestOptions;
32-
import org.elasticsearch.client.Response;
3331
import org.elasticsearch.client.RestHighLevelClient;
3432
import org.elasticsearch.client.rollup.GetRollupJobRequest;
3533
import org.elasticsearch.client.rollup.GetRollupJobResponse;
@@ -47,21 +45,15 @@
4745
import org.elasticsearch.client.rollup.job.config.RollupJobConfig;
4846
import org.elasticsearch.client.rollup.job.config.TermsGroupConfig;
4947
import org.elasticsearch.common.unit.TimeValue;
50-
import org.elasticsearch.common.xcontent.support.XContentMapValues;
5148
import org.elasticsearch.rest.RestStatus;
5249
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
53-
import org.junit.After;
5450
import org.junit.Before;
5551

56-
import java.io.BufferedReader;
5752
import java.io.IOException;
58-
import java.io.InputStreamReader;
59-
import java.nio.charset.StandardCharsets;
6053
import java.util.ArrayList;
6154
import java.util.Arrays;
6255
import java.util.List;
6356
import java.util.Locale;
64-
import java.util.Map;
6557
import java.util.concurrent.CountDownLatch;
6658
import java.util.concurrent.TimeUnit;
6759

@@ -227,62 +219,6 @@ public void onFailure(Exception e) {
227219
assertTrue(latch.await(30L, TimeUnit.SECONDS));
228220
}
229221

230-
@After
231-
public void wipeRollup() throws Exception {
232-
// TODO move this to ESRestTestCase
233-
deleteRollupJobs();
234-
waitForPendingRollupTasks();
235-
}
236-
237-
private void deleteRollupJobs() throws Exception {
238-
Response response = adminClient().performRequest(new Request("GET", "/_xpack/rollup/job/_all"));
239-
Map<String, Object> jobs = entityAsMap(response);
240-
@SuppressWarnings("unchecked")
241-
List<Map<String, Object>> jobConfigs =
242-
(List<Map<String, Object>>) XContentMapValues.extractValue("jobs", jobs);
243-
244-
if (jobConfigs == null) {
245-
return;
246-
}
247-
248-
for (Map<String, Object> jobConfig : jobConfigs) {
249-
@SuppressWarnings("unchecked")
250-
String jobId = (String) ((Map<String, Object>) jobConfig.get("config")).get("id");
251-
Request request = new Request("DELETE", "/_xpack/rollup/job/" + jobId);
252-
request.addParameter("ignore", "404"); // Ignore 404s because they imply someone was racing us to delete this
253-
adminClient().performRequest(request);
254-
}
255-
}
256-
257-
private void waitForPendingRollupTasks() throws Exception {
258-
assertBusy(() -> {
259-
try {
260-
Request request = new Request("GET", "/_cat/tasks");
261-
request.addParameter("detailed", "true");
262-
Response response = adminClient().performRequest(request);
263-
264-
try (BufferedReader responseReader = new BufferedReader(
265-
new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) {
266-
int activeTasks = 0;
267-
String line;
268-
StringBuilder tasksListString = new StringBuilder();
269-
while ((line = responseReader.readLine()) != null) {
270-
271-
// We only care about Rollup jobs, otherwise this fails too easily due to unrelated tasks
272-
if (line.startsWith("xpack/rollup/job") == true) {
273-
activeTasks++;
274-
tasksListString.append(line).append('\n');
275-
}
276-
}
277-
assertEquals(activeTasks + " active tasks found:\n" + tasksListString, 0, activeTasks);
278-
}
279-
} catch (IOException e) {
280-
// Throw an assertion error so we retry
281-
throw new AssertionError("Error getting active tasks list", e);
282-
}
283-
});
284-
}
285-
286222
public void testDeleteRollupJob() throws Exception {
287223
RestHighLevelClient client = highLevelClient();
288224

@@ -303,8 +239,6 @@ public void testDeleteRollupJob() throws Exception {
303239
// Swallow any exception, this test does not test actually cancelling.
304240
}
305241

306-
307-
308242
// tag::rollup-delete-job-execute-listener
309243
ActionListener<DeleteRollupJobResponse> listener = new ActionListener<DeleteRollupJobResponse>() {
310244
@Override
@@ -328,7 +262,5 @@ public void onFailure(Exception e) {
328262
// end::rollup-delete-job-execute-async
329263

330264
assertTrue(latch.await(30L, TimeUnit.SECONDS));
331-
332265
}
333-
334266
}

‎test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

Lines changed: 73 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,11 @@
5252
import org.junit.Before;
5353

5454
import javax.net.ssl.SSLContext;
55+
import java.io.BufferedReader;
5556
import java.io.IOException;
5657
import java.io.InputStream;
58+
import java.io.InputStreamReader;
59+
import java.nio.charset.StandardCharsets;
5760
import java.nio.file.Files;
5861
import java.nio.file.Path;
5962
import java.security.KeyManagementException;
@@ -206,8 +209,8 @@ protected static RestClient adminClient() {
206209

207210
/**
208211
* Returns whether to preserve the state of the cluster upon completion of this test. Defaults to false. If true, overrides the value of
209-
* {@link #preserveIndicesUponCompletion()}, {@link #preserveTemplatesUponCompletion()}, {@link #preserveReposUponCompletion()}, and
210-
* {@link #preserveSnapshotsUponCompletion()}.
212+
* {@link #preserveIndicesUponCompletion()}, {@link #preserveTemplatesUponCompletion()}, {@link #preserveReposUponCompletion()},
213+
* {@link #preserveSnapshotsUponCompletion()}, and {@link #preserveRollupJobsUponCompletion()}.
211214
*
212215
* @return true if the state of the cluster should be preserved
213216
*/
@@ -263,7 +266,18 @@ protected boolean preserveSnapshotsUponCompletion() {
263266
return false;
264267
}
265268

266-
private void wipeCluster() throws IOException {
269+
/**
270+
* Returns whether to preserve the rollup jobs of this test. Defaults to
271+
* not preserving them. Only runs at all if xpack is installed on the
272+
* cluster being tested.
273+
*/
274+
protected boolean preserveRollupJobsUponCompletion() {
275+
return false;
276+
}
277+
278+
private void wipeCluster() throws Exception {
279+
boolean hasXPack = hasXPack();
280+
267281
if (preserveIndicesUponCompletion() == false) {
268282
// wipe indices
269283
try {
@@ -278,7 +292,7 @@ private void wipeCluster() throws IOException {
278292

279293
// wipe index templates
280294
if (preserveTemplatesUponCompletion() == false) {
281-
if (hasXPack()) {
295+
if (hasXPack) {
282296
/*
283297
* Delete only templates that xpack doesn't automatically
284298
* recreate. Deleting them doesn't hurt anything, but it
@@ -310,6 +324,11 @@ private void wipeCluster() throws IOException {
310324
if (preserveClusterSettings() == false) {
311325
wipeClusterSettings();
312326
}
327+
328+
if (hasXPack && false == preserveRollupJobsUponCompletion()) {
329+
wipeRollupJobs();
330+
waitForPendingRollupTasks();
331+
}
313332
}
314333

315334
/**
@@ -372,6 +391,56 @@ private void wipeClusterSettings() throws IOException {
372391
}
373392
}
374393

394+
private void wipeRollupJobs() throws IOException {
395+
Response response = adminClient().performRequest(new Request("GET", "/_xpack/rollup/job/_all"));
396+
Map<String, Object> jobs = entityAsMap(response);
397+
@SuppressWarnings("unchecked")
398+
List<Map<String, Object>> jobConfigs =
399+
(List<Map<String, Object>>) XContentMapValues.extractValue("jobs", jobs);
400+
401+
if (jobConfigs == null) {
402+
return;
403+
}
404+
405+
for (Map<String, Object> jobConfig : jobConfigs) {
406+
@SuppressWarnings("unchecked")
407+
String jobId = (String) ((Map<String, Object>) jobConfig.get("config")).get("id");
408+
Request request = new Request("DELETE", "/_xpack/rollup/job/" + jobId);
409+
request.addParameter("ignore", "404"); // Ignore 404s because they imply someone was racing us to delete this
410+
logger.debug("deleting rollup job [{}]", jobId);
411+
adminClient().performRequest(request);
412+
}
413+
}
414+
415+
private void waitForPendingRollupTasks() throws Exception {
416+
assertBusy(() -> {
417+
try {
418+
Request request = new Request("GET", "/_cat/tasks");
419+
request.addParameter("detailed", "true");
420+
Response response = adminClient().performRequest(request);
421+
422+
try (BufferedReader responseReader = new BufferedReader(
423+
new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) {
424+
int activeTasks = 0;
425+
String line;
426+
StringBuilder tasksListString = new StringBuilder();
427+
while ((line = responseReader.readLine()) != null) {
428+
429+
// We only care about Rollup jobs, otherwise this fails too easily due to unrelated tasks
430+
if (line.startsWith("xpack/rollup/job") == true) {
431+
activeTasks++;
432+
tasksListString.append(line).append('\n');
433+
}
434+
}
435+
assertEquals(activeTasks + " active tasks found:\n" + tasksListString, 0, activeTasks);
436+
}
437+
} catch (IOException e) {
438+
// Throw an assertion error so we retry
439+
throw new AssertionError("Error getting active tasks list", e);
440+
}
441+
});
442+
}
443+
375444
/**
376445
* Logs a message if there are still running tasks. The reasoning is that any tasks still running are state the is trying to bleed into
377446
* other tests.

‎test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,4 +62,8 @@ protected boolean preserveClusterSettings() {
6262
return true;
6363
}
6464

65+
@Override
66+
protected boolean preserveRollupJobsUponCompletion() {
67+
return true;
68+
}
6569
}

‎x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -120,12 +120,6 @@ protected boolean isMachineLearningTest() {
120120
return testName != null && (testName.contains("ml/") || testName.contains("ml\\"));
121121
}
122122

123-
@Override
124-
protected boolean isRollupTest() {
125-
String testName = getTestName();
126-
return testName != null && (testName.contains("rollup/") || testName.contains("rollup\\"));
127-
}
128-
129123
/**
130124
* Deletes users after every test just in case any test adds any.
131125
*/

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java

Lines changed: 0 additions & 92 deletions
This file was deleted.

0 commit comments

Comments
 (0)