diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java index b7f85aa3acf10..ce693b7e7c407 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java @@ -208,8 +208,7 @@ public void testPutAndGetRollupJob() throws Exception { } }); - // TODO when we move cleaning rollup into ESTestCase we can randomly choose the _all version of this request - GetRollupJobRequest getRollupJobRequest = new GetRollupJobRequest(id); + GetRollupJobRequest getRollupJobRequest = randomBoolean() ? new GetRollupJobRequest() : new GetRollupJobRequest(id); GetRollupJobResponse getResponse = execute(getRollupJobRequest, rollupClient::getRollupJob, rollupClient::getRollupJobAsync); assertThat(getResponse.getJobs(), hasSize(1)); JobWrapper job = getResponse.getJobs().get(0); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java index 06308fc2117e9..0a5a688c5fb41 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java @@ -27,9 +27,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.ESRestHighLevelClientTestCase; -import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.rollup.GetRollupJobRequest; import org.elasticsearch.client.rollup.GetRollupJobResponse; @@ -47,21 +45,15 @@ import org.elasticsearch.client.rollup.job.config.RollupJobConfig; import org.elasticsearch.client.rollup.job.config.TermsGroupConfig; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.junit.After; import org.junit.Before; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -227,62 +219,6 @@ public void onFailure(Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } - @After - public void wipeRollup() throws Exception { - // TODO move this to ESRestTestCase - deleteRollupJobs(); - waitForPendingRollupTasks(); - } - - private void deleteRollupJobs() throws Exception { - Response response = adminClient().performRequest(new Request("GET", "/_xpack/rollup/job/_all")); - Map jobs = entityAsMap(response); - @SuppressWarnings("unchecked") - List> jobConfigs = - (List>) XContentMapValues.extractValue("jobs", jobs); - - if (jobConfigs == null) { - return; - } - - for (Map jobConfig : jobConfigs) { - @SuppressWarnings("unchecked") - String jobId = (String) ((Map) jobConfig.get("config")).get("id"); - Request request = new Request("DELETE", "/_xpack/rollup/job/" + jobId); - request.addParameter("ignore", "404"); // Ignore 404s because they imply someone was racing us to delete this - adminClient().performRequest(request); - } - } - - private void waitForPendingRollupTasks() throws Exception { - assertBusy(() -> { - try { - Request request = new Request("GET", "/_cat/tasks"); - request.addParameter("detailed", "true"); - Response response = adminClient().performRequest(request); - - try (BufferedReader responseReader = new BufferedReader( - new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) { - int activeTasks = 0; - String line; - StringBuilder tasksListString = new StringBuilder(); - while ((line = responseReader.readLine()) != null) { - - // We only care about Rollup jobs, otherwise this fails too easily due to unrelated tasks - if (line.startsWith("xpack/rollup/job") == true) { - activeTasks++; - tasksListString.append(line).append('\n'); - } - } - assertEquals(activeTasks + " active tasks found:\n" + tasksListString, 0, activeTasks); - } - } catch (IOException e) { - // Throw an assertion error so we retry - throw new AssertionError("Error getting active tasks list", e); - } - }); - } - public void testDeleteRollupJob() throws Exception { RestHighLevelClient client = highLevelClient(); @@ -303,8 +239,6 @@ public void testDeleteRollupJob() throws Exception { // Swallow any exception, this test does not test actually cancelling. } - - // tag::rollup-delete-job-execute-listener ActionListener listener = new ActionListener() { @Override @@ -328,7 +262,5 @@ public void onFailure(Exception e) { // end::rollup-delete-job-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); - } - } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index a5f23104dea67..1c1f2a7eee67f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -52,8 +52,11 @@ import org.junit.Before; import javax.net.ssl.SSLContext; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyManagementException; @@ -206,8 +209,8 @@ protected static RestClient adminClient() { /** * Returns whether to preserve the state of the cluster upon completion of this test. Defaults to false. If true, overrides the value of - * {@link #preserveIndicesUponCompletion()}, {@link #preserveTemplatesUponCompletion()}, {@link #preserveReposUponCompletion()}, and - * {@link #preserveSnapshotsUponCompletion()}. + * {@link #preserveIndicesUponCompletion()}, {@link #preserveTemplatesUponCompletion()}, {@link #preserveReposUponCompletion()}, + * {@link #preserveSnapshotsUponCompletion()}, and {@link #preserveRollupJobsUponCompletion()}. * * @return true if the state of the cluster should be preserved */ @@ -263,7 +266,18 @@ protected boolean preserveSnapshotsUponCompletion() { return false; } - private void wipeCluster() throws IOException { + /** + * Returns whether to preserve the rollup jobs of this test. Defaults to + * not preserving them. Only runs at all if xpack is installed on the + * cluster being tested. + */ + protected boolean preserveRollupJobsUponCompletion() { + return false; + } + + private void wipeCluster() throws Exception { + boolean hasXPack = hasXPack(); + if (preserveIndicesUponCompletion() == false) { // wipe indices try { @@ -278,7 +292,7 @@ private void wipeCluster() throws IOException { // wipe index templates if (preserveTemplatesUponCompletion() == false) { - if (hasXPack()) { + if (hasXPack) { /* * Delete only templates that xpack doesn't automatically * recreate. Deleting them doesn't hurt anything, but it @@ -310,6 +324,11 @@ private void wipeCluster() throws IOException { if (preserveClusterSettings() == false) { wipeClusterSettings(); } + + if (hasXPack && false == preserveRollupJobsUponCompletion()) { + wipeRollupJobs(); + waitForPendingRollupTasks(); + } } /** @@ -372,6 +391,56 @@ private void wipeClusterSettings() throws IOException { } } + private void wipeRollupJobs() throws IOException { + Response response = adminClient().performRequest(new Request("GET", "/_xpack/rollup/job/_all")); + Map jobs = entityAsMap(response); + @SuppressWarnings("unchecked") + List> jobConfigs = + (List>) XContentMapValues.extractValue("jobs", jobs); + + if (jobConfigs == null) { + return; + } + + for (Map jobConfig : jobConfigs) { + @SuppressWarnings("unchecked") + String jobId = (String) ((Map) jobConfig.get("config")).get("id"); + Request request = new Request("DELETE", "/_xpack/rollup/job/" + jobId); + request.addParameter("ignore", "404"); // Ignore 404s because they imply someone was racing us to delete this + logger.debug("deleting rollup job [{}]", jobId); + adminClient().performRequest(request); + } + } + + private void waitForPendingRollupTasks() throws Exception { + assertBusy(() -> { + try { + Request request = new Request("GET", "/_cat/tasks"); + request.addParameter("detailed", "true"); + Response response = adminClient().performRequest(request); + + try (BufferedReader responseReader = new BufferedReader( + new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) { + int activeTasks = 0; + String line; + StringBuilder tasksListString = new StringBuilder(); + while ((line = responseReader.readLine()) != null) { + + // We only care about Rollup jobs, otherwise this fails too easily due to unrelated tasks + if (line.startsWith("xpack/rollup/job") == true) { + activeTasks++; + tasksListString.append(line).append('\n'); + } + } + assertEquals(activeTasks + " active tasks found:\n" + tasksListString, 0, activeTasks); + } + } catch (IOException e) { + // Throw an assertion error so we retry + throw new AssertionError("Error getting active tasks list", e); + } + }); + } + /** * Logs a message if there are still running tasks. The reasoning is that any tasks still running are state the is trying to bleed into * other tests. diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java index 7e73e795b8a05..861d574b3465e 100644 --- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java @@ -62,4 +62,8 @@ protected boolean preserveClusterSettings() { return true; } + @Override + protected boolean preserveRollupJobsUponCompletion() { + return true; + } } diff --git a/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java b/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java index 366639d2482d5..b5d66dc2031fc 100644 --- a/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java +++ b/x-pack/docs/src/test/java/org/elasticsearch/smoketest/XDocsClientYamlTestSuiteIT.java @@ -120,12 +120,6 @@ protected boolean isMachineLearningTest() { return testName != null && (testName.contains("ml/") || testName.contains("ml\\")); } - @Override - protected boolean isRollupTest() { - String testName = getTestName(); - return testName != null && (testName.contains("rollup/") || testName.contains("rollup\\")); - } - /** * Deletes users after every test just in case any test adds any. */ diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java deleted file mode 100644 index 3eb9aa79b4bc8..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/RollupRestTestStateCleaner.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.rollup; - -import org.apache.http.HttpStatus; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.xpack.core.rollup.job.RollupJob; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; - -public class RollupRestTestStateCleaner { - - public static void clearRollupMetadata(RestClient adminClient) throws Exception { - deleteAllJobs(adminClient); - waitForPendingTasks(adminClient); - // indices will be deleted by the ESRestTestCase class - } - - private static void waitForPendingTasks(RestClient adminClient) throws Exception { - ESTestCase.assertBusy(() -> { - try { - Request request = new Request("GET", "/_cat/tasks"); - request.addParameter("detailed", "true"); - Response response = adminClient.performRequest(request); - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - try (BufferedReader responseReader = new BufferedReader( - new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) { - int activeTasks = 0; - String line; - StringBuilder tasksListString = new StringBuilder(); - while ((line = responseReader.readLine()) != null) { - - // We only care about Rollup jobs, otherwise this fails too easily due to unrelated tasks - if (line.startsWith(RollupJob.NAME) == true) { - activeTasks++; - tasksListString.append(line); - tasksListString.append('\n'); - } - } - assertEquals(activeTasks + " active tasks found:\n" + tasksListString, 0, activeTasks); - } - } - } catch (IOException e) { - throw new AssertionError("Error getting active tasks list", e); - } - }); - } - - @SuppressWarnings("unchecked") - private static void deleteAllJobs(RestClient adminClient) throws Exception { - Response response = adminClient.performRequest(new Request("GET", "/_xpack/rollup/job/_all")); - Map jobs = ESRestTestCase.entityAsMap(response); - List> jobConfigs = - (List>) XContentMapValues.extractValue("jobs", jobs); - - if (jobConfigs == null) { - return; - } - - for (Map jobConfig : jobConfigs) { - String jobId = (String) ((Map) jobConfig.get("config")).get("id"); - try { - response = adminClient.performRequest(new Request("DELETE", "/_xpack/rollup/job/" + jobId)); - } catch (Exception e) { - // ok - } - } - } - - private static String responseEntityToString(Response response) throws Exception { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), - StandardCharsets.UTF_8))) { - return reader.lines().collect(Collectors.joining("\n")); - } - } -} diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index b020109ca298b..5baacf69c8da3 100644 --- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.notifications.AuditorField; -import org.elasticsearch.xpack.core.rollup.RollupRestTestStateCleaner; +import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField; import org.junit.After; import org.junit.Before; @@ -243,11 +243,13 @@ private void disableMonitoring() throws Exception { public void cleanup() throws Exception { disableMonitoring(); clearMlState(); - clearRollupState(); if (isWaitForPendingTasks()) { // This waits for pending tasks to complete, so must go last (otherwise // it could be waiting for pending tasks while monitoring is still running). - XPackRestTestHelper.waitForPendingTasks(adminClient()); + XPackRestTestHelper.waitForPendingTasks(adminClient(), task -> { + // Don't check rollup jobs because we clear them in the superclass. + return task.contains(RollupJob.NAME); + }); } } @@ -260,17 +262,6 @@ private void clearMlState() throws Exception { } } - /** - * Delete any left over rollup jobs - * - * Also reuses the pending-task logic from Ml... should refactor to shared location - */ - private void clearRollupState() throws Exception { - if (isRollupTest()) { - RollupRestTestStateCleaner.clearRollupMetadata(adminClient()); - } - } - /** * Executes an API call using the admin context, waiting for it to succeed. */ @@ -331,11 +322,6 @@ protected boolean isMachineLearningTest() { return testName != null && (testName.contains("=ml/") || testName.contains("=ml\\")); } - protected boolean isRollupTest() { - String testName = getTestName(); - return testName != null && (testName.contains("=rollup/") || testName.contains("=rollup\\")); - } - /** * Should each test wait for pending tasks to finish after execution? * @return Wait for pending tasks diff --git a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java index ebe7530e732c5..b9f8481d0bb97 100644 --- a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java +++ b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java @@ -24,6 +24,11 @@ protected boolean preserveTemplatesUponCompletion() { return true; } + @Override + protected boolean preserveRollupJobsUponCompletion() { + return true; + } + enum CLUSTER_TYPE { OLD, MIXED, diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java index ec912a68b59c2..a1430965339c3 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java @@ -38,6 +38,11 @@ protected boolean preserveTemplatesUponCompletion() { return true; } + @Override + protected boolean preserveRollupJobsUponCompletion() { + return true; + } + enum ClusterType { OLD, MIXED, diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index cb0cc182b86ab..bd1b4ec28feee 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -42,6 +42,11 @@ protected boolean preserveTemplatesUponCompletion() { return true; } + @Override + protected boolean preserveRollupJobsUponCompletion() { + return true; + } + public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { super(testCandidate); }