From 4f960c2a4332cc554a96c78b42718698445a7162 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 13 Jan 2021 14:34:23 +0000 Subject: [PATCH] Extend the selective muting of memory tests on Debian 8 The selective muting implemented for autoscaling in #67159 is extended to the ML tests that also fail when machine memory is reported as 0. Most of the logic to determine when memory will not be accurately reported is now in a utility method in the base class. Relates #66885 Backport of #67422 --- .../elasticsearch/test/ESIntegTestCase.java | 20 +++++++++++++++++++ .../integration/BasicDistributedJobsIT.java | 4 ++++ .../integration/MlDistributedFailureIT.java | 3 +++ .../xpack/ml/integration/TooManyJobsIT.java | 4 ++++ 4 files changed, 31 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index ae1b9049b3a7c..ced0bc8003e7a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -57,6 +57,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.client.ClusterAdminClient; @@ -129,6 +130,7 @@ import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.ingest.IngestMetadata; +import org.elasticsearch.monitor.os.OsInfo; import org.elasticsearch.node.NodeMocksPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; @@ -2392,4 +2394,22 @@ public static String resolveCustomDataPath(String index) { public static boolean inFipsJvm() { return Boolean.parseBoolean(System.getProperty(FIPS_SYSPROP)); } + + /** + * On Debian 8 the "memory" subsystem is not mounted by default + * when cgroups are enabled, and this confuses many versions of + * Java prior to Java 15. Tests that rely on machine memory + * being accurately determined will not work on such setups, + * and can use this method for selective muting. + * See https://github.com/elastic/elasticsearch/issues/67089 + * and https://github.com/elastic/elasticsearch/issues/66885 + */ + protected boolean willSufferDebian8MemoryProblem() { + final NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet(); + final boolean anyDebian8Nodes = response.getNodes() + .stream() + .anyMatch(ni -> ni.getInfo(OsInfo.class).getPrettyName().equals("Debian GNU/Linux 8 (jessie)")); + boolean java15Plus = JavaVersion.current().compareTo(JavaVersion.parse("15")) >= 0; + return anyDebian8Nodes && java15Plus == false; + } } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index 5619c3b6890d6..c735a91395c99 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -423,6 +423,10 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception { } public void testCloseUnassignedLazyJobAndDatafeed() throws Exception { + + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); + internalCluster().ensureAtLeastNumDataNodes(3); ensureStableCluster(3); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index dc0af77dc8a7f..5b1c1b520515b 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -373,6 +373,9 @@ public void testStopAndForceStopDatafeed() throws Exception { public void testJobRelocationIsMemoryAware() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); + internalCluster().ensureAtLeastNumDataNodes(1); ensureStableCluster(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index 569aee4c26154..044cb712f4937 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -126,10 +126,14 @@ public void testLazyNodeValidation() throws Exception { } public void testSingleNode() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); verifyMaxNumberOfJobsLimit(1, randomIntBetween(1, 20), randomBoolean()); } public void testMultipleNodes() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); verifyMaxNumberOfJobsLimit(3, randomIntBetween(1, 20), randomBoolean()); }