Skip ML tests on later glibc for incompatible BWC versions (elastic#87476) (elastic#87484)

Author: mark-vieira

.ci/scripts/run-gradle.sh

@@ -19,5 +19,9 @@ if pwd | grep -v -q ^/dev/shm ; then
    echo "Not running on a ramdisk, reducing number of workers"
    MAX_WORKERS=$(($MAX_WORKERS*2/3))
 fi
+
+# Export glibc version as environment variable since some BWC tests are incompatible with later versions
+export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
+
 set -e
 $GRADLEW -S --max-workers=$MAX_WORKERS $@

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java

@@ -18,6 +18,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -67,6 +68,7 @@ public class BwcVersions {
         "\\W+public static final Version V_(\\d+)_(\\d+)_(\\d+)(_alpha\\d+|_beta\\d+|_rc\\d+)? .*?LUCENE_(\\d+)_(\\d+)_(\\d+)\\);"
     );
     private static final Version MINIMUM_WIRE_COMPATIBLE_VERSION = Version.fromString("7.17.0");
+    private static final String GLIBC_VERSION_ENV_VAR = "GLIBC_VERSION";
 
     private final VersionPair currentVersion;
     private final List<VersionPair> versions;
@@ -308,4 +310,26 @@ public int compareTo(VersionPair o) {
             return this.elasticsearch.compareTo(o.elasticsearch);
         }
     }
+
+    /**
+     * Determine whether the given version of Elasticsearch is compatible with ML features on the host system.
+     *
+     * @see <a href="https://github.com/elastic/elasticsearch/issues/86877">https://github.com/elastic/elasticsearch/issues/86877</a>
+     */
+    public static boolean isMlCompatible(Version version) {
+        Version glibcVersion = Optional.ofNullable(System.getenv(GLIBC_VERSION_ENV_VAR))
+            .map(v -> Version.fromString(v, Version.Mode.RELAXED))
+            .orElse(null);
+
+        // glibc version 2.35 introduced incompatibilities in ML syscall filters that were fixed in 7.17.5+ and 8.2.2+
+        if (glibcVersion != null && glibcVersion.onOrAfter(Version.fromString("2.35", Version.Mode.RELAXED))) {
+            if (version.before(Version.fromString("7.17.5"))) {
+                return false;
+            } else if (version.getMajor() > 7 && version.before(Version.fromString("8.2.2"))) {
+                return false;
+            }
+        }
+
+        return true;
+    }
 }

build-tools/src/main/java/org/elasticsearch/gradle/Version.java

@@ -41,7 +41,7 @@ public enum Mode {
     private static final Pattern pattern = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)(?:-(alpha\\d+|beta\\d+|rc\\d+|SNAPSHOT))?");
 
     private static final Pattern relaxedPattern = Pattern.compile(
-        "v?(\\d+)\\.(\\d+)\\.(\\d+)(?:[\\-+]+([a-zA-Z0-9_]+(?:-[a-zA-Z0-9]+)*))?"
+        "v?(\\d+)\\.(\\d+)(?:\\.(\\d+))?(?:[\\-+]+([a-zA-Z0-9_]+(?:-[a-zA-Z0-9]+)*))?"
     );
 
     public Version(int major, int minor, int revision) {
@@ -73,14 +73,12 @@ public static Version fromString(final String s, final Mode mode) {
             throw new IllegalArgumentException("Invalid version format: '" + s + "'. Should be " + expected);
         }
 
+        String major = matcher.group(1);
+        String minor = matcher.group(2);
+        String revision = matcher.group(3);
         String qualifier = matcher.group(4);
 
-        return new Version(
-            Integer.parseInt(matcher.group(1)),
-            Integer.parseInt(matcher.group(2)),
-            Integer.parseInt(matcher.group(3)),
-            qualifier
-        );
+        return new Version(Integer.parseInt(major), Integer.parseInt(minor), revision == null ? 0 : Integer.parseInt(revision), qualifier);
     }
 
     @Override

build-tools/src/test/java/org/elasticsearch/gradle/VersionTests.java

@@ -42,6 +42,7 @@ public void testRelaxedVersionParsing() {
         assertVersionEquals("6.1.2-foo-bar", 6, 1, 2, Version.Mode.RELAXED);
         assertVersionEquals("16.01.22", 16, 1, 22, Version.Mode.RELAXED);
         assertVersionEquals("20.10.10+dfsg1", 20, 10, 10, Version.Mode.RELAXED);
+        assertVersionEquals("2.15", 2, 15, 0, Version.Mode.RELAXED);
     }
 
     public void testCompareWithStringVersions() {

x-pack/plugin/build.gradle

@@ -60,15 +60,13 @@ tasks.named("processYamlRestTestResources").configure {
   dependsOn("copyExtraResources")
 }
 def restTestBlacklist = []
-tasks.withType(RestIntegTestTask).configureEach {
-  // TODO: fix this rest test to not depend on a hardcoded port!
-  restTestBlacklist.addAll(['getting_started/10_monitor_cluster_health/*'])
-  if (BuildParams.isSnapshotBuild() == false) {
-    // these tests attempt to install basic/internal licenses signed against the dev/public.key
-    // Since there is no infrastructure in place (anytime soon) to generate licenses using the production
-    // private key, these tests are blacklisted in non-snapshot test runs
-    restTestBlacklist.addAll(['xpack/15_basic/*', 'license/20_put_license/*', 'license/30_enterprise_license/*'])
-  }
+// TODO: fix this rest test to not depend on a hardcoded port!
+restTestBlacklist.addAll(['getting_started/10_monitor_cluster_health/*'])
+if (BuildParams.isSnapshotBuild() == false) {
+  // these tests attempt to install basic/internal licenses signed against the dev/public.key
+  // Since there is no infrastructure in place (anytime soon) to generate licenses using the production
+  // private key, these tests are blacklisted in non-snapshot test runs
+  restTestBlacklist.addAll(['xpack/15_basic/*', 'license/20_put_license/*', 'license/30_enterprise_license/*'])
 }
 
 tasks.named("yamlRestTest").configure {

x-pack/qa/rolling-upgrade/build.gradle

@@ -1,5 +1,5 @@
 import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.internal.BwcVersions
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
@@ -121,6 +121,12 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     systemProperty 'tests.path.searchable.snapshots.repo', searchableSnapshotRepository
     nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c->c.allHttpSocketURI.join(",")))
     nonInputProperties.systemProperty('tests.clustername', baseName)
+
+    // Disable ML tests for incompatible systems
+    if (BwcVersions.isMlCompatible(bwcVersion) == false) {
+      exclude '**/MlJobSnapshotUpgradeIT.class', '**/MlMappingsUpgradeIT.class', '**/MLModelDeploymentsUpgradeIT.class', '**/MlTrainedModelsUpgradeIT.class'
+      systemProperty 'tests.rest.blacklist', ['old_cluster/30_ml_jobs_crud/*', 'old_cluster/40_ml_datafeed_crud/*', 'old_cluster/90_ml_data_frame_analytics_crud'].join(',')
+    }
   }
 
   tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) {
@@ -136,7 +142,7 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     systemProperty 'tests.upgrade_from_version', oldVersion
     systemProperty 'tests.path.searchable.snapshots.repo', searchableSnapshotRepository
     // We only need to run these tests once so we may as well do it when we're two thirds upgraded
-    systemProperty 'tests.rest.blacklist', [
+    def excludeList = [
       'mixed_cluster/10_basic/Start scroll in mixed cluster on upgraded node that we will continue after upgrade',
       'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data',
       'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed in mixed cluster',
@@ -149,7 +155,14 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
       'mixed_cluster/120_api_key/Test API key authentication will work in a mixed cluster',
       'mixed_cluster/120_api_key/Create API key with metadata in a mixed cluster',
       'mixed_cluster/130_operator_privileges/Test operator privileges will work in the mixed cluster'
-    ].join(',')
+    ]
+
+    // Disable ML tests for incompatible systems
+    if (BwcVersions.isMlCompatible(bwcVersion) == false) {
+      exclude '**/MlJobSnapshotUpgradeIT.class', '**/MlMappingsUpgradeIT.class', '**/MLModelDeploymentsUpgradeIT.class', '**/MlTrainedModelsUpgradeIT.class'
+      excludeList.addAll(['mixed_cluster/30_ml_jobs_crud/*', 'mixed_cluster/40_ml_datafeed_crud/*', 'mixed_cluster/90_ml_data_frame_analytics_crud'])
+    }
+    systemProperty 'tests.rest.blacklist', excludeList.join(',')
   }
 
   tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) {
@@ -164,6 +177,12 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     systemProperty 'tests.first_round', 'false'
     systemProperty 'tests.upgrade_from_version', oldVersion
     systemProperty 'tests.path.searchable.snapshots.repo', searchableSnapshotRepository
+
+    // Disable ML tests for incompatible systems
+    if (BwcVersions.isMlCompatible(bwcVersion) == false) {
+      exclude '**/MlJobSnapshotUpgradeIT.class', '**/MlMappingsUpgradeIT.class', '**/MLModelDeploymentsUpgradeIT.class', '**/MlTrainedModelsUpgradeIT.class'
+      systemProperty 'tests.rest.blacklist', ['mixed_cluster/30_ml_jobs_crud/*', 'mixed_cluster/40_ml_datafeed_crud/*', 'mixed_cluster/90_ml_data_frame_analytics_crud'].join(',')
+    }
   }
 
   tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
@@ -177,6 +196,12 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     systemProperty 'tests.rest.suite', 'upgraded_cluster'
     systemProperty 'tests.upgrade_from_version', oldVersion
     systemProperty 'tests.path.searchable.snapshots.repo', searchableSnapshotRepository
+
+    // Disable ML tests for incompatible systems
+    if (BwcVersions.isMlCompatible(bwcVersion) == false) {
+      exclude '**/MlJobSnapshotUpgradeIT.class', '**/MlMappingsUpgradeIT.class', '**/MLModelDeploymentsUpgradeIT.class', '**/MlTrainedModelsUpgradeIT.class'
+      systemProperty 'tests.rest.blacklist', ['upgraded_cluster/30_ml_jobs_crud/*', 'upgraded_cluster/40_ml_datafeed_crud/*', 'upgraded_cluster/90_ml_data_frame_analytics_crud'].join(',')
+    }
   }
 
   tasks.register(bwcTaskName(bwcVersion)) {