Introduce aarch64 packaging (#53914)

This commit introduces aarch64 packaging, including bundling an aarch64
JDK distribution. We had to make some interesting choices here:
 - ML binaries are not compiled for aarch64, so for now we disable ML on
   aarch64
 - depending on underlying page sizes, we have to disable class data
   sharing

Author: jasontedor

buildSrc/build.gradle

@@ -187,8 +187,10 @@ if (project != rootProject) {
     distribution project(':distribution:archives:oss-windows-zip')
     distribution project(':distribution:archives:darwin-tar')
     distribution project(':distribution:archives:oss-darwin-tar')
+    distribution project(':distribution:archives:linux-aarch64-tar')
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:oss-linux-tar')
+    distribution project(':distribution:archives:oss-linux-aarch64-tar')
   }
 
   // for external projects we want to remove the marker file indicating we are running the Elasticsearch project

buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+public enum Architecture {
+
+    X64,
+    AARCH64;
+
+    public static Architecture current() {
+        final String architecture = System.getProperty("os.arch", "");
+        switch (architecture) {
+            case "amd64":
+            case "x86_64":
+                return X64;
+            case "aarch64":
+                return AARCH64;
+            default:
+                throw new IllegalArgumentException("can not determine architecture from [" + architecture + "]");
+        }
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

@@ -33,6 +33,7 @@
 
 public class Jdk implements Buildable, Iterable<File> {
 
+    private static final List<String> ALLOWED_ARCHITECTURES = List.of("aarch64", "x64");
     private static final List<String> ALLOWED_VENDORS = List.of("adoptopenjdk", "openjdk");
     private static final List<String> ALLOWED_PLATFORMS = List.of("darwin", "linux", "windows", "mac");
     private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
@@ -44,6 +45,7 @@ public class Jdk implements Buildable, Iterable<File> {
     private final Property<String> vendor;
     private final Property<String> version;
     private final Property<String> platform;
+    private final Property<String> architecture;
     private String baseVersion;
     private String major;
     private String build;
@@ -55,6 +57,7 @@ public class Jdk implements Buildable, Iterable<File> {
         this.vendor = objectFactory.property(String.class);
         this.version = objectFactory.property(String.class);
         this.platform = objectFactory.property(String.class);
+        this.architecture = objectFactory.property(String.class);
     }
 
     public String getName() {
@@ -97,6 +100,19 @@ public void setPlatform(String platform) {
         this.platform.set(platform);
     }
 
+    public String getArchitecture() {
+        return architecture.get();
+    }
+
+    public void setArchitecture(final String architecture) {
+        if (ALLOWED_ARCHITECTURES.contains(architecture) == false) {
+            throw new IllegalArgumentException(
+                "unknown architecture [" + architecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
+            );
+        }
+        this.architecture.set(architecture);
+    }
+
     public String getBaseVersion() {
         return baseVersion;
     }
@@ -153,9 +169,13 @@ void finalizeValues() {
         if (vendor.isPresent() == false) {
             throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]");
         }
+        if (architecture.isPresent() == false) {
+            throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]");
+        }
         version.finalizeValue();
         platform.finalizeValue();
         vendor.finalizeValue();
+        architecture.finalizeValue();
     }
 
     @Override

buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

@@ -47,7 +47,6 @@
 import java.util.concurrent.Callable;
 import java.util.stream.StreamSupport;
 
-import static org.elasticsearch.gradle.Util.capitalize;
 import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
 
@@ -72,7 +71,10 @@ public void apply(Project project) {
                 DependencyHandler dependencies = project.getDependencies();
                 Map<String, Object> depConfig = new HashMap<>();
                 depConfig.put("path", ":"); // root project
-                depConfig.put("configuration", configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform()));
+                depConfig.put(
+                    "configuration",
+                    configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture())
+                );
                 project.getDependencies().add(jdk.getConfigurationName(), dependencies.project(depConfig));
 
                 // ensure a root level jdk download task exists
@@ -87,7 +89,14 @@ public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
     }
 
     private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
-        String extractTaskName = "extract" + capitalize(jdk.getPlatform()) + "Jdk-" + jdk.getVendor() + "-" + jdk.getVersion();
+        String extractTaskName = String.format(
+            Locale.ROOT,
+            "extract-%s-%s-jdk-%s-%s",
+            jdk.getPlatform(),
+            jdk.getArchitecture(),
+            jdk.getVendor(),
+            jdk.getVersion()
+        );
 
         // Skip setup if we've already configured a JDK for this platform, vendor and version
         if (findByName(rootProject.getTasks(), extractTaskName) == null) {
@@ -107,7 +116,7 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
                 repoUrl = "https://artifactory.elstc.co/artifactory/oss-jdk-local/";
                 artifactPattern = String.format(
                     Locale.ROOT,
-                    "adoptopenjdk/OpenJDK%sU-jdk_x64_[module]_hotspot_[revision]_%s.[ext]",
+                    "adoptopenjdk/OpenJDK%sU-jdk_[classifier]_[module]_hotspot_[revision]_%s.[ext]",
                     jdk.getMajor(),
                     jdk.getBuild()
                 );
@@ -121,14 +130,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
                         + jdk.getHash()
                         + "/"
                         + jdk.getBuild()
-                        + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
                 } else {
                     // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
                     artifactPattern = "java/GA/jdk"
                         + jdk.getMajor()
                         + "/"
                         + jdk.getBuild()
-                        + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
                 }
             } else {
                 throw new GradleException("Unknown JDK vendor [" + jdk.getVendor() + "]");
@@ -150,14 +159,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
 
             // Declare a configuration and dependency from which to download the remote JDK
             final ConfigurationContainer configurations = rootProject.getConfigurations();
-            String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
+            String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture());
             Configuration downloadConfiguration = maybeCreate(configurations, downloadConfigName);
             rootProject.getDependencies().add(downloadConfigName, dependencyNotation(jdk));
 
             // Create JDK extract task
             final Provider<Directory> extractPath = rootProject.getLayout()
                 .getBuildDirectory()
-                .dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform());
+                .dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform() + "_" + jdk.getArchitecture());
 
             TaskProvider<?> extractTask = createExtractTask(
                 extractTaskName,
@@ -168,7 +177,13 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
             );
 
             // Declare a configuration for the extracted JDK archive
-            String artifactConfigName = configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
+            String artifactConfigName = configName(
+                "extracted_jdk",
+                jdk.getVendor(),
+                jdk.getVersion(),
+                jdk.getPlatform(),
+                jdk.getArchitecture()
+            );
             maybeCreate(configurations, artifactConfigName);
             rootProject.getArtifacts().add(artifactConfigName, extractPath, artifact -> artifact.builtBy(extractTask));
         }
@@ -254,7 +269,7 @@ private static String dependencyNotation(Jdk jdk) {
             : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
 
-        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
+        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + ":" + jdk.getArchitecture() + "@" + extension;
     }
 
     private static String groupName(Jdk jdk) {

buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -175,12 +175,14 @@ private static Jdk createJdk(
         String name,
         String vendor,
         String version,
-        String platform
+        String platform,
+        String architecture
     ) {
         Jdk jdk = jdksContainer.create(name);
         jdk.setVendor(vendor);
         jdk.setVersion(version);
         jdk.setPlatform(platform);
+        jdk.setArchitecture(architecture);
         return jdk;
     }
 
@@ -212,8 +214,8 @@ private static List<Object> configureVM(Project project) {
 
         NamedDomainObjectContainer<Jdk> jdksContainer = JdkDownloadPlugin.getContainer(project);
         String platform = box.contains("windows") ? "windows" : "linux";
-        Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VENDOR, SYSTEM_JDK_VERSION, platform);
-        Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform);
+        Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VENDOR, SYSTEM_JDK_VERSION, platform, "x64");
+        Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform, "x64");
 
         // setup VM used by these tests
         VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class);

buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java

@@ -36,7 +36,7 @@ public static void setupRoot() {
     }
 
     public void testMissingVendor() {
-        assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "vendor not specified for jdk [testjdk]");
+        assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "x64", "vendor not specified for jdk [testjdk]");
     }
 
     public void testUnknownVendor() {
@@ -46,20 +46,29 @@ public void testUnknownVendor() {
             "unknown",
             "11.0.2+33",
             "linux",
+            "x64",
             "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]"
         );
     }
 
     public void testMissingVersion() {
-        assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "version not specified for jdk [testjdk]");
+        assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "x64", "version not specified for jdk [testjdk]");
     }
 
     public void testBadVersionFormat() {
-        assertJdkError(createProject(), "testjdk", "openjdk", "badversion", "linux", "malformed version [badversion] for jdk [testjdk]");
+        assertJdkError(
+            createProject(),
+            "testjdk",
+            "openjdk",
+            "badversion",
+            "linux",
+            "x64",
+            "malformed version [badversion] for jdk [testjdk]"
+        );
     }
 
     public void testMissingPlatform() {
-        assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "platform not specified for jdk [testjdk]");
+        assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "x64", "platform not specified for jdk [testjdk]");
     }
 
     public void testUnknownPlatform() {
@@ -69,19 +78,44 @@ public void testUnknownPlatform() {
             "openjdk",
             "11.0.2+33",
             "unknown",
+            "x64",
             "unknown platform [unknown] for jdk [testjdk], must be one of [darwin, linux, windows, mac]"
         );
     }
 
-    private void assertJdkError(Project project, String name, String vendor, String version, String platform, String message) {
+    public void testMissingArchitecture() {
+        assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", "linux", null, "architecture not specified for jdk [testjdk]");
+    }
+
+    public void testUnknownArchitecture() {
+        assertJdkError(
+            createProject(),
+            "testjdk",
+            "openjdk",
+            "11.0.2+33",
+            "linux",
+            "unknown",
+            "unknown architecture [unknown] for jdk [testjdk], must be one of [aarch64, x64]"
+        );
+    }
+
+    private void assertJdkError(
+        final Project project,
+        final String name,
+        final String vendor,
+        final String version,
+        final String platform,
+        final String architecture,
+        final String message
+    ) {
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> createJdk(project, name, vendor, version, platform)
+            () -> createJdk(project, name, vendor, version, platform, architecture)
         );
         assertThat(e.getMessage(), equalTo(message));
     }
 
-    private void createJdk(Project project, String name, String vendor, String version, String platform) {
+    private void createJdk(Project project, String name, String vendor, String version, String platform, String architecture) {
         @SuppressWarnings("unchecked")
         NamedDomainObjectContainer<Jdk> jdks = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
         jdks.create(name, jdk -> {
@@ -94,6 +128,9 @@ private void createJdk(Project project, String name, String vendor, String versi
             if (platform != null) {
                 jdk.setPlatform(platform);
             }
+            if (architecture != null) {
+                jdk.setArchitecture(architecture);
+            }
         }).finalizeValues();
     }
 

buildSrc/src/testKit/jdk-download/reuse/build.gradle

@@ -7,5 +7,6 @@ jdks {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "linux"
+    architeecture = "x64"
   }
 }

buildSrc/src/testKit/jdk-download/subproj/build.gradle

@@ -10,16 +10,19 @@ jdks {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "linux"
+    architecture = "x64"
   }
   darwin {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "darwin"
+    architecture = "x64"
   }
   windows {
     vendor = fakeJdkVendor
     version = fakeJdkVersion
     platform = "windows"
+    architecture = "x64"
   }
 }