Skip to content

Commit 27c8bcb

Browse files
jasontedorjasontedor
authored
Introduce aarch64 packaging (#53914) (#53926)
This commit introduces aarch64 packaging, including bundling an aarch64 JDK distribution. We had to make some interesting choices here: - ML binaries are not compiled for aarch64, so for now we disable ML on aarch64 - depending on underlying page sizes, we have to disable class data sharing
1 parent 076ba02 commit 27c8bcb

File tree

26 files changed

+258
-62
lines changed

26 files changed

+258
-62
lines changed

‎buildSrc/build.gradle

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,8 +190,10 @@ if (project != rootProject) {
190190
distribution project(':distribution:archives:oss-windows-zip')
191191
distribution project(':distribution:archives:darwin-tar')
192192
distribution project(':distribution:archives:oss-darwin-tar')
193+
distribution project(':distribution:archives:linux-aarch64-tar')
193194
distribution project(':distribution:archives:linux-tar')
194195
distribution project(':distribution:archives:oss-linux-tar')
196+
distribution project(':distribution:archives:oss-linux-aarch64-tar')
195197
}
196198

197199
// for external projects we want to remove the marker file indicating we are running the Elasticsearch project

‎buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.gradle;
21+
22+
public enum Architecture {
23+
24+
X64,
25+
AARCH64;
26+
27+
public static Architecture current() {
28+
final String architecture = System.getProperty("os.arch", "");
29+
switch (architecture) {
30+
case "amd64":
31+
case "x86_64":
32+
return X64;
33+
case "aarch64":
34+
return AARCH64;
35+
default:
36+
throw new IllegalArgumentException("can not determine architecture from [" + architecture + "]");
37+
}
38+
}
39+
40+
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535

3636
public class Jdk implements Buildable, Iterable<File> {
3737

38+
private static final List<String> ALLOWED_ARCHITECTURES = Collections.unmodifiableList(Arrays.asList("aarch64", "x64"));
3839
private static final List<String> ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptopenjdk", "openjdk"));
3940
private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows", "mac"));
4041
private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
@@ -46,6 +47,7 @@ public class Jdk implements Buildable, Iterable<File> {
4647
private final Property<String> vendor;
4748
private final Property<String> version;
4849
private final Property<String> platform;
50+
private final Property<String> architecture;
4951
private String baseVersion;
5052
private String major;
5153
private String build;
@@ -57,6 +59,7 @@ public class Jdk implements Buildable, Iterable<File> {
5759
this.vendor = objectFactory.property(String.class);
5860
this.version = objectFactory.property(String.class);
5961
this.platform = objectFactory.property(String.class);
62+
this.architecture = objectFactory.property(String.class);
6063
}
6164

6265
public String getName() {
@@ -99,6 +102,19 @@ public void setPlatform(String platform) {
99102
this.platform.set(platform);
100103
}
101104

105+
public String getArchitecture() {
106+
return architecture.get();
107+
}
108+
109+
public void setArchitecture(final String architecture) {
110+
if (ALLOWED_ARCHITECTURES.contains(architecture) == false) {
111+
throw new IllegalArgumentException(
112+
"unknown architecture [" + architecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
113+
);
114+
}
115+
this.architecture.set(architecture);
116+
}
117+
102118
public String getBaseVersion() {
103119
return baseVersion;
104120
}
@@ -167,9 +183,13 @@ void finalizeValues() {
167183
if (vendor.isPresent() == false) {
168184
throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]");
169185
}
186+
if (architecture.isPresent() == false) {
187+
throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]");
188+
}
170189
version.finalizeValue();
171190
platform.finalizeValue();
172191
vendor.finalizeValue();
192+
architecture.finalizeValue();
173193
}
174194

175195
@Override

‎buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@
4747
import java.util.concurrent.Callable;
4848
import java.util.stream.StreamSupport;
4949

50-
import static org.elasticsearch.gradle.Util.capitalize;
5150
import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
5251
import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
5352

@@ -72,7 +71,10 @@ public void apply(Project project) {
7271
DependencyHandler dependencies = project.getDependencies();
7372
Map<String, Object> depConfig = new HashMap<>();
7473
depConfig.put("path", ":"); // root project
75-
depConfig.put("configuration", configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform()));
74+
depConfig.put(
75+
"configuration",
76+
configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture())
77+
);
7678
project.getDependencies().add(jdk.getConfigurationName(), dependencies.project(depConfig));
7779

7880
// ensure a root level jdk download task exists
@@ -87,7 +89,14 @@ public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
8789
}
8890

8991
private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
90-
String extractTaskName = "extract" + capitalize(jdk.getPlatform()) + "Jdk-" + jdk.getVendor() + "-" + jdk.getVersion();
92+
String extractTaskName = String.format(
93+
Locale.ROOT,
94+
"extract-%s-%s-jdk-%s-%s",
95+
jdk.getPlatform(),
96+
jdk.getArchitecture(),
97+
jdk.getVendor(),
98+
jdk.getVersion()
99+
);
91100

92101
// Skip setup if we've already configured a JDK for this platform, vendor and version
93102
if (findByName(rootProject.getTasks(), extractTaskName) == null) {
@@ -107,7 +116,7 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
107116
repoUrl = "https://artifactory.elstc.co/artifactory/oss-jdk-local/";
108117
artifactPattern = String.format(
109118
Locale.ROOT,
110-
"adoptopenjdk/OpenJDK%sU-jdk_x64_[module]_hotspot_[revision]_%s.[ext]",
119+
"adoptopenjdk/OpenJDK%sU-jdk_[classifier]_[module]_hotspot_[revision]_%s.[ext]",
111120
jdk.getMajor(),
112121
jdk.getBuild()
113122
);
@@ -121,14 +130,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
121130
+ jdk.getHash()
122131
+ "/"
123132
+ jdk.getBuild()
124-
+ "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
133+
+ "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
125134
} else {
126135
// simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
127136
artifactPattern = "java/GA/jdk"
128137
+ jdk.getMajor()
129138
+ "/"
130139
+ jdk.getBuild()
131-
+ "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
140+
+ "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
132141
}
133142
} else {
134143
throw new GradleException("Unknown JDK vendor [" + jdk.getVendor() + "]");
@@ -150,14 +159,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
150159

151160
// Declare a configuration and dependency from which to download the remote JDK
152161
final ConfigurationContainer configurations = rootProject.getConfigurations();
153-
String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
162+
String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture());
154163
Configuration downloadConfiguration = maybeCreate(configurations, downloadConfigName);
155164
rootProject.getDependencies().add(downloadConfigName, dependencyNotation(jdk));
156165

157166
// Create JDK extract task
158167
final Provider<Directory> extractPath = rootProject.getLayout()
159168
.getBuildDirectory()
160-
.dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform());
169+
.dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform() + "_" + jdk.getArchitecture());
161170

162171
TaskProvider<?> extractTask = createExtractTask(
163172
extractTaskName,
@@ -168,7 +177,13 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
168177
);
169178

170179
// Declare a configuration for the extracted JDK archive
171-
String artifactConfigName = configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
180+
String artifactConfigName = configName(
181+
"extracted_jdk",
182+
jdk.getVendor(),
183+
jdk.getVersion(),
184+
jdk.getPlatform(),
185+
jdk.getArchitecture()
186+
);
172187
maybeCreate(configurations, artifactConfigName);
173188
rootProject.getArtifacts().add(artifactConfigName, extractPath, artifact -> artifact.builtBy(extractTask));
174189
}
@@ -254,7 +269,7 @@ private static String dependencyNotation(Jdk jdk) {
254269
: jdk.getPlatform();
255270
String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
256271

257-
return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
272+
return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + ":" + jdk.getArchitecture() + "@" + extension;
258273
}
259274

260275
private static String groupName(Jdk jdk) {

‎buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -173,12 +173,14 @@ private static Jdk createJdk(
173173
String name,
174174
String vendor,
175175
String version,
176-
String platform
176+
String platform,
177+
String architecture
177178
) {
178179
Jdk jdk = jdksContainer.create(name);
179180
jdk.setVendor(vendor);
180181
jdk.setVersion(version);
181182
jdk.setPlatform(platform);
183+
jdk.setArchitecture(architecture);
182184
return jdk;
183185
}
184186

@@ -210,7 +212,7 @@ private static List<Object> configureVM(Project project) {
210212

211213
NamedDomainObjectContainer<Jdk> jdksContainer = JdkDownloadPlugin.getContainer(project);
212214
String platform = box.contains("windows") ? "windows" : "linux";
213-
Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform);
215+
Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform, "x64");
214216

215217
// setup VM used by these tests
216218
VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class);

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.elasticsearch.gradle.testclusters;
2020

21+
import org.elasticsearch.gradle.Architecture;
2122
import org.elasticsearch.gradle.DistributionDownloadPlugin;
2223
import org.elasticsearch.gradle.Jdk;
2324
import org.elasticsearch.gradle.JdkDownloadPlugin;
@@ -65,6 +66,7 @@ public void apply(Project project) {
6566
jdk.setVendor(LEGACY_JAVA_VENDOR);
6667
jdk.setVersion(LEGACY_JAVA_VERSION);
6768
jdk.setPlatform(OS.current().name().toLowerCase());
69+
jdk.setArchitecture(Architecture.current().name().toLowerCase());
6870
});
6971

7072
// enable the DSL to describe clusters

‎buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public static void setupRoot() {
3636
}
3737

3838
public void testMissingVendor() {
39-
assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "vendor not specified for jdk [testjdk]");
39+
assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "x64", "vendor not specified for jdk [testjdk]");
4040
}
4141

4242
public void testUnknownVendor() {
@@ -46,20 +46,29 @@ public void testUnknownVendor() {
4646
"unknown",
4747
"11.0.2+33",
4848
"linux",
49+
"x64",
4950
"unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]"
5051
);
5152
}
5253

5354
public void testMissingVersion() {
54-
assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "version not specified for jdk [testjdk]");
55+
assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "x64", "version not specified for jdk [testjdk]");
5556
}
5657

5758
public void testBadVersionFormat() {
58-
assertJdkError(createProject(), "testjdk", "openjdk", "badversion", "linux", "malformed version [badversion] for jdk [testjdk]");
59+
assertJdkError(
60+
createProject(),
61+
"testjdk",
62+
"openjdk",
63+
"badversion",
64+
"linux",
65+
"x64",
66+
"malformed version [badversion] for jdk [testjdk]"
67+
);
5968
}
6069

6170
public void testMissingPlatform() {
62-
assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "platform not specified for jdk [testjdk]");
71+
assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "x64", "platform not specified for jdk [testjdk]");
6372
}
6473

6574
public void testUnknownPlatform() {
@@ -69,19 +78,44 @@ public void testUnknownPlatform() {
6978
"openjdk",
7079
"11.0.2+33",
7180
"unknown",
81+
"x64",
7282
"unknown platform [unknown] for jdk [testjdk], must be one of [darwin, linux, windows, mac]"
7383
);
7484
}
7585

76-
private void assertJdkError(Project project, String name, String vendor, String version, String platform, String message) {
86+
public void testMissingArchitecture() {
87+
assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", "linux", null, "architecture not specified for jdk [testjdk]");
88+
}
89+
90+
public void testUnknownArchitecture() {
91+
assertJdkError(
92+
createProject(),
93+
"testjdk",
94+
"openjdk",
95+
"11.0.2+33",
96+
"linux",
97+
"unknown",
98+
"unknown architecture [unknown] for jdk [testjdk], must be one of [aarch64, x64]"
99+
);
100+
}
101+
102+
private void assertJdkError(
103+
final Project project,
104+
final String name,
105+
final String vendor,
106+
final String version,
107+
final String platform,
108+
final String architecture,
109+
final String message
110+
) {
77111
IllegalArgumentException e = expectThrows(
78112
IllegalArgumentException.class,
79-
() -> createJdk(project, name, vendor, version, platform)
113+
() -> createJdk(project, name, vendor, version, platform, architecture)
80114
);
81115
assertThat(e.getMessage(), equalTo(message));
82116
}
83117

84-
private void createJdk(Project project, String name, String vendor, String version, String platform) {
118+
private void createJdk(Project project, String name, String vendor, String version, String platform, String architecture) {
85119
@SuppressWarnings("unchecked")
86120
NamedDomainObjectContainer<Jdk> jdks = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
87121
jdks.create(name, jdk -> {
@@ -94,6 +128,9 @@ private void createJdk(Project project, String name, String vendor, String versi
94128
if (platform != null) {
95129
jdk.setPlatform(platform);
96130
}
131+
if (architecture != null) {
132+
jdk.setArchitecture(architecture);
133+
}
97134
}).finalizeValues();
98135
}
99136

‎buildSrc/src/testKit/jdk-download/reuse/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,6 @@ jdks {
77
vendor = fakeJdkVendor
88
version = fakeJdkVersion
99
platform = "linux"
10+
architeecture = "x64"
1011
}
1112
}

‎buildSrc/src/testKit/jdk-download/subproj/build.gradle

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,19 @@ jdks {
1010
vendor = fakeJdkVendor
1111
version = fakeJdkVersion
1212
platform = "linux"
13+
architecture = "x64"
1314
}
1415
darwin {
1516
vendor = fakeJdkVendor
1617
version = fakeJdkVersion
1718
platform = "darwin"
19+
architecture = "x64"
1820
}
1921
windows {
2022
vendor = fakeJdkVendor
2123
version = fakeJdkVersion
2224
platform = "windows"
25+
architecture = "x64"
2326
}
2427
}
2528

0 commit comments

Comments
 (0)