Skip to content

Commit 673212b

Browse files
tlrxtlrx
committed
Merge branch 'master' into feature/searchable-snapshots
2 parents 02aa32e + 36377d7 commit 673212b

File tree

616 files changed

+12018
-6561
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

616 files changed

+12018
-6561
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ out/
1010
# include shared intellij config
1111
!.idea/scopes/x_pack.xml
1212
!.idea/inspectionProfiles/Project_Default.xml
13+
!.idea/runConfigurations/Debug_Elasticsearch.xml
1314

1415
# These files are generated in the main tree by IntelliJ
1516
benchmarks/src/main/generated/*

‎.idea/runConfigurations/Debug_Elasticsearch.xml

Lines changed: 11 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎CONTRIBUTING.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,6 @@ be used to test against other JDKs as well, this is not only limited to JDK 11.
103103
> Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME`
104104
and `JAVA11_HOME`, and `JAVA12_HOME` available so that the tests can pass.
105105

106-
> Warning: do not use `sdkman` for Java installations which do not have proper
107-
`jrunscript` for jdk distributions.
108-
109106
Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
110107
using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
111108
script on Windows in the root of the repository. The examples below show the
@@ -125,10 +122,13 @@ You can access Elasticsearch with:
125122

126123
### Importing the project into IntelliJ IDEA
127124

128-
Elasticsearch builds using Java 13. Before importing into IntelliJ you will need
125+
Elasticsearch builds using Java 13. When importing into IntelliJ you will need
129126
to define an appropriate SDK. The convention is that **this SDK should be named
130127
"13"** so that the project import will detect it automatically. For more details
131128
on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk).
129+
SDK definitions are global, so you can add the JDK from any project, or after
130+
project import. Importing with a missing JDK will still work, IntelliJ will
131+
simply report a problem and will refuse to build until resolved.
132132

133133
You can import the Elasticsearch project into IntelliJ IDEA via:
134134

‎build.gradle

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder
3131
import org.gradle.util.DistributionLocator
3232
import org.gradle.util.GradleVersion
3333

34-
import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
34+
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
3535

3636
plugins {
3737
id 'lifecycle-base'
@@ -545,3 +545,15 @@ allprojects {
545545
}
546546
}
547547
}
548+
549+
// TODO: remove this once 7.7 is released and the 7.x branch is 7.8
550+
subprojects {
551+
pluginManager.withPlugin('elasticsearch.testclusters') {
552+
testClusters.all {
553+
if (org.elasticsearch.gradle.info.BuildParams.isSnapshotBuild() == false) {
554+
systemProperty 'es.itv2_feature_flag_registered', 'true'
555+
systemProperty 'es.datastreams_feature_flag_registered', 'true'
556+
}
557+
}
558+
}
559+
}

‎buildSrc/build.gradle

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,8 +187,10 @@ if (project != rootProject) {
187187
distribution project(':distribution:archives:oss-windows-zip')
188188
distribution project(':distribution:archives:darwin-tar')
189189
distribution project(':distribution:archives:oss-darwin-tar')
190+
distribution project(':distribution:archives:linux-aarch64-tar')
190191
distribution project(':distribution:archives:linux-tar')
191192
distribution project(':distribution:archives:oss-linux-tar')
193+
distribution project(':distribution:archives:oss-linux-aarch64-tar')
192194
}
193195

194196
// for external projects we want to remove the marker file indicating we are running the Elasticsearch project

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
3232
import org.elasticsearch.gradle.test.ErrorReportingTestListener
3333
import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
3434
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
35-
import org.elasticsearch.gradle.tool.Boilerplate
35+
import org.elasticsearch.gradle.util.GradleUtils
3636
import org.gradle.api.Action
3737
import org.gradle.api.GradleException
3838
import org.gradle.api.InvalidUserDataException
@@ -82,7 +82,7 @@ import org.gradle.util.GradleVersion
8282
import java.nio.charset.StandardCharsets
8383
import java.nio.file.Files
8484

85-
import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
85+
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
8686

8787
/**
8888
* Encapsulates build configuration for elasticsearch projects.
@@ -148,7 +148,7 @@ class BuildPlugin implements Plugin<Project> {
148148
File securityPolicy = buildResources.copy("fips_java.policy")
149149
File bcfksKeystore = buildResources.copy("cacerts.bcfks")
150150
// This configuration can be removed once system modules are available
151-
Boilerplate.maybeCreate(project.configurations, 'extraJars') {
151+
GradleUtils.maybeCreate(project.configurations, 'extraJars') {
152152
project.dependencies.add('extraJars', "org.bouncycastle:bc-fips:1.0.1")
153153
project.dependencies.add('extraJars', "org.bouncycastle:bctls-fips:1.0.9")
154154
}
@@ -235,7 +235,7 @@ class BuildPlugin implements Plugin<Project> {
235235
static String getJavaHome(final Task task, final int version) {
236236
requireJavaHome(task, version)
237237
JavaHome java = BuildParams.javaVersions.find { it.version == version }
238-
return java == null ? null : java.javaHome.absolutePath
238+
return java == null ? null : java.javaHome.get().absolutePath
239239
}
240240

241241
/**
@@ -453,7 +453,9 @@ class BuildPlugin implements Plugin<Project> {
453453

454454
project.pluginManager.withPlugin('com.github.johnrengelman.shadow') {
455455
// Ensure that when we are compiling against the "original" JAR that we also include any "shadow" dependencies on the compile classpath
456-
project.configurations.getByName(JavaPlugin.API_ELEMENTS_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME))
456+
project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME).dependencies.all { Dependency dependency ->
457+
project.configurations.getByName(JavaPlugin.API_ELEMENTS_CONFIGURATION_NAME).dependencies.add(dependency)
458+
}
457459
}
458460
}
459461

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -314,10 +314,7 @@ class RestTestsFromSnippetsTask extends SnippetsTask {
314314
if (path == null) {
315315
path = '' // Catch requests to the root...
316316
} else {
317-
// Escape some characters that are also escaped by sense
318317
path = path.replace('<', '%3C').replace('>', '%3E')
319-
path = path.replace('{', '%7B').replace('}', '%7D')
320-
path = path.replace('|', '%7C')
321318
}
322319
current.println(" - do:")
323320
if (catchPart != null) {

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -143,16 +143,13 @@ class PrecommitTasks {
143143
ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
144144
project.tasks.withType(CheckForbiddenApis).configureEach {
145145
dependsOn(buildResources)
146-
doFirst {
147-
// we need to defer this configuration since we don't know the runtime java version until execution time
148-
targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
149-
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
150-
project.logger.warn(
151-
"Forbidden APIs does not support Java versions past 13. Will use the signatures from 13 for {}.",
152-
BuildParams.runtimeJavaVersion
153-
)
154-
targetCompatibility = JavaVersion.VERSION_13.majorVersion
155-
}
146+
targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
147+
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
148+
project.logger.warn(
149+
"Forbidden APIs does not support Java versions past 13. Will use the signatures from 13 for {}.",
150+
BuildParams.runtimeJavaVersion
151+
)
152+
targetCompatibility = JavaVersion.VERSION_13.majorVersion
156153
}
157154
bundledSignatures = [
158155
"jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"

‎modules/kibana/build.gradle renamed to ‎buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* not use this file except in compliance with the License.
88
* You may obtain a copy of the License at
99
*
10-
* http://www.apache.org/licenses/LICENSE-2.0
10+
* http://www.apache.org/licenses/LICENSE-2.0
1111
*
1212
* Unless required by applicable law or agreed to in writing,
1313
* software distributed under the License is distributed on an
@@ -17,15 +17,24 @@
1717
* under the License.
1818
*/
1919

20-
esplugin {
21-
description 'Plugin exposing APIs for Kibana system indices'
22-
classname 'org.elasticsearch.kibana.KibanaPlugin'
23-
}
20+
package org.elasticsearch.gradle;
2421

25-
dependencies {
26-
compile project(path: ':modules:reindex', configuration: 'runtime')
27-
}
22+
public enum Architecture {
23+
24+
X64,
25+
AARCH64;
26+
27+
public static Architecture current() {
28+
final String architecture = System.getProperty("os.arch", "");
29+
switch (architecture) {
30+
case "amd64":
31+
case "x86_64":
32+
return X64;
33+
case "aarch64":
34+
return AARCH64;
35+
default:
36+
throw new IllegalArgumentException("can not determine architecture from [" + architecture + "]");
37+
}
38+
}
2839

29-
testClusters.integTest {
30-
module file(project(':modules:reindex').tasks.bundlePlugin.archiveFile)
3140
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
import org.elasticsearch.gradle.docker.DockerSupportService;
2727
import org.elasticsearch.gradle.info.BuildParams;
2828
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
29-
import org.elasticsearch.gradle.tool.Boilerplate;
29+
import org.elasticsearch.gradle.util.GradleUtils;
3030
import org.gradle.api.GradleException;
3131
import org.gradle.api.NamedDomainObjectContainer;
3232
import org.gradle.api.Plugin;
@@ -52,7 +52,7 @@
5252
import java.util.concurrent.Callable;
5353
import java.util.function.Supplier;
5454

55-
import static org.elasticsearch.gradle.Util.capitalize;
55+
import static org.elasticsearch.gradle.util.Util.capitalize;
5656

5757
/**
5858
* A plugin to manage getting and extracting distributions of Elasticsearch.
@@ -77,7 +77,7 @@ public void apply(Project project) {
7777
project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
7878
project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
7979

80-
Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
80+
Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(
8181
project.getGradle().getSharedServices(),
8282
DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
8383
);

‎buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333

3434
public class Jdk implements Buildable, Iterable<File> {
3535

36+
private static final List<String> ALLOWED_ARCHITECTURES = List.of("aarch64", "x64");
3637
private static final List<String> ALLOWED_VENDORS = List.of("adoptopenjdk", "openjdk");
3738
private static final List<String> ALLOWED_PLATFORMS = List.of("darwin", "linux", "windows", "mac");
3839
private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
@@ -44,6 +45,7 @@ public class Jdk implements Buildable, Iterable<File> {
4445
private final Property<String> vendor;
4546
private final Property<String> version;
4647
private final Property<String> platform;
48+
private final Property<String> architecture;
4749
private String baseVersion;
4850
private String major;
4951
private String build;
@@ -55,6 +57,7 @@ public class Jdk implements Buildable, Iterable<File> {
5557
this.vendor = objectFactory.property(String.class);
5658
this.version = objectFactory.property(String.class);
5759
this.platform = objectFactory.property(String.class);
60+
this.architecture = objectFactory.property(String.class);
5861
}
5962

6063
public String getName() {
@@ -97,6 +100,19 @@ public void setPlatform(String platform) {
97100
this.platform.set(platform);
98101
}
99102

103+
public String getArchitecture() {
104+
return architecture.get();
105+
}
106+
107+
public void setArchitecture(final String architecture) {
108+
if (ALLOWED_ARCHITECTURES.contains(architecture) == false) {
109+
throw new IllegalArgumentException(
110+
"unknown architecture [" + architecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
111+
);
112+
}
113+
this.architecture.set(architecture);
114+
}
115+
100116
public String getBaseVersion() {
101117
return baseVersion;
102118
}
@@ -153,9 +169,13 @@ void finalizeValues() {
153169
if (vendor.isPresent() == false) {
154170
throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]");
155171
}
172+
if (architecture.isPresent() == false) {
173+
throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]");
174+
}
156175
version.finalizeValue();
157176
platform.finalizeValue();
158177
vendor.finalizeValue();
178+
architecture.finalizeValue();
159179
}
160180

161181
@Override

‎buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,8 @@
4747
import java.util.concurrent.Callable;
4848
import java.util.stream.StreamSupport;
4949

50-
import static org.elasticsearch.gradle.Util.capitalize;
51-
import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
52-
import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
50+
import static org.elasticsearch.gradle.util.GradleUtils.findByName;
51+
import static org.elasticsearch.gradle.util.GradleUtils.maybeCreate;
5352

5453
public class JdkDownloadPlugin implements Plugin<Project> {
5554

@@ -72,7 +71,10 @@ public void apply(Project project) {
7271
DependencyHandler dependencies = project.getDependencies();
7372
Map<String, Object> depConfig = new HashMap<>();
7473
depConfig.put("path", ":"); // root project
75-
depConfig.put("configuration", configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform()));
74+
depConfig.put(
75+
"configuration",
76+
configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture())
77+
);
7678
project.getDependencies().add(jdk.getConfigurationName(), dependencies.project(depConfig));
7779

7880
// ensure a root level jdk download task exists
@@ -87,7 +89,14 @@ public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
8789
}
8890

8991
private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
90-
String extractTaskName = "extract" + capitalize(jdk.getPlatform()) + "Jdk-" + jdk.getVendor() + "-" + jdk.getVersion();
92+
String extractTaskName = String.format(
93+
Locale.ROOT,
94+
"extract-%s-%s-jdk-%s-%s",
95+
jdk.getPlatform(),
96+
jdk.getArchitecture(),
97+
jdk.getVendor(),
98+
jdk.getVersion()
99+
);
91100

92101
// Skip setup if we've already configured a JDK for this platform, vendor and version
93102
if (findByName(rootProject.getTasks(), extractTaskName) == null) {
@@ -107,7 +116,7 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
107116
repoUrl = "https://artifactory.elstc.co/artifactory/oss-jdk-local/";
108117
artifactPattern = String.format(
109118
Locale.ROOT,
110-
"adoptopenjdk/OpenJDK%sU-jdk_x64_[module]_hotspot_[revision]_%s.[ext]",
119+
"adoptopenjdk/OpenJDK%sU-jdk_[classifier]_[module]_hotspot_[revision]_%s.[ext]",
111120
jdk.getMajor(),
112121
jdk.getBuild()
113122
);
@@ -121,14 +130,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
121130
+ jdk.getHash()
122131
+ "/"
123132
+ jdk.getBuild()
124-
+ "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
133+
+ "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
125134
} else {
126135
// simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
127136
artifactPattern = "java/GA/jdk"
128137
+ jdk.getMajor()
129138
+ "/"
130139
+ jdk.getBuild()
131-
+ "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
140+
+ "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
132141
}
133142
} else {
134143
throw new GradleException("Unknown JDK vendor [" + jdk.getVendor() + "]");
@@ -150,14 +159,14 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
150159

151160
// Declare a configuration and dependency from which to download the remote JDK
152161
final ConfigurationContainer configurations = rootProject.getConfigurations();
153-
String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
162+
String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture());
154163
Configuration downloadConfiguration = maybeCreate(configurations, downloadConfigName);
155164
rootProject.getDependencies().add(downloadConfigName, dependencyNotation(jdk));
156165

157166
// Create JDK extract task
158167
final Provider<Directory> extractPath = rootProject.getLayout()
159168
.getBuildDirectory()
160-
.dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform());
169+
.dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform() + "_" + jdk.getArchitecture());
161170

162171
TaskProvider<?> extractTask = createExtractTask(
163172
extractTaskName,
@@ -168,7 +177,13 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
168177
);
169178

170179
// Declare a configuration for the extracted JDK archive
171-
String artifactConfigName = configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
180+
String artifactConfigName = configName(
181+
"extracted_jdk",
182+
jdk.getVendor(),
183+
jdk.getVersion(),
184+
jdk.getPlatform(),
185+
jdk.getArchitecture()
186+
);
172187
maybeCreate(configurations, artifactConfigName);
173188
rootProject.getArtifacts().add(artifactConfigName, extractPath, artifact -> artifact.builtBy(extractTask));
174189
}
@@ -254,7 +269,7 @@ private static String dependencyNotation(Jdk jdk) {
254269
: jdk.getPlatform();
255270
String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
256271

257-
return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
272+
return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + ":" + jdk.getArchitecture() + "@" + extension;
258273
}
259274

260275
private static String groupName(Jdk jdk) {

0 commit comments

Comments
 (0)