Skip to content

Commit 82d09a2

Browse files
pxsalehipxsalehi
committed
Merge remote-tracking branch 'upstream/main' into ps230502-sharGetFromTranslog
2 parents 5f7d6b6 + e70020d commit 82d09a2

File tree

169 files changed

+2726
-738
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

169 files changed

+2726
-738
lines changed

‎.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,4 @@
1818
export SNYK_TOKEN=$(vault read -field=token secret/elasticsearch-ci/snyk)
1919
unset VAULT_TOKEN
2020
set -x
21-
$WORKSPACE/.ci/scripts/run-gradle.sh uploadSnykDependencyGraph
21+
$WORKSPACE/.ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=%BRANCH%

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
package org.elasticsearch.gradle.internal.conventions;
1010

1111
import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
12+
import org.elasticsearch.gradle.internal.conventions.util.Util;
1213
import org.gradle.api.Plugin;
1314
import org.gradle.api.Project;
1415
import org.gradle.api.model.ObjectFactory;
@@ -35,10 +36,7 @@ class GitInfoPlugin implements Plugin<Project> {
3536

3637
@Override
3738
public void apply(Project project) {
38-
File rootDir = (project.getGradle().getParent() == null) ?
39-
project.getRootDir() :
40-
project.getGradle().getParent().getRootProject().getRootDir();
41-
39+
File rootDir = Util.locateElasticsearchWorkspace(project.getGradle());
4240
gitInfo = objectFactory.property(GitInfo.class).value(factory.provider(() ->
4341
GitInfo.gitInfo(rootDir)
4442
));

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import javax.annotation.Nullable;
2323
import java.io.File;
24+
import java.util.Collection;
2425
import java.util.Optional;
2526
import java.util.function.Supplier;
2627

@@ -105,7 +106,6 @@ private static boolean isJavaExtensionAvailable(Project project) {
105106
return project.getExtensions().getByType(JavaPluginExtension.class) == null;
106107
}
107108

108-
109109
public static Object toStringable(Supplier<String> getter) {
110110
return new Object() {
111111
@Override
@@ -119,21 +119,31 @@ public static SourceSetContainer getJavaSourceSets(Project project) {
119119
return project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets();
120120
}
121121

122-
public static File locateElasticsearchWorkspace(Gradle project) {
123-
if (project.getParent() == null) {
124-
// See if any of these included builds is the Elasticsearch project
125-
for (IncludedBuild includedBuild : project.getIncludedBuilds()) {
126-
File versionProperties = new File(includedBuild.getProjectDir(), "build-tools-internal/version.properties");
127-
if (versionProperties.exists()) {
128-
return includedBuild.getProjectDir();
122+
public static File locateElasticsearchWorkspace(Gradle gradle) {
123+
if(gradle.getRootProject().getName().startsWith("build-tools")) {
124+
File buildToolsParent = gradle.getRootProject().getRootDir().getParentFile();
125+
if(versionFileExists(buildToolsParent)) {
126+
return buildToolsParent;
129127
}
128+
return buildToolsParent;
130129
}
130+
if (gradle.getParent() == null) {
131+
// See if any of these included builds is the Elasticsearch gradle
132+
for (IncludedBuild includedBuild : gradle.getIncludedBuilds()) {
133+
if (versionFileExists(includedBuild.getProjectDir())) {
134+
return includedBuild.getProjectDir();
135+
}
136+
}
131137

132-
// Otherwise assume this project is the root elasticsearch workspace
133-
return project.getRootProject().getRootDir();
134-
} else {
135-
// We're an included build, so keep looking
136-
return locateElasticsearchWorkspace(project.getParent());
138+
// Otherwise assume this gradle is the root elasticsearch workspace
139+
return gradle.getRootProject().getRootDir();
140+
} else {
141+
// We're an included build, so keep looking
142+
return locateElasticsearchWorkspace(gradle.getParent());
143+
}
137144
}
145+
146+
private static boolean versionFileExists(File rootDir) {
147+
return new File(rootDir, "build-tools-internal/version.properties").exists();
138148
}
139149
}

‎build-tools-internal/build.gradle

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,10 @@ gradlePlugin {
107107
id = 'elasticsearch.internal-test-rerun'
108108
implementationClass = 'org.elasticsearch.gradle.internal.test.rerun.TestRerunPlugin'
109109
}
110+
javaToolChainPlugin {
111+
id = 'elasticsearch.java-toolchain'
112+
implementationClass = 'org.elasticsearch.gradle.internal.toolchain.JavaToolChainResolverPlugin'
113+
}
110114
javaDoc {
111115
id = 'elasticsearch.java-doc'
112116
implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavadocPlugin'

‎build-tools-internal/settings.gradle

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
pluginManagement {
2+
includeBuild "../build-conventions"
3+
includeBuild "../build-tools"
4+
}
5+
16
dependencyResolutionManagement {
27
versionCatalogs {
38
buildLibs {

‎build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -245,20 +245,20 @@ Node parseXml(Object path) {
245245
return xml
246246
}
247247

248-
Pair<File, IncludedBuild> locateElasticsearchWorkspace(Gradle project) {
249-
if (project.parent == null) {
250-
// See if any of these included builds is the Elasticsearch project
251-
for (IncludedBuild includedBuild : project.includedBuilds) {
248+
Pair<File, IncludedBuild> locateElasticsearchWorkspace(Gradle gradle) {
249+
if (gradle.parent == null) {
250+
// See if any of these included builds is the Elasticsearch gradle
251+
for (IncludedBuild includedBuild : gradle.includedBuilds) {
252252
File versionProperties = new File(includedBuild.projectDir, 'build-tools-internal/version.properties')
253253
if (versionProperties.exists()) {
254254
return Pair.of(includedBuild.projectDir, includedBuild)
255255
}
256256
}
257257

258-
// Otherwise assume this project is the root elasticsearch workspace
259-
return Pair.of(project.getRootProject().getRootDir(), null)
258+
// Otherwise assume this gradle is the root elasticsearch workspace
259+
return Pair.of(gradle.getRootProject().getRootDir(), null)
260260
} else {
261261
// We're an included build, so keep looking
262-
return locateElasticsearchWorkspace(project.parent)
262+
return locateElasticsearchWorkspace(gradle.parent)
263263
}
264264
}

‎build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,12 @@ configure(allprojects) {
3535
test.executable = "${BuildParams.runtimeJavaHome}/bin/java" +
3636
(OS.current() == OS.WINDOWS ? '.exe' : '')
3737
} else {
38-
test.dependsOn(project.jdks.provisioned_runtime)
39-
test.executable = rootProject.jdks.provisioned_runtime.getBinJavaPath()
38+
test.javaLauncher = javaToolchains.launcherFor {
39+
languageVersion = JavaLanguageVersion.of(VersionProperties.bundledJdkMajorVersion)
40+
vendor = VersionProperties.bundledJdkVendor == "openjdk" ?
41+
JvmVendorSpec.ORACLE :
42+
JvmVendorSpec.matching(VersionProperties.bundledJdkVendor)
43+
}
4044
}
4145
}
4246
project.plugins.withId("elasticsearch.testclusters") { testClustersPlugin ->

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -84,11 +84,11 @@ public void apply(Project project) {
8484
TaskProvider<LoggedExec> addRemoteTaskProvider = tasks.register("addRemote", LoggedExec.class, addRemote -> {
8585
addRemote.dependsOn(findRemoteTaskProvider);
8686
addRemote.onlyIf("remote exists", task -> ((boolean) extraProperties.get("remoteExists")) == false);
87-
addRemote.getWorkingDir().set(gitExtension.getCheckoutDir().get());
87+
addRemote.getWorkingDir().set(gitExtension.getCheckoutDir());
8888
String remoteRepo = remote.get();
8989
// for testing only we can override the base remote url
9090
String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
91-
.getOrElse("https://github.com/" + remoteRepo + "/elasticsearch.git");
91+
.getOrElse("https://github.com/" + remoteRepo + "/" + project.getRootProject().getName());
9292
addRemote.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl);
9393
});
9494

@@ -105,8 +105,9 @@ public void apply(Project project) {
105105
});
106106
fetchLatest.onlyIf("online and gitFetchLatest == true", t -> isOffline == false && gitFetchLatest.get());
107107
fetchLatest.dependsOn(addRemoteTaskProvider);
108-
fetchLatest.getWorkingDir().set(gitExtension.getCheckoutDir().get());
109-
fetchLatest.commandLine("git", "fetch", "--all");
108+
fetchLatest.getWorkingDir().set(gitExtension.getCheckoutDir());
109+
// Fetch latest from remotes, including tags, overriding any existing local refs
110+
fetchLatest.commandLine("git", "fetch", "--all", "--tags", "--force");
110111
});
111112

112113
String projectPath = project.getPath();
@@ -119,13 +120,20 @@ public void execute(Task task) {
119120
String bwcBranch = gitExtension.getBwcBranch().get();
120121
final String refspec = providerFactory.systemProperty("bwc.refspec." + bwcBranch)
121122
.orElse(providerFactory.systemProperty("tests.bwc.refspec." + bwcBranch))
123+
.orElse(
124+
providerFactory.provider(
125+
() -> task.getExtensions().getExtraProperties().has("refspec")
126+
? task.getExtensions().getExtraProperties().get("refspec").toString()
127+
: null
128+
)
129+
)
122130
.getOrElse(remote.get() + "/" + bwcBranch);
123131

124132
String effectiveRefSpec = maybeAlignedRefSpec(task.getLogger(), refspec);
125133
task.getLogger().lifecycle("Performing checkout of {}...", refspec);
126134
LoggedExec.exec(execOperations, spec -> {
127135
spec.workingDir(checkoutDir);
128-
spec.commandLine("git", "checkout", effectiveRefSpec);
136+
spec.commandLine("git", "checkout", "--recurse-submodules", effectiveRefSpec);
129137
});
130138

131139
String checkoutHash = GitInfo.gitInfo(checkoutDir).getRevision();

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,11 +172,11 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
172172
return projectName + distribution.getType().getName();
173173
}
174174

175-
private static class ProjectBasedDistributionDependency implements DistributionDependency {
175+
public static class ProjectBasedDistributionDependency implements DistributionDependency {
176176

177177
private Function<String, Dependency> function;
178178

179-
ProjectBasedDistributionDependency(Function<String, Dependency> function) {
179+
public ProjectBasedDistributionDependency(Function<String, Dependency> function) {
180180
this.function = function;
181181
}
182182

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,13 @@
2222

2323
import java.util.Arrays;
2424

25+
/**
26+
* @deprecated We wanna get rid from this and custom jdk downloads via this plugin and
27+
* make leverage the gradle toolchain resolver capabilities.
28+
*
29+
* @See @org.elasticsearch.gradle.internal.toolchain.JavaToolChainResolverPlugin
30+
* */
31+
@Deprecated
2532
public class JdkDownloadPlugin implements Plugin<Project> {
2633

2734
public static final String VENDOR_ADOPTIUM = "adoptium";
@@ -161,7 +168,6 @@ public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
161168
private static String dependencyNotation(Jdk jdk) {
162169
String platformDep = isJdkOnMacOsPlatform(jdk) ? (jdk.getVendor().equals(VENDOR_ADOPTIUM) ? "mac" : "macos") : jdk.getPlatform();
163170
String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
164-
165171
return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + ":" + jdk.getArchitecture() + "@" + extension;
166172
}
167173

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,24 @@
2020
import org.gradle.api.Project;
2121
import org.gradle.api.logging.Logger;
2222
import org.gradle.api.logging.Logging;
23+
import org.gradle.api.model.ObjectFactory;
24+
import org.gradle.api.plugins.JvmToolchainsPlugin;
25+
import org.gradle.api.provider.Property;
2326
import org.gradle.api.provider.Provider;
2427
import org.gradle.api.provider.ProviderFactory;
2528
import org.gradle.internal.jvm.Jvm;
2629
import org.gradle.internal.jvm.inspection.JvmInstallationMetadata;
2730
import org.gradle.internal.jvm.inspection.JvmMetadataDetector;
2831
import org.gradle.internal.jvm.inspection.JvmVendor;
2932
import org.gradle.jvm.toolchain.JavaLanguageVersion;
33+
import org.gradle.jvm.toolchain.JavaLauncher;
34+
import org.gradle.jvm.toolchain.JavaToolchainService;
3035
import org.gradle.jvm.toolchain.JavaToolchainSpec;
3136
import org.gradle.jvm.toolchain.JvmVendorSpec;
3237
import org.gradle.jvm.toolchain.internal.InstallationLocation;
3338
import org.gradle.jvm.toolchain.internal.JavaInstallationRegistry;
3439
import org.gradle.util.GradleVersion;
40+
import org.jetbrains.annotations.NotNull;
3541

3642
import java.io.BufferedReader;
3743
import java.io.File;
@@ -56,26 +62,33 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
5662
private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
5763
private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/elasticsearch/Version.java";
5864

65+
private ObjectFactory objectFactory;
5966
private final JavaInstallationRegistry javaInstallationRegistry;
6067
private final JvmMetadataDetector metadataDetector;
6168
private final ProviderFactory providers;
69+
private JavaToolchainService toolChainService;
6270

6371
@Inject
6472
public GlobalBuildInfoPlugin(
73+
ObjectFactory objectFactory,
6574
JavaInstallationRegistry javaInstallationRegistry,
6675
JvmMetadataDetector metadataDetector,
6776
ProviderFactory providers
6877
) {
78+
this.objectFactory = objectFactory;
6979
this.javaInstallationRegistry = javaInstallationRegistry;
7080
this.metadataDetector = new ErrorTraceMetadataDetector(metadataDetector);
7181
this.providers = providers;
82+
7283
}
7384

7485
@Override
7586
public void apply(Project project) {
7687
if (project != project.getRootProject()) {
7788
throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
7889
}
90+
project.getPlugins().apply(JvmToolchainsPlugin.class);
91+
toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
7992
GradleVersion minimumGradleVersion = GradleVersion.version(getResourceContents("/minimumGradleVersion"));
8093
if (GradleVersion.current().compareTo(minimumGradleVersion) < 0) {
8194
throw new GradleException("Gradle " + minimumGradleVersion.getVersion() + "+ is required");
@@ -87,8 +100,7 @@ public void apply(Project project) {
87100
File runtimeJavaHome = findRuntimeJavaHome();
88101
boolean isRuntimeJavaHomeSet = Jvm.current().getJavaHome().equals(runtimeJavaHome) == false;
89102

90-
File rootDir = project.getRootDir();
91-
GitInfo gitInfo = GitInfo.gitInfo(rootDir);
103+
GitInfo gitInfo = GitInfo.gitInfo(project.getRootDir());
92104

93105
BuildParams.init(params -> {
94106
params.reset();
@@ -114,10 +126,18 @@ public void apply(Project project) {
114126
params.setTestSeed(getTestSeed());
115127
params.setIsCi(System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null);
116128
params.setDefaultParallel(ParallelDetector.findDefaultParallel(project));
129+
// TODO: Test if cc issues are coming from here
130+
params.setDefaultParallel(8);
117131
params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
118132
params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));
119133
AtomicReference<BwcVersions> cache = new AtomicReference<>();
120-
params.setBwcVersions(providers.provider(() -> cache.updateAndGet(val -> val == null ? resolveBwcVersions(rootDir) : val)));
134+
params.setBwcVersions(
135+
providers.provider(
136+
() -> cache.updateAndGet(
137+
val -> val == null ? resolveBwcVersions(Util.locateElasticsearchWorkspace(project.getGradle())) : val
138+
)
139+
)
140+
);
121141
});
122142

123143
// Enforce the minimum compiler version
@@ -219,7 +239,7 @@ private InstallationLocation getJavaInstallation(File javaHome) {
219239

220240
private boolean isSameFile(File javaHome, InstallationLocation installationLocation) {
221241
try {
222-
return Files.isSameFile(installationLocation.getLocation().toPath(), javaHome.toPath());
242+
return Files.isSameFile(javaHome.toPath(), installationLocation.getLocation().toPath());
223243
} catch (IOException ioException) {
224244
throw new UncheckedIOException(ioException);
225245
}
@@ -294,9 +314,14 @@ private File findRuntimeJavaHome() {
294314
}
295315

296316
private String findJavaHome(String version) {
297-
Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv");
298317
String javaHomeEnvVar = getJavaHomeEnvVarName(version);
318+
String env = System.getenv(javaHomeEnvVar);
319+
return env != null ? resolveJavaHomeFromEnvVariable(javaHomeEnvVar) : resolveJavaHomeFromToolChainService(version);
320+
}
299321

322+
@NotNull
323+
private String resolveJavaHomeFromEnvVariable(String javaHomeEnvVar) {
324+
Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv");
300325
// Provide a useful error if we're looking for a Java home version that we haven't told Gradle about yet
301326
Arrays.stream(javaHomeNames.get().split(","))
302327
.filter(s -> s.equals(javaHomeEnvVar))
@@ -309,7 +334,6 @@ private String findJavaHome(String version) {
309334
+ "updated in gradle.properties file."
310335
)
311336
);
312-
313337
String versionedJavaHome = System.getenv(javaHomeEnvVar);
314338
if (versionedJavaHome == null) {
315339
final String exceptionMessage = String.format(
@@ -320,12 +344,25 @@ private String findJavaHome(String version) {
320344
+ "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details.",
321345
javaHomeEnvVar
322346
);
323-
324347
throw new GradleException(exceptionMessage);
325348
}
326349
return versionedJavaHome;
327350
}
328351

352+
@NotNull
353+
private String resolveJavaHomeFromToolChainService(String version) {
354+
Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
355+
Provider<JavaLauncher> javaLauncherProvider = toolChainService.launcherFor(
356+
javaToolchainSpec -> javaToolchainSpec.getLanguageVersion().value(value)
357+
);
358+
359+
try {
360+
return javaLauncherProvider.get().getMetadata().getInstallationPath().getAsFile().getCanonicalPath();
361+
} catch (IOException e) {
362+
throw new RuntimeException(e);
363+
}
364+
}
365+
329366
private static String getJavaHomeEnvVarName(String version) {
330367
return "JAVA" + version + "_HOME";
331368
}
@@ -369,7 +406,7 @@ private static class MetadataBasedToolChainMatcher implements Action<JavaToolcha
369406
private final JvmVendorSpec expectedVendorSpec;
370407
private final JavaLanguageVersion expectedJavaLanguageVersion;
371408

372-
public MetadataBasedToolChainMatcher(JvmInstallationMetadata metadata) {
409+
MetadataBasedToolChainMatcher(JvmInstallationMetadata metadata) {
373410
expectedVendorSpec = JvmVendorSpec.matching(metadata.getVendor().getRawVendor());
374411
expectedJavaLanguageVersion = JavaLanguageVersion.of(metadata.getLanguageVersion().getMajorVersion());
375412
}

0 commit comments

Comments
 (0)