Skip to content

Commit 8457b81

Browse files
dimitris-athanasioudimitris-athanasiou
committed
Merge branch 'master' into feature-ml-data-frame-analytics
2 parents 88496c2 + 0dd73ef commit 8457b81

File tree

1,374 files changed

+43685
-9145
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,374 files changed

+43685
-9145
lines changed

‎benchmarks/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ dependencies {
3333
exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
3434
}
3535
compile "org.openjdk.jmh:jmh-core:$versions.jmh"
36-
compile "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
36+
annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
3737
// Dependencies of JMH
3838
runtime 'net.sf.jopt-simple:jopt-simple:4.6'
3939
runtime 'org.apache.commons:commons-math3:3.2'

‎build.gradle

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,16 @@ import org.elasticsearch.gradle.Version
2424
import org.elasticsearch.gradle.VersionCollection
2525
import org.elasticsearch.gradle.VersionProperties
2626
import org.elasticsearch.gradle.plugin.PluginBuildPlugin
27+
import org.gradle.api.tasks.options.Option
28+
import org.gradle.util.GradleVersion
29+
import org.gradle.util.DistributionLocator
2730
import org.gradle.plugins.ide.eclipse.model.SourceFolder
2831
import com.carrotsearch.gradle.junit4.RandomizedTestingTask
2932

3033
import java.util.function.Predicate
3134

3235
plugins {
33-
id 'com.gradle.build-scan' version '1.13.2'
36+
id 'com.gradle.build-scan' version '2.0.2'
3437
id 'base'
3538
}
3639
if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
@@ -527,7 +530,7 @@ allprojects {
527530
class Run extends DefaultTask {
528531
boolean debug = false
529532

530-
@org.gradle.api.internal.tasks.options.Option(
533+
@Option(
531534
option = "debug-jvm",
532535
description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
533536
)

‎buildSrc/build.gradle

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -147,10 +147,8 @@ if (project == rootProject) {
147147
mavenLocal()
148148
}
149149
}
150-
test {
151-
include "**/*Tests.class"
152-
exclude "**/*IT.class"
153-
}
150+
// only run tests as build-tools
151+
test.enabled = false
154152
}
155153

156154
/*****************************************************************************
@@ -164,6 +162,10 @@ if (project != rootProject) {
164162
apply plugin: 'nebula.maven-base-publish'
165163
apply plugin: 'nebula.maven-scm'
166164

165+
// we need to apply these again to override the build plugin
166+
targetCompatibility = "10"
167+
sourceCompatibility = "10"
168+
167169
// groovydoc succeeds, but has some weird internal exception...
168170
groovydoc.enabled = false
169171

@@ -176,9 +178,17 @@ if (project != rootProject) {
176178
jarHell.enabled = false
177179
thirdPartyAudit.enabled = false
178180

179-
// tests can't be run with randomized test runner
180-
// it's fine as we run them as part of :buildSrc
181-
test.enabled = false
181+
test {
182+
include "**/*Tests.class"
183+
exclude "**/*IT.class"
184+
// The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test
185+
// with compiler instead on the ones that are too old.
186+
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_10) {
187+
jvm = "${project.compilerJavaHome}/bin/java"
188+
}
189+
}
190+
191+
// This can't be an RandomizedTestingTask because we can't yet reference it
182192
task integTest(type: Test) {
183193
// integration test requires the local testing repo for example plugin builds
184194
dependsOn project.rootProject.allprojects.collect {

‎buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ class TestProgressLogger implements AggregatedEventListener {
125125

126126
@Subscribe
127127
void onTestResult(AggregatedTestResultEvent e) throws IOException {
128-
final String statusMessage
128+
String statusMessage
129129
testsCompleted++
130130
switch (e.status) {
131131
case ERROR:

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

Lines changed: 102 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ import org.gradle.api.tasks.compile.JavaCompile
5151
import org.gradle.api.tasks.javadoc.Javadoc
5252
import org.gradle.internal.jvm.Jvm
5353
import org.gradle.process.ExecResult
54+
import org.gradle.process.ExecSpec
5455
import org.gradle.util.GradleVersion
5556

5657
import java.nio.charset.StandardCharsets
@@ -69,7 +70,7 @@ class BuildPlugin implements Plugin<Project> {
6970
+ 'elasticsearch.standalone-rest-test, and elasticsearch.build '
7071
+ 'are mutually exclusive')
7172
}
72-
final String minimumGradleVersion
73+
String minimumGradleVersion = null
7374
InputStream is = getClass().getResourceAsStream("/minimumGradleVersion")
7475
try { minimumGradleVersion = IOUtils.toString(is, StandardCharsets.UTF_8.toString()) } finally { is.close() }
7576
if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion.trim())) {
@@ -232,8 +233,97 @@ class BuildPlugin implements Plugin<Project> {
232233
project.ext.java9Home = project.rootProject.ext.java9Home
233234
}
234235

236+
static void requireDocker(final Task task) {
237+
final Project rootProject = task.project.rootProject
238+
if (rootProject.hasProperty('requiresDocker') == false) {
239+
/*
240+
* This is our first time encountering a task that requires Docker. We will add an extension that will let us track the tasks
241+
* that register as requiring Docker. We will add a delayed execution that when the task graph is ready if any such tasks are
242+
* in the task graph, then we check two things:
243+
* - the Docker binary is available
244+
* - we can execute a Docker command that requires privileges
245+
*
246+
* If either of these fail, we fail the build.
247+
*/
248+
final boolean buildDocker
249+
final String buildDockerProperty = System.getProperty("build.docker")
250+
if (buildDockerProperty == null || buildDockerProperty == "true") {
251+
buildDocker = true
252+
} else if (buildDockerProperty == "false") {
253+
buildDocker = false
254+
} else {
255+
throw new IllegalArgumentException(
256+
"expected build.docker to be unset or one of \"true\" or \"false\" but was [" + buildDockerProperty + "]")
257+
}
258+
rootProject.rootProject.ext.buildDocker = buildDocker
259+
rootProject.rootProject.ext.requiresDocker = []
260+
rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
261+
// check if the Docker binary exists and record its path
262+
final List<String> maybeDockerBinaries = ['/usr/bin/docker', '/usr/local/bin/docker']
263+
final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
264+
265+
int exitCode
266+
String dockerErrorOutput
267+
if (dockerBinary == null) {
268+
exitCode = -1
269+
dockerErrorOutput = null
270+
} else {
271+
// the Docker binary executes, check that we can execute a privileged command
272+
final ByteArrayOutputStream output = new ByteArrayOutputStream()
273+
final ExecResult result = LoggedExec.exec(rootProject, { ExecSpec it ->
274+
it.commandLine dockerBinary, "images"
275+
it.errorOutput = output
276+
it.ignoreExitValue = true
277+
})
278+
if (result.exitValue == 0) {
279+
return
280+
}
281+
exitCode = result.exitValue
282+
dockerErrorOutput = output.toString()
283+
}
284+
final List<String> tasks =
285+
((List<Task>)rootProject.requiresDocker).findAll { taskGraph.hasTask(it) }.collect { " ${it.path}".toString()}
286+
if (tasks.isEmpty() == false) {
287+
/*
288+
* There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
289+
* exist or because execution of a privileged Docker command failed.
290+
*/
291+
String message
292+
if (dockerBinary == null) {
293+
message = String.format(
294+
Locale.ROOT,
295+
"Docker (checked [%s]) is required to run the following task%s: \n%s",
296+
maybeDockerBinaries.join(","),
297+
tasks.size() > 1 ? "s" : "",
298+
tasks.join('\n'))
299+
} else {
300+
assert exitCode > 0 && dockerErrorOutput != null
301+
message = String.format(
302+
Locale.ROOT,
303+
"a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
304+
"the problem is that Docker exited with exit code [%d] with standard error output [%s]",
305+
dockerBinary,
306+
tasks.size() > 1 ? "s" : "",
307+
tasks.join('\n'),
308+
exitCode,
309+
dockerErrorOutput.trim())
310+
}
311+
throw new GradleException(
312+
message + "\nyou can address this by attending to the reported issue, "
313+
+ "removing the offending tasks from being executed, "
314+
+ "or by passing -Dbuild.docker=false")
315+
}
316+
}
317+
}
318+
if (rootProject.buildDocker) {
319+
rootProject.requiresDocker.add(task)
320+
} else {
321+
task.enabled = false
322+
}
323+
}
324+
235325
private static String findCompilerJavaHome() {
236-
final String compilerJavaHome = System.getenv('JAVA_HOME')
326+
String compilerJavaHome = System.getenv('JAVA_HOME')
237327
final String compilerJavaProperty = System.getProperty('compiler.java')
238328
if (compilerJavaProperty != null) {
239329
compilerJavaHome = findJavaHome(compilerJavaProperty)
@@ -770,13 +860,22 @@ class BuildPlugin implements Plugin<Project> {
770860
}
771861

772862
static void applyCommonTestConfig(Project project) {
773-
project.tasks.withType(RandomizedTestingTask) {
863+
project.tasks.withType(RandomizedTestingTask) {task ->
774864
jvm "${project.runtimeJavaHome}/bin/java"
775865
parallelism System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel)
776866
ifNoTests System.getProperty('tests.ifNoTests', 'fail')
777867
onNonEmptyWorkDirectory 'wipe'
778868
leaveTemporary true
779869

870+
// Make sure all test tasks are configured properly
871+
if (name != "test") {
872+
project.tasks.matching { it.name == "test"}.all { testTask ->
873+
task.testClassesDirs = testTask.testClassesDirs
874+
task.classpath = testTask.classpath
875+
task.shouldRunAfter testTask
876+
}
877+
}
878+
780879
// TODO: why are we not passing maxmemory to junit4?
781880
jvmArg '-Xmx' + System.getProperty('tests.heap.size', '512m')
782881
jvmArg '-Xms' + System.getProperty('tests.heap.size', '512m')

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -190,8 +190,7 @@ class PrecommitTasks {
190190
Task checkstyleTask = project.tasks.create('checkstyle')
191191
// Apply the checkstyle plugin to create `checkstyleMain` and `checkstyleTest`. It only
192192
// creates them if there is main or test code to check and it makes `check` depend
193-
// on them. But we want `precommit` to depend on `checkstyle` which depends on them so
194-
// we have to swap them.
193+
// on them. We also want `precommit` to depend on `checkstyle`.
195194
project.pluginManager.apply('checkstyle')
196195
project.checkstyle {
197196
config = project.resources.text.fromFile(checkstyleConf, 'UTF-8')
@@ -202,7 +201,6 @@ class PrecommitTasks {
202201
}
203202

204203
project.tasks.withType(Checkstyle) { task ->
205-
project.tasks[JavaBasePlugin.CHECK_TASK_NAME].dependsOn.remove(task)
206204
checkstyleTask.dependsOn(task)
207205
task.dependsOn(copyCheckstyleConf)
208206
task.inputs.file(checkstyleSuppressions)
@@ -217,7 +215,7 @@ class PrecommitTasks {
217215
private static Task configureNamingConventions(Project project) {
218216
if (project.sourceSets.findByName("test")) {
219217
Task namingConventionsTask = project.tasks.create('namingConventions', NamingConventionsTask)
220-
namingConventionsTask.javaHome = project.runtimeJavaHome
218+
namingConventionsTask.javaHome = project.compilerJavaHome
221219
return namingConventionsTask
222220
}
223221
return null

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.elasticsearch.gradle.test
2020

21+
import java.util.stream.Collectors
2122
import org.apache.tools.ant.DefaultLogger
2223
import org.apache.tools.ant.taskdefs.condition.Os
2324
import org.elasticsearch.gradle.BuildPlugin
@@ -92,7 +93,8 @@ class ClusterFormationTasks {
9293
throw new Exception("tests.distribution=integ-test-zip is not supported")
9394
}
9495
configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
95-
if (config.numBwcNodes > 0) {
96+
boolean hasBwcNodes = config.numBwcNodes > 0
97+
if (hasBwcNodes) {
9698
if (config.bwcVersion == null) {
9799
throw new IllegalArgumentException("Must specify bwcVersion when numBwcNodes > 0")
98100
}
@@ -111,8 +113,8 @@ class ClusterFormationTasks {
111113
for (int i = 0; i < config.numNodes; i++) {
112114
// we start N nodes and out of these N nodes there might be M bwc nodes.
113115
// for each of those nodes we might have a different configuration
114-
final Configuration distro
115-
final String elasticsearchVersion
116+
Configuration distro
117+
String elasticsearchVersion
116118
if (i < config.numBwcNodes) {
117119
elasticsearchVersion = config.bwcVersion.toString()
118120
if (project.bwcVersions.unreleased.contains(config.bwcVersion)) {
@@ -134,6 +136,16 @@ class ClusterFormationTasks {
134136
esConfig['discovery.zen.hosts_provider'] = 'file'
135137
}
136138
esConfig['discovery.zen.ping.unicast.hosts'] = []
139+
if (hasBwcNodes == false && esConfig['discovery.type'] == null) {
140+
esConfig['discovery.type'] = 'zen2'
141+
esConfig['cluster.initial_master_nodes'] = nodes.stream().map({ n ->
142+
if (n.config.settings['node.name'] == null) {
143+
return "node-" + n.nodeNum
144+
} else {
145+
return n.config.settings['node.name']
146+
}
147+
}).collect(Collectors.toList())
148+
}
137149
esConfig
138150
}
139151
dependsOn = startDependencies
@@ -595,7 +607,7 @@ class ClusterFormationTasks {
595607
}
596608

597609
static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, String pluginName, String prefix) {
598-
final FileCollection pluginZip;
610+
FileCollection pluginZip;
599611
if (node.nodeVersion != Version.fromString(VersionProperties.elasticsearch)) {
600612
pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, pluginName))
601613
} else {

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,24 +19,19 @@
1919
package org.elasticsearch.gradle.test
2020

2121
import com.carrotsearch.gradle.junit4.RandomizedTestingTask
22-
import org.elasticsearch.gradle.BuildPlugin
2322
import org.elasticsearch.gradle.VersionProperties
2423
import org.gradle.api.DefaultTask
25-
import org.gradle.api.Project
2624
import org.gradle.api.Task
2725
import org.gradle.api.execution.TaskExecutionAdapter
28-
import org.gradle.api.internal.tasks.options.Option
29-
import org.gradle.api.provider.Property
30-
import org.gradle.api.provider.Provider
3126
import org.gradle.api.tasks.Copy
3227
import org.gradle.api.tasks.Input
3328
import org.gradle.api.tasks.TaskState
29+
import org.gradle.api.tasks.options.Option
3430
import org.gradle.plugins.ide.idea.IdeaPlugin
3531

3632
import java.nio.charset.StandardCharsets
3733
import java.nio.file.Files
3834
import java.util.stream.Stream
39-
4035
/**
4136
* A wrapper task around setting up a cluster and running rest tests.
4237
*/
@@ -53,7 +48,7 @@ public class RestIntegTestTask extends DefaultTask {
5348

5449
/** Flag indicating whether the rest tests in the rest spec should be run. */
5550
@Input
56-
Property<Boolean> includePackaged = project.objects.property(Boolean)
51+
Boolean includePackaged = false
5752

5853
public RestIntegTestTask() {
5954
runner = project.tasks.create("${name}Runner", RandomizedTestingTask.class)
@@ -109,7 +104,7 @@ public class RestIntegTestTask extends DefaultTask {
109104
}
110105

111106
// copy the rest spec/tests into the test resources
112-
Task copyRestSpec = createCopyRestSpecTask(project, includePackaged)
107+
Task copyRestSpec = createCopyRestSpecTask()
113108
runner.dependsOn(copyRestSpec)
114109

115110
// this must run after all projects have been configured, so we know any project
@@ -130,7 +125,7 @@ public class RestIntegTestTask extends DefaultTask {
130125

131126
/** Sets the includePackaged property */
132127
public void includePackaged(boolean include) {
133-
includePackaged.set(include)
128+
includePackaged = include
134129
}
135130

136131
@Option(
@@ -215,7 +210,7 @@ public class RestIntegTestTask extends DefaultTask {
215210
* @param project The project to add the copy task to
216211
* @param includePackagedTests true if the packaged tests should be copied, false otherwise
217212
*/
218-
static Task createCopyRestSpecTask(Project project, Provider<Boolean> includePackagedTests) {
213+
Task createCopyRestSpecTask() {
219214
project.configurations {
220215
restSpec
221216
}
@@ -237,7 +232,7 @@ public class RestIntegTestTask extends DefaultTask {
237232
project.afterEvaluate {
238233
copyRestSpec.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
239234
include 'rest-api-spec/api/**'
240-
if (includePackagedTests.get()) {
235+
if (includePackaged) {
241236
include 'rest-api-spec/test/**'
242237
}
243238
}

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package org.elasticsearch.gradle.test
22

33
import org.gradle.api.DefaultTask
44
import org.gradle.api.Task
5-
import org.gradle.api.internal.tasks.options.Option
5+
import org.gradle.api.tasks.options.Option
66
import org.gradle.util.ConfigureUtil
77

88
public class RunTask extends DefaultTask {

0 commit comments

Comments
 (0)