Skip to content

Commit 68bc59c

Browse files
dimitris-athanasioudimitris-athanasiou
committed
Merge branch 'master' into feature-ml-data-frame-analytics
2 parents 801665a + 1d458e3 commit 68bc59c

File tree

830 files changed

+24388
-8273
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

830 files changed

+24388
-8273
lines changed

‎.ci/packer_cache.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ while [ -h "$SCRIPT" ] ; do
1616
done
1717

1818
source $(dirname "${SCRIPT}")/java-versions.properties
19-
JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} ./gradlew resolveAllDependencies --parallel
19+
JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} ./gradlew --parallel resolveAllDependencies composePull

‎TESTING.asciidoc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,9 @@ If you want to just run the precommit checks:
251251
./gradlew precommit
252252
---------------------------------------------------------------------------
253253

254+
Some of these checks will require `docker-compose` installed for bringing up
255+
test fixtures. If it's not present those checks will be skipped automatically.
256+
254257
== Testing the REST layer
255258

256259
The available integration tests make use of the java API to communicate with

‎buildSrc/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ dependencies {
123123
compile "org.elasticsearch:jna:4.5.1"
124124
compile 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
125125
compile 'de.thetaphi:forbiddenapis:2.6'
126+
compile 'com.avast.gradle:docker-compose-gradle-plugin:0.4.5'
126127
testCompile "junit:junit:${props.getProperty('junit')}"
127128
}
128129

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

Lines changed: 37 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,7 @@ class BuildPlugin implements Plugin<Project> {
215215
project.rootProject.ext.inFipsJvm = inFipsJvm
216216
project.rootProject.ext.gradleJavaVersion = JavaVersion.toVersion(gradleJavaVersion)
217217
project.rootProject.ext.java9Home = "${-> findJavaHome("9")}"
218+
project.rootProject.ext.defaultParallel = findDefaultParallel(project.rootProject)
218219
}
219220

220221
project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
@@ -773,32 +774,9 @@ class BuildPlugin implements Plugin<Project> {
773774
}
774775

775776
static void applyCommonTestConfig(Project project) {
776-
String defaultParallel = 'auto'
777-
// Count physical cores on any Linux distro ( don't count hyper-threading )
778-
if (project.file("/proc/cpuinfo").exists()) {
779-
Map<String, Integer> socketToCore = [:]
780-
String currentID = ""
781-
project.file("/proc/cpuinfo").readLines().forEach({ line ->
782-
if (line.contains(":")) {
783-
List<String> parts = line.split(":", 2).collect({it.trim()})
784-
String name = parts[0], value = parts[1]
785-
// the ID of the CPU socket
786-
if (name == "physical id") {
787-
currentID = value
788-
}
789-
// Number of cores not including hyper-threading
790-
if (name == "cpu cores") {
791-
assert currentID.isEmpty() == false
792-
socketToCore[currentID] = Integer.valueOf(value)
793-
currentID = ""
794-
}
795-
}
796-
})
797-
defaultParallel = socketToCore.values().sum().toString();
798-
}
799777
project.tasks.withType(RandomizedTestingTask) {
800778
jvm "${project.runtimeJavaHome}/bin/java"
801-
parallelism System.getProperty('tests.jvms', defaultParallel)
779+
parallelism System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel)
802780
ifNoTests System.getProperty('tests.ifNoTests', 'fail')
803781
onNonEmptyWorkDirectory 'wipe'
804782
leaveTemporary true
@@ -902,6 +880,41 @@ class BuildPlugin implements Plugin<Project> {
902880
}
903881
}
904882

883+
private static String findDefaultParallel(Project project) {
884+
if (project.file("/proc/cpuinfo").exists()) {
885+
// Count physical cores on any Linux distro ( don't count hyper-threading )
886+
Map<String, Integer> socketToCore = [:]
887+
String currentID = ""
888+
project.file("/proc/cpuinfo").readLines().forEach({ line ->
889+
if (line.contains(":")) {
890+
List<String> parts = line.split(":", 2).collect({it.trim()})
891+
String name = parts[0], value = parts[1]
892+
// the ID of the CPU socket
893+
if (name == "physical id") {
894+
currentID = value
895+
}
896+
// Number of cores not including hyper-threading
897+
if (name == "cpu cores") {
898+
assert currentID.isEmpty() == false
899+
socketToCore[currentID] = Integer.valueOf(value)
900+
currentID = ""
901+
}
902+
}
903+
})
904+
return socketToCore.values().sum().toString();
905+
} else if ('Mac OS X'.equals(System.getProperty('os.name'))) {
906+
// Ask macOS to count physical CPUs for us
907+
ByteArrayOutputStream stdout = new ByteArrayOutputStream()
908+
project.exec {
909+
executable 'sysctl'
910+
args '-n', 'hw.physicalcpu'
911+
standardOutput = stdout
912+
}
913+
return stdout.toString('UTF-8').trim();
914+
}
915+
return 'auto';
916+
}
917+
905918
/** Configures the test task */
906919
static Task configureTest(Project project) {
907920
project.tasks.getByName('test') {

‎buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixtureExtension.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.gradle.testfixtures;
20+
21+
import org.gradle.api.NamedDomainObjectContainer;
22+
import org.gradle.api.Project;
23+
24+
public class TestFixtureExtension {
25+
26+
private final Project project;
27+
final NamedDomainObjectContainer<Project> fixtures;
28+
29+
public TestFixtureExtension(Project project) {
30+
this.project = project;
31+
this.fixtures = project.container(Project.class);
32+
}
33+
34+
public void useFixture(String path) {
35+
Project fixtureProject = this.project.findProject(path);
36+
if (fixtureProject == null) {
37+
throw new IllegalArgumentException("Could not find test fixture " + fixtureProject);
38+
}
39+
if (fixtureProject.file(TestFixturesPlugin.DOCKER_COMPOSE_YML).exists() == false) {
40+
throw new IllegalArgumentException(
41+
"Project " + path + " is not a valid test fixture: missing " + TestFixturesPlugin.DOCKER_COMPOSE_YML
42+
);
43+
}
44+
fixtures.add(fixtureProject);
45+
}
46+
47+
48+
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.gradle.testfixtures;
20+
21+
import com.avast.gradle.dockercompose.ComposeExtension;
22+
import com.avast.gradle.dockercompose.DockerComposePlugin;
23+
import org.elasticsearch.gradle.precommit.JarHellTask;
24+
import org.elasticsearch.gradle.precommit.ThirdPartyAuditTask;
25+
import org.gradle.api.DefaultTask;
26+
import org.gradle.api.Plugin;
27+
import org.gradle.api.Project;
28+
import org.gradle.api.Task;
29+
import org.gradle.api.plugins.BasePlugin;
30+
import org.gradle.api.tasks.Input;
31+
import org.gradle.api.tasks.TaskContainer;
32+
33+
import java.lang.reflect.InvocationTargetException;
34+
import java.lang.reflect.Method;
35+
import java.util.Collections;
36+
37+
public class TestFixturesPlugin implements Plugin<Project> {
38+
39+
static final String DOCKER_COMPOSE_YML = "docker-compose.yml";
40+
41+
@Override
42+
public void apply(Project project) {
43+
TaskContainer tasks = project.getTasks();
44+
45+
TestFixtureExtension extension = project.getExtensions().create(
46+
"testFixtures", TestFixtureExtension.class, project
47+
);
48+
49+
if (project.file(DOCKER_COMPOSE_YML).exists()) {
50+
// convenience boilerplate with build plugin
51+
// Can't reference tasks that are implemented in Groovy, use reflection instead
52+
disableTaskByType(tasks, getTaskClass("org.elasticsearch.gradle.precommit.LicenseHeadersTask"));
53+
disableTaskByType(tasks, getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"));
54+
disableTaskByType(tasks, ThirdPartyAuditTask.class);
55+
disableTaskByType(tasks, JarHellTask.class);
56+
57+
if (dockerComposeSupported(project) == false) {
58+
return;
59+
}
60+
61+
project.apply(spec -> spec.plugin(BasePlugin.class));
62+
project.apply(spec -> spec.plugin(DockerComposePlugin.class));
63+
ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
64+
composeExtension.setUseComposeFiles(Collections.singletonList(DOCKER_COMPOSE_YML));
65+
composeExtension.setRemoveContainers(true);
66+
composeExtension.setExecutable(
67+
project.file("/usr/local/bin/docker-compose").exists() ?
68+
"/usr/local/bin/docker-compose" : "/usr/bin/docker-compose"
69+
);
70+
71+
project.getTasks().getByName("clean").dependsOn("composeDown");
72+
} else {
73+
if (dockerComposeSupported(project) == false) {
74+
project.getLogger().warn(
75+
"Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " +
76+
"but none could not be found so these will be skipped", project.getPath()
77+
);
78+
tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->
79+
task.setEnabled(false)
80+
);
81+
return;
82+
}
83+
tasks.withType(getTaskClass("com.carrotsearch.gradle.junit4.RandomizedTestingTask"), task ->
84+
extension.fixtures.all(fixtureProject -> {
85+
task.dependsOn(fixtureProject.getTasks().getByName("composeUp"));
86+
task.finalizedBy(fixtureProject.getTasks().getByName("composeDown"));
87+
// Configure ports for the tests as system properties.
88+
// We only know these at execution time so we need to do it in doFirst
89+
task.doFirst(it ->
90+
fixtureProject.getExtensions().getByType(ComposeExtension.class).getServicesInfos()
91+
.forEach((service, infos) ->
92+
infos.getPorts()
93+
.forEach((container, host) -> setSystemProperty(
94+
it,
95+
"test.fixtures." + fixtureProject.getName() + "." + service + "." + container,
96+
host
97+
))
98+
));
99+
}));
100+
}
101+
}
102+
103+
@Input
104+
public boolean dockerComposeSupported(Project project) {
105+
// Don't look for docker-compose on the PATH yet that would pick up on Windows as well
106+
return
107+
project.file("/usr/local/bin/docker-compose").exists() == false &&
108+
project.file("/usr/bin/docker-compose").exists() == false &&
109+
Boolean.parseBoolean(System.getProperty("tests.fixture.enabled", "true")) == false;
110+
}
111+
112+
private void setSystemProperty(Task task, String name, Object value) {
113+
try {
114+
Method systemProperty = task.getClass().getMethod("systemProperty", String.class, Object.class);
115+
systemProperty.invoke(task, name, value);
116+
} catch (NoSuchMethodException e) {
117+
throw new IllegalArgumentException("Could not find systemProperty method on RandomizedTestingTask", e);
118+
} catch (IllegalAccessException | InvocationTargetException e) {
119+
throw new IllegalArgumentException("Could not call systemProperty method on RandomizedTestingTask", e);
120+
}
121+
}
122+
123+
private void disableTaskByType(TaskContainer tasks, Class<? extends Task> type) {
124+
tasks.withType(type, task -> task.setEnabled(false));
125+
}
126+
127+
@SuppressWarnings("unchecked")
128+
private Class<? extends DefaultTask> getTaskClass(String type) {
129+
Class<?> aClass;
130+
try {
131+
aClass = Class.forName(type);
132+
if (DefaultTask.class.isAssignableFrom(aClass) == false) {
133+
throw new IllegalArgumentException("Not a task type: " + type);
134+
}
135+
} catch (ClassNotFoundException e) {
136+
throw new IllegalArgumentException("No such task: " + type);
137+
}
138+
return (Class<? extends DefaultTask>) aClass;
139+
}
140+
141+
}

‎buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test.fixtures.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
implementation-class=org.elasticsearch.gradle.testfixtures.TestFixturesPlugin

‎buildSrc/src/main/resources/eclipse.settings/org.eclipse.jdt.core.prefs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ eclipse.preferences.version=1
1616
# org.eclipse.jdt.core.compiler.problem.potentialNullReference=warning
1717

1818
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
19+
org.eclipse.jdt.core.formatter.comment.line_length=140
1920
org.eclipse.jdt.core.formatter.lineSplit=140
2021
org.eclipse.jdt.core.formatter.tabulation.char=space
2122
org.eclipse.jdt.core.formatter.tabulation.size=4

‎buildSrc/version.properties

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@ slf4j = 1.6.2
1515
# when updating the JNA version, also update the version in buildSrc/build.gradle
1616
jna = 4.5.1
1717

18-
netty = 4.1.30.Final
18+
netty = 4.1.31.Final
1919
joda = 2.10.1
2020

2121
# test dependencies
22-
randomizedrunner = 2.7.0
22+
randomizedrunner = 2.7.1
2323
junit = 4.12
2424
httpclient = 4.5.2
2525
# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy

0 commit comments

Comments
 (0)