Skip to content

Commit ce68d75

Browse files
committed
Merge remote-tracking branch 'elastic/master' into use-plugins-for-node-tool
2 parents d2eef8e + fffba25 commit ce68d75

File tree

739 files changed

+21106
-7015
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

739 files changed

+21106
-7015
lines changed

CONTRIBUTING.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,9 @@ and `JAVA11_HOME` available so that the tests can pass.
107107
`jrunscript` for jdk distributions.
108108

109109
Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
110-
using the wrapper via the `gradlew` script in the root of the repository.
110+
using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
111+
script on Windows in the root of the repository. The examples below show the
112+
usage on Unix.
111113

112114
We support development in the Eclipse and IntelliJ IDEs.
113115
For Eclipse, the minimum version that we support is [4.13][eclipse].

buildSrc/build.gradle

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
* under the License.
1818
*/
1919

20+
import org.gradle.internal.jvm.Jvm
2021
import org.gradle.util.GradleVersion
2122

2223
plugins {
@@ -168,6 +169,9 @@ if (project != rootProject) {
168169
forbiddenApisTest.enabled = false
169170
jarHell.enabled = false
170171
thirdPartyAudit.enabled = false
172+
if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))){
173+
test.enabled = false
174+
}
171175

172176
configurations {
173177
distribution
@@ -223,8 +227,13 @@ if (project != rootProject) {
223227
}
224228
check.dependsOn(integTest)
225229

230+
// for now we hardcode the tests for our build to use the gradle jvm.
231+
tasks.withType(Test).configureEach {
232+
it.executable = Jvm.current().getJavaExecutable()
233+
}
234+
226235
/*
227-
* We alread configure publication and we don't need or want this one that
236+
* We already configure publication and we don't need or want this one that
228237
* comes from the java-gradle-plugin.
229238
*/
230239
afterEvaluate {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -720,6 +720,9 @@ class BuildPlugin implements Plugin<Project> {
720720
// TODO: remove this once ctx isn't added to update script params in 7.0
721721
test.systemProperty 'es.scripting.update.ctx_in_params', 'false'
722722

723+
// TODO: remove this property in 8.0
724+
test.systemProperty 'es.search.rewrite_sort', 'true'
725+
723726
// TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
724727
test.systemProperty 'es.transport.cname_in_publish_address', 'true'
725728

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ class SnippetsTask extends DefaultTask {
6666
exclude 'build.gradle'
6767
// That is where the snippets go, not where they come from!
6868
exclude 'build'
69+
exclude 'build-idea'
70+
exclude 'build-eclipse'
6971
}
7072

7173
/**

buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,12 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
4141
private final RegularFileProperty outputFile;
4242
private final RegularFileProperty compilerVersionFile;
4343
private final RegularFileProperty runtimeVersionFile;
44-
private final RegularFileProperty fipsJvmFile;
4544

4645
@Inject
4746
public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) {
4847
this.outputFile = objectFactory.fileProperty();
4948
this.compilerVersionFile = objectFactory.fileProperty();
5049
this.runtimeVersionFile = objectFactory.fileProperty();
51-
this.fipsJvmFile = objectFactory.fileProperty();
5250
}
5351

5452
@Input
@@ -113,11 +111,6 @@ public RegularFileProperty getRuntimeVersionFile() {
113111
return runtimeVersionFile;
114112
}
115113

116-
@OutputFile
117-
public RegularFileProperty getFipsJvmFile() {
118-
return fipsJvmFile;
119-
}
120-
121114
@TaskAction
122115
public void generate() {
123116
String javaVendorVersion = System.getProperty("java.vendor.version", System.getProperty("java.vendor"));
@@ -130,7 +123,6 @@ public void generate() {
130123
String runtimeJavaVersionDetails = gradleJavaVersionDetails;
131124
JavaVersion runtimeJavaVersionEnum = JavaVersion.current();
132125
File gradleJavaHome = Jvm.current().getJavaHome();
133-
boolean inFipsJvm = false;
134126

135127
try {
136128
if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
@@ -146,8 +138,6 @@ public void generate() {
146138
if (runtimeJavaHome.exists()) {
147139
runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome);
148140
runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome));
149-
150-
inFipsJvm = Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
151141
} else {
152142
throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist");
153143
}
@@ -213,7 +203,6 @@ public void generate() {
213203

214204
writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name());
215205
writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name());
216-
writeToFile(fipsJvmFile.getAsFile().get(), Boolean.toString(inFipsJvm));
217206
}
218207

219208
private void writeToFile(File file, String content) {

buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,14 +76,12 @@ public void apply(Project project) {
7676
task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info"));
7777
task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version"));
7878
task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version"));
79-
task.getFipsJvmFile().set(new File(project.getBuildDir(), "in-fips-jvm"));
8079
});
8180

8281
PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> {
8382
task.getBuildInfoFile().set(generateTask.getOutputFile());
8483
task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile());
8584
task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile());
86-
task.getFipsJvmFile().set(generateTask.getFipsJvmFile());
8785
task.setGlobalInfoListeners(extension.listeners);
8886
});
8987

@@ -103,6 +101,7 @@ public void apply(Project project) {
103101
params.setIsCi(System.getenv("JENKINS_URL") != null);
104102
params.setIsInternal(GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null);
105103
params.setDefaultParallel(findDefaultParallel(project));
104+
params.setInFipsJvm(isInFipsJvm());
106105
});
107106

108107
project.allprojects(p -> {
@@ -153,6 +152,10 @@ private static String getJavaHomeEnvVarName(String version) {
153152
return "JAVA" + version + "_HOME";
154153
}
155154

155+
private static boolean isInFipsJvm() {
156+
return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
157+
}
158+
156159
private static String getResourceContents(String resourcePath) {
157160
try (BufferedReader reader = new BufferedReader(
158161
new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath))

buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,13 @@ public class PrintGlobalBuildInfoTask extends DefaultTask {
1616
private final RegularFileProperty buildInfoFile;
1717
private final RegularFileProperty compilerVersionFile;
1818
private final RegularFileProperty runtimeVersionFile;
19-
private final RegularFileProperty fipsJvmFile;
2019
private List<Runnable> globalInfoListeners = new ArrayList<>();
2120

2221
@Inject
2322
public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) {
2423
this.buildInfoFile = objectFactory.fileProperty();
2524
this.compilerVersionFile = objectFactory.fileProperty();
2625
this.runtimeVersionFile = objectFactory.fileProperty();
27-
this.fipsJvmFile = objectFactory.fileProperty();
2826
}
2927

3028
@InputFile
@@ -42,11 +40,6 @@ public RegularFileProperty getRuntimeVersionFile() {
4240
return runtimeVersionFile;
4341
}
4442

45-
@InputFile
46-
public RegularFileProperty getFipsJvmFile() {
47-
return fipsJvmFile;
48-
}
49-
5043
public void setGlobalInfoListeners(List<Runnable> globalInfoListeners) {
5144
this.globalInfoListeners = globalInfoListeners;
5245
}
@@ -57,6 +50,7 @@ public void print() {
5750
getLogger().quiet("Elasticsearch Build Hamster says Hello!");
5851
getLogger().quiet(getFileText(getBuildInfoFile()).asString());
5952
getLogger().quiet(" Random Testing Seed : " + BuildParams.getTestSeed());
53+
getLogger().quiet(" In FIPS 140 mode : " + BuildParams.isInFipsJvm());
6054
getLogger().quiet("=======================================");
6155

6256
setGlobalProperties();
@@ -76,7 +70,6 @@ private void setGlobalProperties() {
7670
BuildParams.init(params -> {
7771
params.setCompilerJavaVersion(JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString()));
7872
params.setRuntimeJavaVersion(JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString()));
79-
params.setInFipsJvm(Boolean.parseBoolean(getFileText(getFipsJvmFile()).asString()));
8073
});
8174
}
8275
}

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,11 @@ public synchronized void start() {
415415
} catch (IOException e) {
416416
throw new UncheckedIOException("Failed to create working directory for " + this, e);
417417
}
418+
419+
copyExtraJars();
420+
421+
copyExtraConfigFiles();
422+
418423
createConfiguration();
419424

420425
if (plugins.isEmpty() == false) {
@@ -453,10 +458,6 @@ public synchronized void start() {
453458

454459
installModules();
455460

456-
copyExtraConfigFiles();
457-
458-
copyExtraJars();
459-
460461
if (isSettingTrue("xpack.security.enabled")) {
461462
if (credentials.isEmpty()) {
462463
user(Collections.emptyMap());
@@ -677,6 +678,10 @@ private Map<String, String> getESEnvironment() {
677678
if (systemProperties.isEmpty() == false) {
678679
systemPropertiesString = " " + systemProperties.entrySet().stream()
679680
.map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
681+
// ES_PATH_CONF is also set as an environment variable and for a reference to ${ES_PATH_CONF}
682+
// to work ES_JAVA_OPTS, we need to make sure that ES_PATH_CONF before ES_JAVA_OPTS. Instead,
683+
// we replace the reference with the actual value in other environment variables
684+
.map(p -> p.replace("${ES_PATH_CONF}", configFile.getParent().toString()))
680685
.collect(Collectors.joining(" "));
681686
}
682687
String jvmArgsString = "";

buildSrc/version.properties

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,8 @@ joda = 2.10.4
2929
# - distribution/tools/plugin-cli
3030
# - x-pack/plugin/security
3131
bouncycastle = 1.61
32-
3332
# test dependencies
34-
randomizedrunner = 2.7.1
33+
randomizedrunner = 2.7.4
3534
junit = 4.12
3635
httpclient = 4.5.10
3736
httpcore = 4.4.12

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import org.elasticsearch.client.RequestConverters.EndpointBuilder;
3030
import org.elasticsearch.client.core.PageParams;
3131
import org.elasticsearch.client.ml.CloseJobRequest;
32+
import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
33+
import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
3234
import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
3335
import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
3436
import org.elasticsearch.client.ml.DeleteCalendarRequest;
@@ -58,6 +60,8 @@
5860
import org.elasticsearch.client.ml.GetModelSnapshotsRequest;
5961
import org.elasticsearch.client.ml.GetOverallBucketsRequest;
6062
import org.elasticsearch.client.ml.GetRecordsRequest;
63+
import org.elasticsearch.client.ml.GetTrainedModelsRequest;
64+
import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest;
6165
import org.elasticsearch.client.ml.MlInfoRequest;
6266
import org.elasticsearch.client.ml.OpenJobRequest;
6367
import org.elasticsearch.client.ml.PostCalendarEventRequest;
@@ -700,15 +704,85 @@ static Request evaluateDataFrame(EvaluateDataFrameRequest evaluateRequest) throw
700704
return request;
701705
}
702706

703-
static Request estimateMemoryUsage(PutDataFrameAnalyticsRequest estimateRequest) throws IOException {
707+
static Request explainDataFrameAnalytics(ExplainDataFrameAnalyticsRequest explainRequest) throws IOException {
708+
EndpointBuilder endpoint = new EndpointBuilder().addPathPartAsIs("_ml", "data_frame", "analytics");
709+
if (explainRequest.getId() != null) {
710+
endpoint.addPathPart(explainRequest.getId());
711+
}
712+
endpoint.addPathPartAsIs("_explain");
713+
714+
Request request = new Request(HttpPost.METHOD_NAME, endpoint.build());
715+
if (explainRequest.getConfig() != null) {
716+
request.setEntity(createEntity(explainRequest.getConfig(), REQUEST_BODY_CONTENT_TYPE));
717+
}
718+
return request;
719+
}
720+
721+
static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) {
704722
String endpoint = new EndpointBuilder()
705-
.addPathPartAsIs("_ml", "data_frame", "analytics", "_estimate_memory_usage")
723+
.addPathPartAsIs("_ml", "inference")
724+
.addPathPart(Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getIds()))
706725
.build();
707-
Request request = new Request(HttpPost.METHOD_NAME, endpoint);
708-
request.setEntity(createEntity(estimateRequest, REQUEST_BODY_CONTENT_TYPE));
726+
RequestConverters.Params params = new RequestConverters.Params();
727+
if (getTrainedModelsRequest.getPageParams() != null) {
728+
PageParams pageParams = getTrainedModelsRequest.getPageParams();
729+
if (pageParams.getFrom() != null) {
730+
params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString());
731+
}
732+
if (pageParams.getSize() != null) {
733+
params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString());
734+
}
735+
}
736+
if (getTrainedModelsRequest.getAllowNoMatch() != null) {
737+
params.putParam(GetTrainedModelsRequest.ALLOW_NO_MATCH,
738+
Boolean.toString(getTrainedModelsRequest.getAllowNoMatch()));
739+
}
740+
if (getTrainedModelsRequest.getDecompressDefinition() != null) {
741+
params.putParam(GetTrainedModelsRequest.DECOMPRESS_DEFINITION,
742+
Boolean.toString(getTrainedModelsRequest.getDecompressDefinition()));
743+
}
744+
if (getTrainedModelsRequest.getIncludeDefinition() != null) {
745+
params.putParam(GetTrainedModelsRequest.INCLUDE_MODEL_DEFINITION,
746+
Boolean.toString(getTrainedModelsRequest.getIncludeDefinition()));
747+
}
748+
Request request = new Request(HttpGet.METHOD_NAME, endpoint);
749+
request.addParameters(params.asMap());
750+
return request;
751+
}
752+
753+
static Request getTrainedModelsStats(GetTrainedModelsStatsRequest getTrainedModelsStatsRequest) {
754+
String endpoint = new EndpointBuilder()
755+
.addPathPartAsIs("_ml", "inference")
756+
.addPathPart(Strings.collectionToCommaDelimitedString(getTrainedModelsStatsRequest.getIds()))
757+
.addPathPart("_stats")
758+
.build();
759+
RequestConverters.Params params = new RequestConverters.Params();
760+
if (getTrainedModelsStatsRequest.getPageParams() != null) {
761+
PageParams pageParams = getTrainedModelsStatsRequest.getPageParams();
762+
if (pageParams.getFrom() != null) {
763+
params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString());
764+
}
765+
if (pageParams.getSize() != null) {
766+
params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString());
767+
}
768+
}
769+
if (getTrainedModelsStatsRequest.getAllowNoMatch() != null) {
770+
params.putParam(GetTrainedModelsStatsRequest.ALLOW_NO_MATCH,
771+
Boolean.toString(getTrainedModelsStatsRequest.getAllowNoMatch()));
772+
}
773+
Request request = new Request(HttpGet.METHOD_NAME, endpoint);
774+
request.addParameters(params.asMap());
709775
return request;
710776
}
711777

778+
static Request deleteTrainedModel(DeleteTrainedModelRequest deleteRequest) {
779+
String endpoint = new EndpointBuilder()
780+
.addPathPartAsIs("_ml", "inference")
781+
.addPathPart(deleteRequest.getId())
782+
.build();
783+
return new Request(HttpDelete.METHOD_NAME, endpoint);
784+
}
785+
712786
static Request putFilter(PutFilterRequest putFilterRequest) throws IOException {
713787
String endpoint = new EndpointBuilder()
714788
.addPathPartAsIs("_ml")

0 commit comments

Comments
 (0)