Skip to content

Commit 459b66c

Browse files
henningandersenhenningandersen
committed
Merge remote-tracking branch 'origin/master' into deprecate_reindex_sort
2 parents 5c49c63 + e9941c2 commit 459b66c

File tree

108 files changed

+2685
-1120
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+2685
-1120
lines changed

‎buildSrc/build.gradle

+10-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
* under the License.
1818
*/
1919

20+
import org.gradle.internal.jvm.Jvm
2021
import org.gradle.util.GradleVersion
2122

2223
plugins {
@@ -168,6 +169,9 @@ if (project != rootProject) {
168169
forbiddenApisTest.enabled = false
169170
jarHell.enabled = false
170171
thirdPartyAudit.enabled = false
172+
if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))){
173+
test.enabled = false
174+
}
171175

172176
configurations {
173177
distribution
@@ -223,8 +227,13 @@ if (project != rootProject) {
223227
}
224228
check.dependsOn(integTest)
225229

230+
// for now we hardcode the tests for our build to use the gradle jvm.
231+
tasks.withType(Test).configureEach {
232+
it.executable = Jvm.current().getJavaExecutable()
233+
}
234+
226235
/*
227-
* We alread configure publication and we don't need or want this one that
236+
* We already configure publication and we don't need or want this one that
228237
* comes from the java-gradle-plugin.
229238
*/
230239
afterEvaluate {

‎buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java

-11
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,12 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
4141
private final RegularFileProperty outputFile;
4242
private final RegularFileProperty compilerVersionFile;
4343
private final RegularFileProperty runtimeVersionFile;
44-
private final RegularFileProperty fipsJvmFile;
4544

4645
@Inject
4746
public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) {
4847
this.outputFile = objectFactory.fileProperty();
4948
this.compilerVersionFile = objectFactory.fileProperty();
5049
this.runtimeVersionFile = objectFactory.fileProperty();
51-
this.fipsJvmFile = objectFactory.fileProperty();
5250
}
5351

5452
@Input
@@ -113,11 +111,6 @@ public RegularFileProperty getRuntimeVersionFile() {
113111
return runtimeVersionFile;
114112
}
115113

116-
@OutputFile
117-
public RegularFileProperty getFipsJvmFile() {
118-
return fipsJvmFile;
119-
}
120-
121114
@TaskAction
122115
public void generate() {
123116
String javaVendorVersion = System.getProperty("java.vendor.version", System.getProperty("java.vendor"));
@@ -130,7 +123,6 @@ public void generate() {
130123
String runtimeJavaVersionDetails = gradleJavaVersionDetails;
131124
JavaVersion runtimeJavaVersionEnum = JavaVersion.current();
132125
File gradleJavaHome = Jvm.current().getJavaHome();
133-
boolean inFipsJvm = false;
134126

135127
try {
136128
if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
@@ -146,8 +138,6 @@ public void generate() {
146138
if (runtimeJavaHome.exists()) {
147139
runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome);
148140
runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome));
149-
150-
inFipsJvm = Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
151141
} else {
152142
throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist");
153143
}
@@ -213,7 +203,6 @@ public void generate() {
213203

214204
writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name());
215205
writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name());
216-
writeToFile(fipsJvmFile.getAsFile().get(), Boolean.toString(inFipsJvm));
217206
}
218207

219208
private void writeToFile(File file, String content) {

‎buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -76,14 +76,12 @@ public void apply(Project project) {
7676
task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info"));
7777
task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version"));
7878
task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version"));
79-
task.getFipsJvmFile().set(new File(project.getBuildDir(), "in-fips-jvm"));
8079
});
8180

8281
PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> {
8382
task.getBuildInfoFile().set(generateTask.getOutputFile());
8483
task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile());
8584
task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile());
86-
task.getFipsJvmFile().set(generateTask.getFipsJvmFile());
8785
task.setGlobalInfoListeners(extension.listeners);
8886
});
8987

@@ -103,6 +101,7 @@ public void apply(Project project) {
103101
params.setIsCi(System.getenv("JENKINS_URL") != null);
104102
params.setIsInternal(GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null);
105103
params.setDefaultParallel(findDefaultParallel(project));
104+
params.setInFipsJvm(isInFipsJvm());
106105
});
107106

108107
project.allprojects(p -> {
@@ -153,6 +152,10 @@ private static String getJavaHomeEnvVarName(String version) {
153152
return "JAVA" + version + "_HOME";
154153
}
155154

155+
private static boolean isInFipsJvm() {
156+
return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
157+
}
158+
156159
private static String getResourceContents(String resourcePath) {
157160
try (BufferedReader reader = new BufferedReader(
158161
new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath))

‎buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java

+1-8
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,13 @@ public class PrintGlobalBuildInfoTask extends DefaultTask {
1616
private final RegularFileProperty buildInfoFile;
1717
private final RegularFileProperty compilerVersionFile;
1818
private final RegularFileProperty runtimeVersionFile;
19-
private final RegularFileProperty fipsJvmFile;
2019
private List<Runnable> globalInfoListeners = new ArrayList<>();
2120

2221
@Inject
2322
public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) {
2423
this.buildInfoFile = objectFactory.fileProperty();
2524
this.compilerVersionFile = objectFactory.fileProperty();
2625
this.runtimeVersionFile = objectFactory.fileProperty();
27-
this.fipsJvmFile = objectFactory.fileProperty();
2826
}
2927

3028
@InputFile
@@ -42,11 +40,6 @@ public RegularFileProperty getRuntimeVersionFile() {
4240
return runtimeVersionFile;
4341
}
4442

45-
@InputFile
46-
public RegularFileProperty getFipsJvmFile() {
47-
return fipsJvmFile;
48-
}
49-
5043
public void setGlobalInfoListeners(List<Runnable> globalInfoListeners) {
5144
this.globalInfoListeners = globalInfoListeners;
5245
}
@@ -57,6 +50,7 @@ public void print() {
5750
getLogger().quiet("Elasticsearch Build Hamster says Hello!");
5851
getLogger().quiet(getFileText(getBuildInfoFile()).asString());
5952
getLogger().quiet(" Random Testing Seed : " + BuildParams.getTestSeed());
53+
getLogger().quiet(" In FIPS 140 mode : " + BuildParams.isInFipsJvm());
6054
getLogger().quiet("=======================================");
6155

6256
setGlobalProperties();
@@ -76,7 +70,6 @@ private void setGlobalProperties() {
7670
BuildParams.init(params -> {
7771
params.setCompilerJavaVersion(JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString()));
7872
params.setRuntimeJavaVersion(JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString()));
79-
params.setInFipsJvm(Boolean.parseBoolean(getFileText(getFipsJvmFile()).asString()));
8073
});
8174
}
8275
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

+12-7
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,11 @@ public synchronized void start() {
415415
} catch (IOException e) {
416416
throw new UncheckedIOException("Failed to create working directory for " + this, e);
417417
}
418+
419+
copyExtraJars();
420+
421+
copyExtraConfigFiles();
422+
418423
createConfiguration();
419424

420425
if (plugins.isEmpty() == false) {
@@ -438,7 +443,7 @@ public synchronized void start() {
438443
runElaticsearchBinScript("elasticsearch-keystore", "create");
439444

440445
keystoreSettings.forEach((key, value) ->
441-
runElaticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
446+
runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
442447
);
443448

444449
for (Map.Entry<String, File> entry : keystoreFiles.entrySet()) {
@@ -453,10 +458,6 @@ public synchronized void start() {
453458

454459
installModules();
455460

456-
copyExtraConfigFiles();
457-
458-
copyExtraJars();
459-
460461
if (isSettingTrue("xpack.security.enabled")) {
461462
if (credentials.isEmpty()) {
462463
user(Collections.emptyMap());
@@ -622,7 +623,7 @@ public void user(Map<String, String> userSpec) {
622623
credentials.add(cred);
623624
}
624625

625-
private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) {
626+
private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
626627
if (
627628
Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false &&
628629
Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false
@@ -663,7 +664,7 @@ private void runElaticsearchBinScriptWithInput(String input, String tool, String
663664
}
664665

665666
private void runElaticsearchBinScript(String tool, String... args) {
666-
runElaticsearchBinScriptWithInput("", tool, args);
667+
runElasticsearchBinScriptWithInput("", tool, args);
667668
}
668669

669670
private Map<String, String> getESEnvironment() {
@@ -676,6 +677,10 @@ private Map<String, String> getESEnvironment() {
676677
if (systemProperties.isEmpty() == false) {
677678
systemPropertiesString = " " + systemProperties.entrySet().stream()
678679
.map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
680+
// ES_PATH_CONF is also set as an environment variable and for a reference to ${ES_PATH_CONF}
681+
// to work ES_JAVA_OPTS, we need to make sure that ES_PATH_CONF before ES_JAVA_OPTS. Instead,
682+
// we replace the reference with the actual value in other environment variables
683+
.map(p -> p.replace("${ES_PATH_CONF}", configFile.getParent().toString()))
679684
.collect(Collectors.joining(" "));
680685
}
681686
String jvmArgsString = "";

‎client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

+33
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import org.elasticsearch.client.ml.GetModelSnapshotsRequest;
5959
import org.elasticsearch.client.ml.GetOverallBucketsRequest;
6060
import org.elasticsearch.client.ml.GetRecordsRequest;
61+
import org.elasticsearch.client.ml.GetTrainedModelsRequest;
6162
import org.elasticsearch.client.ml.MlInfoRequest;
6263
import org.elasticsearch.client.ml.OpenJobRequest;
6364
import org.elasticsearch.client.ml.PostCalendarEventRequest;
@@ -709,6 +710,38 @@ static Request estimateMemoryUsage(PutDataFrameAnalyticsRequest estimateRequest)
709710
return request;
710711
}
711712

713+
static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) {
714+
String endpoint = new EndpointBuilder()
715+
.addPathPartAsIs("_ml", "inference")
716+
.addPathPart(Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getIds()))
717+
.build();
718+
RequestConverters.Params params = new RequestConverters.Params();
719+
if (getTrainedModelsRequest.getPageParams() != null) {
720+
PageParams pageParams = getTrainedModelsRequest.getPageParams();
721+
if (pageParams.getFrom() != null) {
722+
params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString());
723+
}
724+
if (pageParams.getSize() != null) {
725+
params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString());
726+
}
727+
}
728+
if (getTrainedModelsRequest.getAllowNoMatch() != null) {
729+
params.putParam(GetTrainedModelsRequest.ALLOW_NO_MATCH,
730+
Boolean.toString(getTrainedModelsRequest.getAllowNoMatch()));
731+
}
732+
if (getTrainedModelsRequest.getDecompressDefinition() != null) {
733+
params.putParam(GetTrainedModelsRequest.DECOMPRESS_DEFINITION,
734+
Boolean.toString(getTrainedModelsRequest.getDecompressDefinition()));
735+
}
736+
if (getTrainedModelsRequest.getIncludeDefinition() != null) {
737+
params.putParam(GetTrainedModelsRequest.INCLUDE_MODEL_DEFINITION,
738+
Boolean.toString(getTrainedModelsRequest.getIncludeDefinition()));
739+
}
740+
Request request = new Request(HttpGet.METHOD_NAME, endpoint);
741+
request.addParameters(params.asMap());
742+
return request;
743+
}
744+
712745
static Request putFilter(PutFilterRequest putFilterRequest) throws IOException {
713746
String endpoint = new EndpointBuilder()
714747
.addPathPartAsIs("_ml")

‎client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java

+46
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@
7373
import org.elasticsearch.client.ml.GetOverallBucketsResponse;
7474
import org.elasticsearch.client.ml.GetRecordsRequest;
7575
import org.elasticsearch.client.ml.GetRecordsResponse;
76+
import org.elasticsearch.client.ml.GetTrainedModelsRequest;
77+
import org.elasticsearch.client.ml.GetTrainedModelsResponse;
7678
import org.elasticsearch.client.ml.MlInfoRequest;
7779
import org.elasticsearch.client.ml.MlInfoResponse;
7880
import org.elasticsearch.client.ml.OpenJobRequest;
@@ -2290,4 +2292,48 @@ public Cancellable estimateMemoryUsageAsync(PutDataFrameAnalyticsRequest request
22902292
listener,
22912293
Collections.emptySet());
22922294
}
2295+
2296+
/**
2297+
* Gets trained model configs
2298+
* <p>
2299+
* For additional info
2300+
* see <a href="TODO">
2301+
* GET Trained Model Configs documentation</a>
2302+
*
2303+
* @param request The {@link GetTrainedModelsRequest}
2304+
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
2305+
* @return {@link GetTrainedModelsResponse} response object
2306+
*/
2307+
public GetTrainedModelsResponse getTrainedModels(GetTrainedModelsRequest request,
2308+
RequestOptions options) throws IOException {
2309+
return restHighLevelClient.performRequestAndParseEntity(request,
2310+
MLRequestConverters::getTrainedModels,
2311+
options,
2312+
GetTrainedModelsResponse::fromXContent,
2313+
Collections.emptySet());
2314+
}
2315+
2316+
/**
2317+
* Gets trained model configs asynchronously and notifies listener upon completion
2318+
* <p>
2319+
* For additional info
2320+
* see <a href="TODO">
2321+
* GET Trained Model Configs documentation</a>
2322+
*
2323+
* @param request The {@link GetTrainedModelsRequest}
2324+
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
2325+
* @param listener Listener to be notified upon request completion
2326+
* @return cancellable that may be used to cancel the request
2327+
*/
2328+
public Cancellable getTrainedModelsAsync(GetTrainedModelsRequest request,
2329+
RequestOptions options,
2330+
ActionListener<GetTrainedModelsResponse> listener) {
2331+
return restHighLevelClient.performRequestAsyncAndParseEntity(request,
2332+
MLRequestConverters::getTrainedModels,
2333+
options,
2334+
GetTrainedModelsResponse::fromXContent,
2335+
listener,
2336+
Collections.emptySet());
2337+
}
2338+
22932339
}

0 commit comments

Comments
 (0)