Skip to content

Commit 4363363

Browse files
committed
Merge branch '6.x' into ccr-6.x
* 6.x: Force execution of fetch tasks (#31974) [TEST] Mute SlackMessageTests.testTemplateRender Docs: Explain closing the high level client [test] disable java packaging tests for suse XContentTests : Insert random fields at random positions (#30867) Add Get Snapshots High Level REST API (#31980) Fix unreachable error condition in AmazonS3Fixture (#32005) [6.x][ML] Ensure immutability of MlMetadata (#31994) Add Expected Reciprocal Rank metric (#31891) SQL: Add support for single parameter text manipulating functions (#31874) muted tests due to #31940 Work around reported problem in eclipse (#31960) Move build integration tests out of :buildSrc project (#31961) [Test] Reactive 3rd party tests on CI (#31919) Fix assertIngestDocument wrongfully passing (#31913) (#31951) SQL: Support for escape sequences (#31884) SQL: HAVING clause should accept only aggregates (#31872) Docs: fix typo in datehistogram (#31972) Switch url repository rest tests to new style requests (#31944) Switch reindex tests to new style requests (#31941) Switch test framework to new style requests (#31939) Docs: Added note about cloud service to installation and getting started [DOCS] Removes alternative docker pull example (#31934) ingest: date_index_name processor template resolution (#31841) Test: fix null failure in watcher test (#31968) Watcher: Slack message empty text (#31596) Switch low level rest tests to new style Requests (#31938) Switch high level rest tests to new style requests (#31937) HLREST: Bundle the x-pack protocol project (#31904) [ML] Mute test failing due to Java 11 date time format parsing bug (#31899) Increase logging level for testStressMaybeFlush rolling upgrade should use a replica to prevent relocations while running a scroll [test] port archive distribution packaging tests (#31314) HLRest: Move xPackInfo() to xPack().info() (#31905) Increase logging level for :qa:rolling-upgrade Backport: Add template config for Beat state to X-Pack Monitoring (#31809) (#31893) Fix building AD URL from domain name (#31849) Fix broken NaN check in MovingFunctions#stdDev() (#31888) Change trappy float comparison (#31889) Add opaque_id to audit logging (#31878) add support for is_write_index in put-alias body parsing (#31674) Ingest: Enable Templated Fieldnames in Rename (#31690) (#31896) Ingest: Add ignore_missing option to RemoveProc (#31693) (#31892) [Docs] Fix typo in the Rollup API Quick Reference (#31855) Watcher: Add ssl.trust email account setting (#31684) [PkiRealm] Invalidate cache on role mappings change (#31510) [Security] Check auth scheme case insensitively (#31490) HLREST: Add x-pack-info API (#31870) Remove link to oss-MSI (#31844) Painless: Restructure Definition/Whitelist (#31879)
2 parents 04b5681 + 71380bd commit 4363363

File tree

259 files changed

+10512
-4204
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

259 files changed

+10512
-4204
lines changed

build.gradle

+3
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,9 @@ allprojects {
429429
if (isEclipse) {
430430
// set this so generated dirs will be relative to eclipse build
431431
project.buildDir = eclipseBuild
432+
// Work around https://docs.gradle.org/current/userguide/java_gradle_plugin.html confusing Eclipse by the metadata
433+
// it adds to the classpath
434+
project.file("$buildDir/pluginUnderTestMetadata").mkdirs()
432435
}
433436
eclipse.classpath.file.whenMerged { classpath ->
434437
// give each source folder a unique corresponding output folder

buildSrc/build.gradle

+16
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,10 @@ if (project == rootProject) {
128128
}
129129
mavenCentral()
130130
}
131+
test {
132+
include "**/*Tests.class"
133+
exclude "**/*IT.class"
134+
}
131135
}
132136

133137
/*****************************************************************************
@@ -152,6 +156,18 @@ if (project != rootProject) {
152156
jarHell.enabled = false
153157
thirdPartyAudit.enabled = false
154158

159+
// tests can't be run with randomized test runner
160+
// it's fine as we run them as part of :buildSrc
161+
test.enabled = false
162+
task integTest(type: Test) {
163+
exclude "**/*Tests.class"
164+
include "**/*IT.class"
165+
testClassesDirs = sourceSets.test.output.classesDirs
166+
classpath = sourceSets.test.runtimeClasspath
167+
inputs.dir(file("src/testKit"))
168+
}
169+
check.dependsOn(integTest)
170+
155171
// TODO: re-enable once randomizedtesting gradle code is published and removed from here
156172
licenseHeaders.enabled = false
157173

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

+13-3
Original file line numberDiff line numberDiff line change
@@ -550,9 +550,15 @@ class VagrantTestPlugin implements Plugin<Project> {
550550
javaPackagingTest.command = 'ssh'
551551
javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"']
552552
} else {
553+
// powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however
554+
// remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user
555+
// without administrator elevation), which we need to do for these tests. passing --elevated runs the session
556+
// as a scheduled job locally on the vm as a true administrator to get around this limitation
557+
//
558+
// https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225
559+
// https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html
553560
javaPackagingTest.command = 'winrm'
554-
// winrm commands run as administrator
555-
javaPackagingTest.args = ['--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
561+
javaPackagingTest.args = ['--elevated', '--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
556562
}
557563

558564
TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
@@ -563,7 +569,11 @@ class VagrantTestPlugin implements Plugin<Project> {
563569
project.gradle.removeListener(javaPackagingReproListener)
564570
}
565571
if (project.extensions.esvagrant.boxes.contains(box)) {
566-
packagingTest.dependsOn(javaPackagingTest)
572+
// these tests are temporarily disabled for suse boxes while we debug an issue
573+
// https://github.com/elastic/elasticsearch/issues/30295
574+
if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
575+
packagingTest.dependsOn(javaPackagingTest)
576+
}
567577
}
568578

569579
/*

client/rest-high-level/build.gradle

+111-11
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
import org.elasticsearch.gradle.precommit.PrecommitTasks
2-
31
/*
42
* Licensed to Elasticsearch under one or more contributor
53
* license agreements. See the NOTICE file distributed with
@@ -18,29 +16,86 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
1816
* specific language governing permissions and limitations
1917
* under the License.
2018
*/
19+
20+
import org.elasticsearch.gradle.precommit.PrecommitTasks
21+
import org.gradle.api.XmlProvider
22+
import org.gradle.api.publish.maven.MavenPublication
23+
24+
buildscript {
25+
repositories {
26+
maven {
27+
url 'https://plugins.gradle.org/m2/'
28+
}
29+
}
30+
dependencies {
31+
classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
32+
}
33+
}
34+
2135
apply plugin: 'elasticsearch.build'
2236
apply plugin: 'elasticsearch.rest-test'
2337
apply plugin: 'nebula.maven-base-publish'
2438
apply plugin: 'nebula.maven-scm'
39+
apply plugin: 'com.github.johnrengelman.shadow'
2540

2641
group = 'org.elasticsearch.client'
2742
archivesBaseName = 'elasticsearch-rest-high-level-client'
2843

2944
publishing {
30-
publications {
31-
nebula {
32-
artifactId = archivesBaseName
45+
publications {
46+
nebula(MavenPublication) {
47+
artifact shadowJar
48+
artifactId = archivesBaseName
49+
/*
50+
* Configure the pom to include the "shadow" as compile dependencies
51+
* because that is how we're using them but remove all other dependencies
52+
* because they've been shaded into the jar.
53+
*/
54+
pom.withXml { XmlProvider xml ->
55+
Node root = xml.asNode()
56+
root.remove(root.dependencies)
57+
Node dependenciesNode = root.appendNode('dependencies')
58+
project.configurations.shadow.allDependencies.each {
59+
if (false == it instanceof SelfResolvingDependency) {
60+
Node dependencyNode = dependenciesNode.appendNode('dependency')
61+
dependencyNode.appendNode('groupId', it.group)
62+
dependencyNode.appendNode('artifactId', it.name)
63+
dependencyNode.appendNode('version', it.version)
64+
dependencyNode.appendNode('scope', 'compile')
65+
}
3366
}
67+
}
3468
}
69+
}
70+
}
71+
72+
/*
73+
* We need somewhere to configure dependencies that we don't wish to shade
74+
* into the high level REST client. The shadow plugin creates a "shadow"
75+
* configuration which is *almost* exactly that. It is never bundled into
76+
* the shaded jar but is used for main source compilation. Unfortunately,
77+
* by default it is not used for *test* source compilation and isn't used
78+
* in tests at all. This change makes it available for test compilation.
79+
* A change below makes it available for testing.
80+
*/
81+
sourceSets {
82+
test {
83+
compileClasspath += configurations.shadow
84+
}
3585
}
3686

3787
dependencies {
38-
compile "org.elasticsearch:elasticsearch:${version}"
39-
compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
40-
compile "org.elasticsearch.plugin:parent-join-client:${version}"
41-
compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
42-
compile "org.elasticsearch.plugin:rank-eval-client:${version}"
43-
compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
88+
/*
89+
* Everything in the "shadow" configuration is *not* copied into the
90+
* shadowJar.
91+
*/
92+
shadow "org.elasticsearch:elasticsearch:${version}"
93+
shadow "org.elasticsearch.client:elasticsearch-rest-client:${version}"
94+
shadow "org.elasticsearch.plugin:parent-join-client:${version}"
95+
shadow "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
96+
shadow "org.elasticsearch.plugin:rank-eval-client:${version}"
97+
shadow "org.elasticsearch.plugin:lang-mustache-client:${version}"
98+
compile project(':x-pack:protocol')
4499

45100
testCompile "org.elasticsearch.client:test:${version}"
46101
testCompile "org.elasticsearch.test:framework:${version}"
@@ -63,3 +118,48 @@ forbiddenApisMain {
63118
signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
64119
signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
65120
}
121+
122+
shadowJar {
123+
classifier = null
124+
mergeServiceFiles()
125+
}
126+
127+
// We don't need normal jar, we use shadow jar instead
128+
jar.enabled = false
129+
assemble.dependsOn shadowJar
130+
131+
javadoc {
132+
/*
133+
* Bundle all of the javadoc from all of the shaded projects into this one
134+
* so we don't *have* to publish javadoc for all of the "client" jars.
135+
*/
136+
configurations.compile.dependencies.all { Dependency dep ->
137+
Project p = dependencyToProject(dep)
138+
if (p != null) {
139+
evaluationDependsOn(p.path)
140+
source += p.sourceSets.main.allJava
141+
}
142+
}
143+
}
144+
145+
/*
146+
* Use the jar for testing so we have tests of the bundled jar.
147+
* Use the "shadow" configuration for testing because we need things
148+
* in it.
149+
*/
150+
test {
151+
classpath -= compileJava.outputs.files
152+
classpath -= configurations.compile
153+
classpath -= configurations.runtime
154+
classpath += configurations.shadow
155+
classpath += shadowJar.outputs.files
156+
dependsOn shadowJar
157+
}
158+
integTestRunner {
159+
classpath -= compileJava.outputs.files
160+
classpath -= configurations.compile
161+
classpath -= configurations.runtime
162+
classpath += configurations.shadow
163+
classpath += shadowJar.outputs.files
164+
dependsOn shadowJar
165+
}

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

+37
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
4040
import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
4141
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
42+
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
4243
import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
4344
import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
4445
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
@@ -103,6 +104,7 @@
103104
import org.elasticsearch.common.xcontent.XContentType;
104105
import org.elasticsearch.index.VersionType;
105106
import org.elasticsearch.index.rankeval.RankEvalRequest;
107+
import org.elasticsearch.protocol.xpack.XPackInfoRequest;
106108
import org.elasticsearch.rest.action.search.RestSearchAction;
107109
import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
108110
import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -114,8 +116,10 @@
114116
import java.net.URI;
115117
import java.net.URISyntaxException;
116118
import java.nio.charset.Charset;
119+
import java.util.EnumSet;
117120
import java.util.Locale;
118121
import java.util.StringJoiner;
122+
import java.util.stream.Collectors;
119123

120124
final class RequestConverters {
121125
static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON;
@@ -950,6 +954,26 @@ static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throw
950954
return request;
951955
}
952956

957+
static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) {
958+
EndpointBuilder endpointBuilder = new EndpointBuilder().addPathPartAsIs("_snapshot")
959+
.addPathPart(getSnapshotsRequest.repository());
960+
String endpoint;
961+
if (getSnapshotsRequest.snapshots().length == 0) {
962+
endpoint = endpointBuilder.addPathPart("_all").build();
963+
} else {
964+
endpoint = endpointBuilder.addCommaSeparatedPathParts(getSnapshotsRequest.snapshots()).build();
965+
}
966+
967+
Request request = new Request(HttpGet.METHOD_NAME, endpoint);
968+
969+
Params parameters = new Params(request);
970+
parameters.withMasterTimeout(getSnapshotsRequest.masterNodeTimeout());
971+
parameters.putParam("ignore_unavailable", Boolean.toString(getSnapshotsRequest.ignoreUnavailable()));
972+
parameters.putParam("verbose", Boolean.toString(getSnapshotsRequest.verbose()));
973+
974+
return request;
975+
}
976+
953977
static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) {
954978
String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot")
955979
.addPathPart(deleteSnapshotRequest.repository())
@@ -1055,6 +1079,19 @@ static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest)
10551079
return request;
10561080
}
10571081

1082+
static Request xPackInfo(XPackInfoRequest infoRequest) {
1083+
Request request = new Request(HttpGet.METHOD_NAME, "/_xpack");
1084+
if (false == infoRequest.isVerbose()) {
1085+
request.addParameter("human", "false");
1086+
}
1087+
if (false == infoRequest.getCategories().equals(EnumSet.allOf(XPackInfoRequest.Category.class))) {
1088+
request.addParameter("categories", infoRequest.getCategories().stream()
1089+
.map(c -> c.toString().toLowerCase(Locale.ROOT))
1090+
.collect(Collectors.joining(",")));
1091+
}
1092+
return request;
1093+
}
1094+
10581095
private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
10591096
BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
10601097
return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

+18-5
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ public class RestHighLevelClient implements Closeable {
202202
private final IngestClient ingestClient = new IngestClient(this);
203203
private final SnapshotClient snapshotClient = new SnapshotClient(this);
204204
private final TasksClient tasksClient = new TasksClient(this);
205+
private final XPackClient xPackClient = new XPackClient(this);
205206

206207
/**
207208
* Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
@@ -292,6 +293,19 @@ public final TasksClient tasks() {
292293
return tasksClient;
293294
}
294295

296+
/**
297+
* A wrapper for the {@link RestHighLevelClient} that provides methods for
298+
* accessing the Elastic Licensed X-Pack APIs that are shipped with the
299+
* default distribution of Elasticsearch. All of these APIs will 404 if run
300+
* against the OSS distribution of Elasticsearch.
301+
* <p>
302+
* See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-api.html">
303+
* X-Pack APIs on elastic.co</a> for more information.
304+
*/
305+
public final XPackClient xpack() {
306+
return xPackClient;
307+
}
308+
295309
/**
296310
* Executes a bulk request using the Bulk API.
297311
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html">Bulk API on elastic.co</a>
@@ -979,7 +993,6 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions
979993
emptySet());
980994
}
981995

982-
983996
/**
984997
* Executes a request using the Multi Search Template API.
985998
*
@@ -989,9 +1002,9 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions
9891002
public final MultiSearchTemplateResponse multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplateRequest,
9901003
RequestOptions options) throws IOException {
9911004
return performRequestAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
992-
options, MultiSearchTemplateResponse::fromXContext, emptySet());
993-
}
994-
1005+
options, MultiSearchTemplateResponse::fromXContext, emptySet());
1006+
}
1007+
9951008
/**
9961009
* Asynchronously executes a request using the Multi Search Template API
9971010
*
@@ -1003,7 +1016,7 @@ public final void multiSearchTemplateAsync(MultiSearchTemplateRequest multiSearc
10031016
ActionListener<MultiSearchTemplateResponse> listener) {
10041017
performRequestAsyncAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate,
10051018
options, MultiSearchTemplateResponse::fromXContext, listener, emptySet());
1006-
}
1019+
}
10071020

10081021
/**
10091022
* Asynchronously executes a request using the Ranking Evaluation API.

client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java

+31
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
3333
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
3434
import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
35+
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
36+
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
3537

3638
import java.io.IOException;
3739

@@ -190,6 +192,35 @@ public void createSnapshotAsync(CreateSnapshotRequest createSnapshotRequest, Req
190192
CreateSnapshotResponse::fromXContent, listener, emptySet());
191193
}
192194

195+
/**
196+
* Get snapshots.
197+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
198+
* API on elastic.co</a>
199+
*
200+
* @param getSnapshotsRequest the request
201+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
202+
* @return the response
203+
* @throws IOException in case there is a problem sending the request or parsing back the response
204+
*/
205+
public GetSnapshotsResponse get(GetSnapshotsRequest getSnapshotsRequest, RequestOptions options) throws IOException {
206+
return restHighLevelClient.performRequestAndParseEntity(getSnapshotsRequest, RequestConverters::getSnapshots, options,
207+
GetSnapshotsResponse::fromXContent, emptySet());
208+
}
209+
210+
/**
211+
* Asynchronously get snapshots.
212+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
213+
* API on elastic.co</a>
214+
*
215+
* @param getSnapshotsRequest the request
216+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
217+
* @param listener the listener to be notified upon request completion
218+
*/
219+
public void getAsync(GetSnapshotsRequest getSnapshotsRequest, RequestOptions options, ActionListener<GetSnapshotsResponse> listener) {
220+
restHighLevelClient.performRequestAsyncAndParseEntity(getSnapshotsRequest, RequestConverters::getSnapshots, options,
221+
GetSnapshotsResponse::fromXContent, listener, emptySet());
222+
}
223+
193224
/**
194225
* Deletes a snapshot.
195226
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore

0 commit comments

Comments
 (0)