Merge branch 'master' into feature/searchable-snapshots

Author: tlrx

.ci/matrix-vagrant-packaging.yml

@@ -0,0 +1,17 @@
+# This file is used as part of a matrix build in Jenkins where the
+# values below are included as an axis of the matrix.
+
+# This axis of the build matrix represents the versions subprojects
+# of the :qa:os project which will be tested on a metal CI worker
+# for the purposes of verifying of Vagrant-based tooling
+
+OS:
+  - centos-7
+  - debian-9
+  - fedora-28
+  - fedora-29
+  - oel-7
+  - sles-12
+  - ubuntu-1804
+  - windows-2012r2
+  - windows-2016

buildSrc/src/main/resources/fips_java.security

@@ -1,6 +1,7 @@
 security.provider.1=org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider
 security.provider.2=org.bouncycastle.jsse.provider.BouncyCastleJsseProvider fips:BCFIPS
 security.provider.3=SUN
+security.provider.4=SunJGSS
 securerandom.source=file:/dev/urandom
 securerandom.strongAlgorithms=NativePRNGBlocking:SUN,DRBG:SUN
 securerandom.drbg.config=

buildSrc/version.properties

@@ -31,7 +31,7 @@ joda              = 2.10.4
 #  - x-pack/plugin/security
 bouncycastle      = 1.61
 # test dependencies
-randomizedrunner  = 2.7.4
+randomizedrunner  = 2.7.6
 junit             = 4.12
 httpclient        = 4.5.10
 httpcore          = 4.4.12

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -275,22 +275,15 @@ private static Request getStyleRequest(String method, GetRequest getRequest) {
         return request;
     }
 
-    static Request sourceExists(GetRequest getRequest) {
-        Params parameters = new Params();
-        parameters.withPreference(getRequest.preference());
-        parameters.withRouting(getRequest.routing());
-        parameters.withRefresh(getRequest.refresh());
-        parameters.withRealtime(getRequest.realtime());
-        parameters.withFetchSourceContext(getRequest.fetchSourceContext());
-        // Version params are not currently supported by the _source API so are not passed
-
-        String endpoint = endpoint(getRequest.index(), "_source", getRequest.id());
-        Request request = new Request(HttpHead.METHOD_NAME, endpoint);
-        request.addParameters(parameters.asMap());
-        return request;
+    static Request sourceExists(GetSourceRequest getSourceRequest) {
+        return sourceRequest(getSourceRequest, HttpHead.METHOD_NAME);
     }
 
     static Request getSource(GetSourceRequest getSourceRequest) {
+        return sourceRequest(getSourceRequest, HttpGet.METHOD_NAME);
+    }
+
+    private static Request sourceRequest(GetSourceRequest getSourceRequest, String httpMethodName) {
         Params parameters = new Params();
         parameters.withPreference(getSourceRequest.preference());
         parameters.withRouting(getSourceRequest.routing());
@@ -299,7 +292,7 @@ static Request getSource(GetSourceRequest getSourceRequest) {
         parameters.withFetchSourceContext(getSourceRequest.fetchSourceContext());
 
         String endpoint = endpoint(getSourceRequest.index(), "_source", getSourceRequest.id());
-        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Request request = new Request(httpMethodName, endpoint);
         request.addParameters(parameters.asMap());
         return request;
     }

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -843,9 +843,13 @@ public final Cancellable existsAsync(GetRequest getRequest, RequestOptions optio
      * @param getRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @return <code>true</code> if the document and _source field exists, <code>false</code> otherwise
+     * @deprecated use {@link #existsSource(GetSourceRequest, RequestOptions)} instead
      */
+    @Deprecated
     public boolean existsSource(GetRequest getRequest, RequestOptions options) throws IOException {
-        return performRequest(getRequest, RequestConverters::sourceExists, options, RestHighLevelClient::convertExistsResponse, emptySet());
+        GetSourceRequest getSourceRequest = GetSourceRequest.from(getRequest);
+        return performRequest(getSourceRequest, RequestConverters::sourceExists, options,
+            RestHighLevelClient::convertExistsResponse, emptySet());
     }
 
     /**
@@ -856,37 +860,68 @@ public boolean existsSource(GetRequest getRequest, RequestOptions options) throw
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      * @return cancellable that may be used to cancel the request
+     * @deprecated use {@link #existsSourceAsync(GetSourceRequest, RequestOptions, ActionListener)} instead
      */
+    @Deprecated
     public final Cancellable existsSourceAsync(GetRequest getRequest, RequestOptions options, ActionListener<Boolean> listener) {
-        return performRequestAsync(getRequest, RequestConverters::sourceExists, options,
+        GetSourceRequest getSourceRequest = GetSourceRequest.from(getRequest);
+        return performRequestAsync(getSourceRequest, RequestConverters::sourceExists, options,
+            RestHighLevelClient::convertExistsResponse, listener, emptySet());
+    }
+
+    /**
+     * Checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API
+     * on elastic.co</a>
+     * @param getSourceRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return <code>true</code> if the document and _source field exists, <code>false</code> otherwise
+     */
+    public boolean existsSource(GetSourceRequest getSourceRequest, RequestOptions options) throws IOException {
+        return performRequest(getSourceRequest, RequestConverters::sourceExists, options,
+            RestHighLevelClient::convertExistsResponse, emptySet());
+    }
+
+    /**
+     * Asynchronously checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API
+     * on elastic.co</a>
+     * @param getSourceRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     * @return cancellable that may be used to cancel the request
+     */
+    public final Cancellable existsSourceAsync(GetSourceRequest getSourceRequest, RequestOptions options,
+                                               ActionListener<Boolean> listener) {
+        return performRequestAsync(getSourceRequest, RequestConverters::sourceExists, options,
             RestHighLevelClient::convertExistsResponse, listener, emptySet());
     }
 
     /**
      * Retrieves the source field only of a document using GetSource API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Get Source API
      * on elastic.co</a>
-     * @param getRequest the request
+     * @param getSourceRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @return the response
      */
-    public GetSourceResponse getSource(GetSourceRequest getRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(getRequest, RequestConverters::getSource, options,
+    public GetSourceResponse getSource(GetSourceRequest getSourceRequest, RequestOptions options) throws IOException {
+        return performRequestAndParseEntity(getSourceRequest, RequestConverters::getSource, options,
             GetSourceResponse::fromXContent, emptySet());
     }
 
     /**
      * Asynchronously retrieves the source field only of a document using GetSource API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Get Source API
      * on elastic.co</a>
-     * @param getRequest the request
+     * @param getSourceRequest the request
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener the listener to be notified upon request completion
      * @return cancellable that may be used to cancel the request
      */
-    public final Cancellable getSourceAsync(GetSourceRequest getRequest, RequestOptions options,
+    public final Cancellable getSourceAsync(GetSourceRequest getSourceRequest, RequestOptions options,
                                             ActionListener<GetSourceResponse> listener) {
-        return performRequestAsyncAndParseEntity(getRequest, RequestConverters::getSource, options,
+        return performRequestAsyncAndParseEntity(getSourceRequest, RequestConverters::getSource, options,
             GetSourceResponse::fromXContent, listener, emptySet());
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/core/GetSourceRequest.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.client.core;
 
+import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.client.Validatable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -43,6 +44,15 @@ public GetSourceRequest(String index, String id) {
         this.id = id;
     }
 
+    public static GetSourceRequest from(GetRequest getRequest) {
+        return new GetSourceRequest(getRequest.index(), getRequest.id())
+            .routing(getRequest.routing())
+            .preference(getRequest.preference())
+            .refresh(getRequest.refresh())
+            .realtime(getRequest.realtime())
+            .fetchSourceContext(getRequest.fetchSourceContext());
+    }
+
     /**
      * Controls the shard routing of the request. Using this value to hash the shard
      * and not the id.

client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java

@@ -181,7 +181,9 @@ public void testExists() throws IOException {
         }
     }
 
-    public void testSourceExists() throws IOException {
+    // used deprecated API existsSource(GetRequest, RequestOptions)
+    // see test `testSourceExists` with new API tests
+    public void testDeprecatedSourceExists() throws IOException {
         {
             GetRequest getRequest = new GetRequest("index", "id");
             assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
@@ -204,6 +206,25 @@ public void testSourceExists() throws IOException {
         }
     }
 
+    public void testSourceExists() throws IOException {
+        {
+            GetSourceRequest getRequest = new GetSourceRequest("index", "id");
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+        IndexRequest index = new IndexRequest("index").id("id");
+        index.source("{\"field1\":\"value1\",\"field2\":\"value2\"}", XContentType.JSON);
+        index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        highLevelClient().index(index, RequestOptions.DEFAULT);
+        {
+            GetSourceRequest getRequest = new GetSourceRequest("index", "id");
+            assertTrue(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+        {
+            GetSourceRequest getRequest = new GetSourceRequest("index", "does_not_exist");
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+    }
+
     public void testSourceDoesNotExist() throws IOException {
         final String noSourceIndex = "no_source";
         {
@@ -230,7 +251,11 @@ public void testSourceDoesNotExist() throws IOException {
         {
             GetRequest getRequest = new GetRequest(noSourceIndex, "1");
             assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
+            // used deprecated API existsSource(GetRequest, RequestOptions)
             assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+            // used new API existsSource(GetSourceRequest, RequestOptions)
+            GetSourceRequest getSourceRequest = new GetSourceRequest(noSourceIndex, "1");
+            assertFalse(execute(getSourceRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
         }
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -154,21 +154,17 @@ public void testGet() {
     }
 
     public void testSourceExists() throws IOException {
-        doTestSourceExists((index, id) -> new GetRequest(index, id));
-    }
-
-    public void testSourceExistsWithType() throws IOException {
-        doTestSourceExists((index, id) -> new GetRequest(index, id));
+        doTestSourceExists((index, id) -> new GetSourceRequest(index, id));
     }
 
     public void testGetSource() throws IOException {
         doTestGetSource((index, id) -> new GetSourceRequest(index, id));
     }
 
-    private static void doTestSourceExists(BiFunction<String, String, GetRequest> requestFunction) throws IOException {
+    private static void doTestSourceExists(BiFunction<String, String, GetSourceRequest> requestFunction) throws IOException {
         String index = randomAlphaOfLengthBetween(3, 10);
         String id = randomAlphaOfLengthBetween(3, 10);
-        final GetRequest getRequest = requestFunction.apply(index, id);
+        final GetSourceRequest getRequest = requestFunction.apply(index, id);
 
         Map<String, String> expectedParams = new HashMap<>();
         if (randomBoolean()) {

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddFileKeyStoreCommandTests.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.env.Environment;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 
 public class AddFileKeyStoreCommandTests extends KeyStoreCommandTestCase {
@@ -192,7 +193,17 @@ public void testIncorrectPassword() throws Exception {
         terminal.addSecretInput("thewrongkeystorepassword");
         UserException e = expectThrows(UserException.class, () -> execute("foo", file.toString()));
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                e.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
     }
 
     public void testAddToUnprotectedKeystore() throws Exception {

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.env.Environment;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasToString;
 
@@ -57,7 +58,17 @@ public void testInvalidPassphrease() throws Exception {
         terminal.addSecretInput("thewrongpassword");
         UserException e = expectThrows(UserException.class, () -> execute("foo2"));
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                e.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
 
     }
 

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/ChangeKeyStorePasswordCommandTests.java

@@ -26,6 +26,7 @@
 
 import java.util.Map;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 
 public class ChangeKeyStorePasswordCommandTests extends KeyStoreCommandTestCase {
@@ -90,6 +91,16 @@ public void testChangeKeyStorePasswordWrongExistingPassword() throws Exception {
         // We'll only be prompted once (for the old password)
         UserException e = expectThrows(UserException.class, this::execute);
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                e.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
     }
 }

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java

@@ -59,6 +59,7 @@
 import java.util.Locale;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -114,7 +115,17 @@ public void testDecryptKeyStoreWithWrongPassword() throws Exception {
             SecurityException.class,
             () -> loadedkeystore.decrypt(new char[] { 'i', 'n', 'v', 'a', 'l', 'i', 'd' })
         );
-        assertThat(exception.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                exception.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(exception.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
     }
 
     public void testCannotReadStringFromClosedKeystore() throws Exception {

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/ListKeyStoreCommandTests.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.env.Environment;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 
 public class ListKeyStoreCommandTests extends KeyStoreCommandTestCase {
@@ -76,7 +77,17 @@ public void testListWithIncorrectPassword() throws Exception {
         terminal.addSecretInput("thewrongkeystorepassword");
         UserException e = expectThrows(UserException.class, this::execute);
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                e.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
     }
 
     public void testListWithUnprotectedKeystore() throws Exception {