Merge branch 'master' into java-time

Author: spinscale

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -103,10 +103,9 @@ public void apply(Project project) {
     @Input
     public boolean dockerComposeSupported(Project project) {
         // Don't look for docker-compose on the PATH yet that would pick up on Windows as well
-        return
-            project.file("/usr/local/bin/docker-compose").exists() == false &&
-            project.file("/usr/bin/docker-compose").exists() == false &&
-            Boolean.parseBoolean(System.getProperty("tests.fixture.enabled", "true")) == false;
+        final boolean hasDockerCompose = project.file("/usr/local/bin/docker-compose").exists() ||
+            project.file("/usr/bin/docker-compose").exists();
+        return hasDockerCompose && Boolean.parseBoolean(System.getProperty("tests.fixture.enabled", "true"));
     }
 
     private void setSystemProperty(Task task, String name, Object value) {

buildSrc/version.properties

@@ -15,7 +15,7 @@ slf4j             = 1.6.2
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 4.5.1
 
-netty             = 4.1.31.Final
+netty             = 4.1.32.Final
 joda              = 2.10.1
 
 # test dependencies

client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java

@@ -82,7 +82,7 @@ public void testRequests() throws Exception {
         RestHighLevelClient client = highLevelClient();
         {
             //tag::migration-request-ctor
-            IndexRequest request = new IndexRequest("index", "doc", "id"); // <1>
+            IndexRequest request = new IndexRequest("index", "_doc", "id"); // <1>
             request.source("{\"field\":\"value\"}", XContentType.JSON);
             //end::migration-request-ctor
 
@@ -93,7 +93,7 @@ public void testRequests() throws Exception {
         }
         {
             //tag::migration-request-async-execution
-            DeleteRequest request = new DeleteRequest("index", "doc", "id"); // <1>
+            DeleteRequest request = new DeleteRequest("index", "id"); // <1>
             client.deleteAsync(request, RequestOptions.DEFAULT, new ActionListener<DeleteResponse>() { // <2>
                 @Override
                 public void onResponse(DeleteResponse deleteResponse) {
@@ -106,11 +106,11 @@ public void onFailure(Exception e) {
                 }
             });
             //end::migration-request-async-execution
-            assertBusy(() -> assertFalse(client.exists(new GetRequest("index", "doc", "id"), RequestOptions.DEFAULT)));
+            assertBusy(() -> assertFalse(client.exists(new GetRequest("index", "_doc", "id"), RequestOptions.DEFAULT)));
         }
         {
             //tag::migration-request-sync-execution
-            DeleteRequest request = new DeleteRequest("index", "doc", "id");
+            DeleteRequest request = new DeleteRequest("index", "id");
             DeleteResponse response = client.delete(request, RequestOptions.DEFAULT); // <1>
             //end::migration-request-sync-execution
             assertEquals(RestStatus.NOT_FOUND, response.status());

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java

@@ -204,9 +204,9 @@ public void onFailure(Exception e) {
             GetWatchRequest request = new GetWatchRequest("my_watch_id");
             //end::get-watch-request
 
-            //tag::ack-watch-execute
+            //tag::get-watch-execute
             GetWatchResponse response = client.watcher().getWatch(request, RequestOptions.DEFAULT);
-            //end::get-watch-request
+            //end::get-watch-execute
 
             //tag::get-watch-response
             String watchId = response.getId(); // <1>

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java

@@ -1,3 +1,4 @@
+#!/bin/bash
 #
 # This script is executed in the pre-installation phase
 #
@@ -9,16 +10,22 @@
 #       $1=1       : indicates an new install
 #       $1=2       : indicates an upgrade
 
+err_exit() {
+    echo "$@" >&2
+    exit 1
+}
+
 # Check for these at preinst time due to failures in postinst if they do not exist
 if [ -x "$JAVA_HOME/bin/java" ]; then
     JAVA="$JAVA_HOME/bin/java"
+elif command -v java; then
+    JAVA=`command -v java`
 else
-    JAVA=`which java`
+    JAVA=""
 fi
 
 if [ -z "$JAVA" ]; then
-    echo "could not find java; set JAVA_HOME or ensure java is in PATH"
-    exit 1
+    err_exit "could not find java; set JAVA_HOME or ensure java is in PATH"
 fi
 
 case "$1" in
@@ -75,8 +82,7 @@ case "$1" in
     ;;
 
     *)
-        echo "pre install script called with unknown argument \`$1'" >&2
-        exit 1
+        err_exit "pre install script called with unknown argument \`$1'"
     ;;
 esac
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/WatcherDocumentationIT.java

@@ -45,7 +45,7 @@ else
 fi
 
 if [ ! -x "$JAVA" ]; then
-  echo "could not find java; set JAVA_HOME or ensure java is in PATH"
+  echo "could not find java; set JAVA_HOME or ensure java is in PATH" >&2
   exit 1
 fi
 

distribution/packages/src/common/scripts/preinst

@@ -10,15 +10,14 @@
 [id="{upid}-{api}-request"]
 ==== Delete Request
 
-A +{request}+ has no arguments
+A +{request}+ has two required arguments:
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
 include-tagged::{doc-tests-file}[{api}-request]
 --------------------------------------------------
 <1> Index
-<2> Type
-<3> Document id
+<2> Document id
 
 ==== Optional arguments
 The following arguments can optionally be provided:

distribution/src/bin/elasticsearch-env

@@ -1,9 +1,9 @@
 [[docs-delete]]
 == Delete API
 
-The delete API allows to delete a typed JSON document from a specific
+The delete API allows to delete a JSON document from a specific
 index based on its id. The following example deletes the JSON document
-from an index called `twitter`, under a type called `_doc`, with id `1`:
+from an index called `twitter` with ID `1`:
 
 [source,js]
 --------------------------------------------------
@@ -92,10 +92,7 @@ the request.
 If an <<docs-index_,external versioning variant>> is used,
 the delete operation automatically creates an index if it has not been
 created before (check out the <<indices-create-index,create index API>>
-for manually creating an index), and also automatically creates a
-dynamic type mapping for the specific type if it has not been created
-before (check out the <<indices-put-mapping,put mapping>>
-API for manually creating type mapping).
+for manually creating an index).
 
 [float]
 [[delete-distributed]]

docs/java-rest/high-level/document/delete.asciidoc

@@ -0,0 +1,35 @@
+[role="xpack"]
+[testenv="basic"]
+[[index-lifecycle-and-snapshots]]
+== Restoring Snapshots of Managed Indices
+
+beta[]
+
+When restoring a snapshot that contains indices managed by Index Lifecycle
+Management, the lifecycle will automatically continue to execute after the
+snapshot is restored. Notably, the `min_age` is relative to the original
+creation or rollover of the index, rather than when the index was restored. For
+example, a monthly index that is restored partway through its lifecycle after an
+accidental deletion will be continue through its lifecycle as expected: The
+index will be shrunk, reallocated to different nodes, or deleted on the same
+schedule whether or not it has been restored from a snapshot.
+
+However, there may be cases where you need to restore an index from a snapshot,
+but do not want it to automatically continue through its lifecycle, particularly
+if the index would rapidly progress through lifecycle phases due to its age. For
+example, you may wish to add or update documents in an index before it is marked
+read only or shrunk, or prevent an index from automatically being deleted.
+
+To stop lifecycle policy execution on an index restored from a snapshot, before
+restoring the snapshot, <<start-stop-ilm,lifecycle policy execution can be
+paused>> to allow the policy to be removed.
+
+For example, the following workflow can be used in the above situation to
+prevent the execution of the lifecycle policy for an index:
+
+1. Pause execution of all lifecycle policies using the <<ilm-stop,Stop ILM API>>
+2. Restore the snapshot.
+3. Perform whatever operations you wish before resuming lifecycle execution, or
+    remove the lifecycle policy from the index using the
+    <<ilm-remove-policy,Remove Policy from Index API>>
+4. Resume execution of lifecycle policies using the <<ilm-start,Start ILM API>>

docs/reference/docs/delete.asciidoc

@@ -64,4 +64,6 @@ include::update-lifecycle-policy.asciidoc[]
 
 include::error-handling.asciidoc[]
 
+include::ilm-and-snapshots.asciidoc[]
+
 include::start-stop-ilm.asciidoc[]

docs/reference/ilm/ilm-and-snapshots.asciidoc

@@ -32,33 +32,34 @@ PUT _xpack/ml/anomaly_detectors/population
 {
   "description" : "Population analysis",
   "analysis_config" : {
-    "bucket_span":"10m",
+    "bucket_span":"15m",
     "influencers": [
-      "username"
+      "clientip"
     ],
     "detectors": [
       {
         "function": "mean",
-        "field_name": "bytesSent",
-        "over_field_name": "username" <1>
+        "field_name": "bytes",
+        "over_field_name": "clientip" <1>
       }
     ]
   },
   "data_description" : {
-    "time_field":"@timestamp",
+    "time_field":"timestamp",
     "time_format": "epoch_ms"
   }
 }
 ----------------------------------
 //CONSOLE
 // TEST[skip:needs-licence]
-<1> This `over_field_name` property indicates that the metrics for each user (
-  as identified by their `username` value) are analyzed relative to other users
+<1> This `over_field_name` property indicates that the metrics for each client (
+  as identified by their IP address) are analyzed relative to other clients
   in each bucket.
 
 If your data is stored in {es}, you can use the population job wizard in {kib}
-to create a job with these same properties. For example, the population job
-wizard provides the following job settings:
+to create a job with these same properties. For example, if you add the sample
+web logs in {kib}, you can use the following job settings in the population job
+wizard:
 
 [role="screenshot"]
 image::images/ml-population-job.jpg["Job settings in the population job wizard]
@@ -81,6 +82,6 @@ details about the anomalies:
 [role="screenshot"]
 image::images/ml-population-anomaly.jpg["Anomaly details for a specific user"]
 
-In this example, the user identified as `antonette` sent a high volume of bytes
-on the date and time shown. This event is anomalous because the mean is two times
-higher than the expected behavior of the population.
+In this example, the client IP address `29.64.62.83` received a high volume of
+bytes on the date and time shown. This event is anomalous because the mean is
+three times higher than the expected behavior of the population.

docs/reference/ilm/index.asciidoc

@@ -30,6 +30,11 @@ There is a soft limit on the number of shards in a cluster, based on the number
 of nodes in the cluster. This is intended to prevent operations which may
 unintentionally destabilize the cluster.
 
+IMPORTANT: This limit is intended as a safety net, not a sizing recommendation. The
+exact number of shards your cluster can safely support depends on your hardware
+configuration and workload, but should remain well below this limit in almost
+all cases, as the default limit is set quite high.
+
 If an operation, such as creating a new index, restoring a snapshot of an index,
 or opening a closed index would lead to the number of shards in the cluster
 going over this limit, the operation will fail with an error indicating the
@@ -53,8 +58,8 @@ adjusted using the following property:
      Controls the number of shards allowed in the cluster per data node.
 
 For example, a 3-node cluster with the default setting would allow 3,000 shards
-total, across all open indexes. If the above setting is changed to 1,500, then
-the cluster would allow 4,500 shards total.
+total, across all open indexes. If the above setting is changed to 500, then
+the cluster would allow 1,500 shards total.
 
 NOTE: If there are no data nodes in the cluster, the limit will not be enforced.
 This allows the creation of indices during cluster creation if dedicated master

docs/reference/ml/images/ml-population-anomaly.jpg

@@ -105,7 +105,7 @@ endif::[]
 
 include::skip-set-kernel-parameters.asciidoc[]
 
-ifeval::["{release-state}"=="released"]
+ifeval::["{release-state}"!="unreleased"]
 
 [NOTE]
 ==================================================

docs/reference/ml/images/ml-population-job.jpg

@@ -90,7 +90,7 @@ sudo zypper install elasticsearch <3>
 
 endif::[]
 
-ifeval::["{release-state}"=="released"]
+ifeval::["{release-state}"!="unreleased"]
 
 [NOTE]
 ==================================================

docs/reference/ml/images/ml-population-results.jpg

@@ -39,6 +39,20 @@ public class RethrottleRequest extends BaseTasksRequest<RethrottleRequest> {
      */
     private Float requestsPerSecond;
 
+    public RethrottleRequest() {
+    }
+
+    public RethrottleRequest(StreamInput in) throws IOException {
+        super(in);
+        this.requestsPerSecond = in.readFloat();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeFloat(requestsPerSecond);
+    }
+
     /**
      * The throttle to apply to all matching requests in sub-requests per second. 0 means set no throttle and that is the default.
      */
@@ -80,15 +94,4 @@ public ActionRequestValidationException validate() {
         return validationException;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        requestsPerSecond = in.readFloat();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeFloat(requestsPerSecond);
-    }
 }

docs/reference/ml/populations.asciidoc

@@ -184,9 +184,7 @@ public void testRethrottleRequest() throws IOException {
         } else {
             request.setTaskId(new TaskId(randomAlphaOfLength(5), randomLong()));
         }
-        RethrottleRequest tripped = new RethrottleRequest();
-        // We use readFrom here because Rethrottle does not support the Writeable.Reader interface
-        tripped.readFrom(toInputByteStream(request));
+        RethrottleRequest tripped = new RethrottleRequest(toInputByteStream(request));
         assertEquals(request.getRequestsPerSecond(), tripped.getRequestsPerSecond(), 0.00001);
         assertArrayEquals(request.getActions(), tripped.getActions());
         assertEquals(request.getTaskId(), tripped.getTaskId());

docs/reference/modules/cluster/misc.asciidoc

@@ -109,6 +109,8 @@ thirdPartyAudit.excludes = [
         'org.jboss.marshalling.Unmarshaller',
 
         // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional
+        'org.slf4j.helpers.FormattingTuple',
+        'org.slf4j.helpers.MessageFormatter',
         'org.slf4j.Logger',
         'org.slf4j.LoggerFactory',
         'org.slf4j.spi.LocationAwareLogger',

docs/reference/setup/install/deb.asciidoc

@@ -0,0 +1 @@
+046ede57693788181b2cafddc3a5967ed2f621c8

docs/reference/setup/install/rpm.asciidoc

@@ -0,0 +1 @@
+8f32bd79c5a16f014a4372ed979dc62b39ede33a

modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequest.java

@@ -0,0 +1 @@
+0b9218adba7353ad5a75fcb639e4755d64bd6ddf

modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java

@@ -0,0 +1 @@
+e95de4f762606f492328e180c8ad5438565a5e3b

modules/transport-netty4/build.gradle

@@ -0,0 +1 @@
+b4e3fa13f219df14a9455cc2111f133374428be0

modules/transport-netty4/licenses/netty-buffer-4.1.31.Final.jar.sha1

@@ -0,0 +1 @@
+3e0114715cb125a12db8d982b2208e552a91256d

modules/transport-netty4/licenses/netty-buffer-4.1.32.Final.jar.sha1

@@ -0,0 +1 @@
+d5e5a8ff9c2bc7d91ddccc536a5aca1a4355bd8b