Merge branch 'master' into fix/28953

Author: Yogesh Gaikwad

buildSrc/src/main/resources/forbidden/es-all-signatures.txt

@@ -48,3 +48,5 @@ java.nio.channels.SocketChannel#connect(java.net.SocketAddress)
 # org.elasticsearch.common.Booleans#parseBoolean(java.lang.String) directly on the string.
 @defaultMessage use org.elasticsearch.common.Booleans#parseBoolean(java.lang.String)
 java.lang.Boolean#getBoolean(java.lang.String)
+
+org.apache.lucene.util.IOUtils @ use @org.elasticsearch.core.internal.io instead

distribution/build.gradle

@@ -265,6 +265,12 @@ subprojects {
         'def': "#-XX:HeapDumpPath=/heap/dump/path"
       ],
 
+      'error.file': [
+        'deb': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
+        'rpm': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
+        'def': "#-XX:ErrorFile=/error/file/path"
+      ],
+
       'stopping.timeout': [
         'rpm': 86400,
       ],

distribution/packages/src/common/scripts/postinst

@@ -95,19 +95,15 @@ chown -R root:elasticsearch /etc/elasticsearch
 chmod g+s /etc/elasticsearch
 chmod 0750 /etc/elasticsearch
 
-if [ -f /etc/default/elasticsearch ]; then
-    chown root:elasticsearch /etc/default/elasticsearch
+if [ -f ${path.env} ]; then
+    chown root:elasticsearch ${path.env}
 fi
 
-if [ -f /etc/sysconfig/elasticsearch ]; then
-    chown root:elasticsearch /etc/sysconfig/elasticsearch
-fi
-
-if [ ! -f "$ES_PATH_CONF"/elasticsearch.keystore ]; then
+if [ ! -f /etc/elasticsearch/elasticsearch.keystore ]; then
     /usr/share/elasticsearch/bin/elasticsearch-keystore create
-    chown root:elasticsearch "$ES_PATH_CONF"/elasticsearch.keystore
-    chmod 660 "$ES_PATH_CONF"/elasticsearch.keystore
-    md5sum "$ES_PATH_CONF"/elasticsearch.keystore > "$ES_PATH_CONF"/.elasticsearch.keystore.initial_md5sum
+    chown root:elasticsearch /etc/elasticsearch/elasticsearch.keystore
+    chmod 660 /etc/elasticsearch/elasticsearch.keystore
+    md5sum /etc/elasticsearch/elasticsearch.keystore > /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum
 fi
 
 ${scripts.footer}

distribution/packages/src/common/scripts/prerm

@@ -42,11 +42,6 @@ case "$1" in
     ;;
 esac
 
-ES_ENV_FILE="${path.env}"
-if [ -f "$ES_ENV_FILE" ]; then
-    . "$ES_ENV_FILE"
-fi
-
 # Stops the service
 if [ "$STOP_REQUIRED" = "true" ]; then
     echo -n "Stopping elasticsearch service..."
@@ -70,9 +65,9 @@ if [ "$STOP_REQUIRED" = "true" ]; then
     echo " OK"
 fi
 
-if [ -f "$ES_PATH_CONF"/elasticsearch.keystore ]; then
-  if md5sum --status -c "$ES_PATH_CONF"/.elasticsearch.keystore.initial_md5sum; then
-    rm "$ES_PATH_CONF"/elasticsearch.keystore "$ES_PATH_CONF"/.elasticsearch.keystore.initial_md5sum
+if [ -f /etc/elasticsearch/elasticsearch.keystore ]; then
+  if md5sum --status -c /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum; then
+    rm /etc/elasticsearch/elasticsearch.keystore /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum
   fi
 fi
 

distribution/src/config/jvm.options

@@ -81,6 +81,9 @@
 # ensure the directory exists and has sufficient space
 ${heap.dump.path}
 
+# specify an alternative path for JVM fatal error logs
+${error.file}
+
 ## JDK 8 GC logging
 
 8:-XX:+PrintGCDetails

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -24,7 +24,7 @@
 
 import org.apache.lucene.search.spell.LevensteinDistance;
 import org.apache.lucene.util.CollectionUtil;
-import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.cli.EnvironmentAwareCommand;

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/PluginCli.java

@@ -19,7 +19,7 @@
 
 package org.elasticsearch.plugins;
 
-import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.LoggingAwareMultiCommand;
 import org.elasticsearch.cli.MultiCommand;

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java

@@ -21,7 +21,7 @@
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;

docs/reference/aggregations/bucket/daterange-aggregation.asciidoc

@@ -1,8 +1,14 @@
 [[search-aggregations-bucket-daterange-aggregation]]
 === Date Range Aggregation
 
-A range aggregation that is dedicated for date values. The main difference between this aggregation and the normal <<search-aggregations-bucket-range-aggregation,range>> aggregation is that the `from` and `to` values can be expressed in <<date-math,Date Math>> expressions, and it is also possible to specify a date format by which the `from` and `to` response fields will be returned.
-Note that this aggregation includes the `from` value and excludes the `to` value for each range.
+A range aggregation that is dedicated for date values. The main difference
+between this aggregation and the normal
+<<search-aggregations-bucket-range-aggregation,range>>
+aggregation is that the `from` and `to` values can be expressed in
+<<date-math,Date Math>> expressions, and it is also possible to specify a date
+format by which the `from` and `to` response fields will be returned.
+Note that this aggregation includes the `from` value and excludes the `to` value
+for each range.
 
 Example:
 
@@ -30,8 +36,9 @@ POST /sales/_search?size=0
 <1> < now minus 10 months, rounded down to the start of the month.
 <2> >= now minus 10 months, rounded down to the start of the month.
 
-In the example above, we created two range buckets, the first will "bucket" all documents dated prior to 10 months ago and
-the second will "bucket" all documents dated since 10 months ago
+In the example above, we created two range buckets, the first will "bucket" all
+documents dated prior to 10 months ago and the second will "bucket" all
+documents dated since 10 months ago
 
 Response:
 
@@ -61,12 +68,52 @@ Response:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
+==== Missing Values
+
+The `missing` parameter defines how documents that are missing a value should
+be treated. By default they will be ignored but it is also possible to treat
+them as if they had a value. This is done by adding a set of fieldname :
+value mappings to specify default values per field.
+
+[source,js]
+--------------------------------------------------
+POST /sales/_search?size=0
+{
+   "aggs": {
+       "range": {
+           "date_range": {
+               "field": "date",
+               "missing": "1976/11/30",
+               "ranges": [
+                  { 
+                    "key": "Older",
+                    "to": "2016/02/01" 
+                  }, <1>
+                  { 
+                    "key": "Newer",
+                    "from": "2016/02/01", 
+                    "to" : "now/d" 
+                  }
+              ]
+          }
+      }
+   }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
+<1> Documents without a value in the `date` field will be added to the "Older"
+bucket, as if they had a date value of "1899-12-31". 
+
 [[date-format-pattern]]
 ==== Date Format/Pattern
 
-NOTE: this information was copied from http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[JodaDate]
+NOTE: this information was copied from
+http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[JodaDate]
 
-All ASCII letters are reserved as format pattern letters, which are defined as follows:
+All ASCII letters are reserved as format pattern letters, which are defined
+as follows:
 
 [options="header"]
 |=======
@@ -104,30 +151,41 @@ All ASCII letters are reserved as format pattern letters, which are defined as f
 
 The count of pattern letters determine the format.
 
-Text:: If the number of pattern letters is 4 or more, the full form is used; otherwise a short or abbreviated form is used if available.
+Text:: If the number of pattern letters is 4 or more, the full form is used;
+otherwise a short or abbreviated form is used if available.
 
-Number:: The minimum number of digits. Shorter numbers are zero-padded to this amount.
+Number:: The minimum number of digits. Shorter numbers are zero-padded to
+this amount.
 
-Year:: Numeric presentation for year and weekyear fields are handled specially. For example, if the count of 'y' is 2, the year will be displayed as the zero-based year of the century, which is two digits.
+Year:: Numeric presentation for year and weekyear fields are handled
+specially. For example, if the count of 'y' is 2, the year will be displayed
+as the zero-based year of the century, which is two digits.
 
 Month:: 3 or over, use text, otherwise use number.
 
-Zone:: 'Z' outputs offset without a colon, 'ZZ' outputs the offset with a colon, 'ZZZ' or more outputs the zone id.
+Zone:: 'Z' outputs offset without a colon, 'ZZ' outputs the offset with a
+colon, 'ZZZ' or more outputs the zone id.
 
 Zone names:: Time zone names ('z') cannot be parsed.
 
-Any characters in the pattern that are not in the ranges of ['a'..'z'] and ['A'..'Z'] will be treated as quoted text. For instance, characters like ':', '.', ' ', '#' and '?' will appear in the resulting time text even they are not embraced within single quotes.
+Any characters in the pattern that are not in the ranges of ['a'..'z'] and
+['A'..'Z'] will be treated as quoted text. For instance, characters like ':',
+ '.', ' ', '#' and '?' will appear in the resulting time text even they are
+ not embraced within single quotes.
 
 [[time-zones]]
 ==== Time zone in date range aggregations
 
-Dates can be converted from another time zone to UTC by specifying the `time_zone` parameter.
+Dates can be converted from another time zone to UTC by specifying the
+`time_zone` parameter.
 
-Time zones may either be specified as an ISO 8601 UTC offset (e.g. +01:00 or -08:00) or as one of
-the http://www.joda.org/joda-time/timezones.html[time zone ids] from the TZ database.
+Time zones may either be specified as an ISO 8601 UTC offset (e.g. +01:00 or
+-08:00) or as one of the http://www.joda.org/joda-time/timezones.html [time
+zone ids] from the TZ database.
 
-The `time_zone` parameter is also applied to rounding in date math expressions. As an example,
-to round to the beginning of the day in the CET time zone, you can do the following:
+The `time_zone` parameter is also applied to rounding in date math expressions.
+As an example, to round to the beginning of the day in the CET time zone, you
+can do the following:
 
 [source,js]
 --------------------------------------------------
@@ -156,7 +214,8 @@ POST /sales/_search?size=0
 
 ==== Keyed Response
 
-Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array:
+Setting the `keyed` flag to `true` will associate a unique string key with each
+bucket and return the ranges as a hash rather than an array:
 
 [source,js]
 --------------------------------------------------

docs/reference/index-modules/slowlog.asciidoc

@@ -22,6 +22,8 @@ index.search.slowlog.threshold.fetch.warn: 1s
 index.search.slowlog.threshold.fetch.info: 800ms
 index.search.slowlog.threshold.fetch.debug: 500ms
 index.search.slowlog.threshold.fetch.trace: 200ms
+
+index.search.slowlog.level: info
 --------------------------------------------------
 
 All of the above settings are _dynamic_ and are set per-index.

docs/reference/indices/put-mapping.asciidoc

@@ -1,24 +1,14 @@
 [[indices-put-mapping]]
 == Put Mapping
 
-The PUT mapping API allows you to add a new type to an existing index, or add new
-fields to an existing type:
+The PUT mapping API allows you to add fields to an existing index or to change search only settings of existing fields.
 
 [source,js]
 --------------------------------------------------
 PUT twitter <1>
 {}
 
 PUT twitter/_mapping/_doc <2>
-{
-  "properties": {
-    "name": {
-      "type": "text"
-    }
-  }
-}
-
-PUT twitter/_mapping/_doc <3>
 {
   "properties": {
     "email": {
@@ -29,8 +19,7 @@ PUT twitter/_mapping/_doc <3>
 --------------------------------------------------
 // CONSOLE
 <1> <<indices-create-index,Creates an index>> called `twitter` without any type mapping.
-<2> Uses the PUT mapping API to add a new mapping type called `user`.
-<3> Uses the PUT mapping API to add a new field called `email` to the `user` mapping type.
+<2> Uses the PUT mapping API to add a new field called `email` to the `_doc` mapping type.
 
 More information on how to define type mappings can be found in the
 <<mapping,mapping>> section.
@@ -125,4 +114,3 @@ PUT my_index/_mapping/_doc
 
 Each <<mapping-params,mapping parameter>> specifies whether or not its setting
 can be updated on an existing field.
-

docs/reference/search/request/script-fields.asciidoc

@@ -21,7 +21,7 @@ GET /_search
         "test2" : {
             "script" : {
                 "lang": "painless",
-                "source": "doc['my_field_name'].value * factor",
+                "source": "doc['my_field_name'].value * params.factor",
                 "params" : {
                     "factor"  : 2.0
                 }

docs/reference/setup/important-settings/error-file.asciidoc

@@ -0,0 +1,16 @@
+[[error-file-path]]
+=== JVM fatal error logs
+
+The <<rpm,RPM>> and <<deb,Debian>> package distributions default to configuring
+the JVM to write fatal error logs to `/var/lib/elasticsearch`; these are logs
+produced by the JVM when it encounters a fatal error (e.g., a segmentation
+fault). If this path is not suitable for receiving logs, you should modify the
+entry `-XX:ErrorFile=/var/lib/elasticsearch/hs_err_pid%p.log` in
+<<jvm-options,`jvm.options`>> to an alternate path.
+
+Note that the archive distributions do not configure the error file path by
+default. Instead, the JVM will default to writing to the working directory for
+the Elasticsearch process. If you wish to configure an error file path, you
+should modify the entry `#-XX:ErrorFile=/error/file/path` in
+<<jvm-options,`jvm.options`>> to remove the comment marker `#` and to specify an
+actual path.