Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (31 commits)
  Fix eclipse build. (#28236)
  Never return null from Strings.tokenizeToStringArray (#28224)
  Fallback to TransportMasterNodeAction for cluster health retries (#28195)
  [Docs] Changes to ingest.asciidoc (#28212)
  TEST: Update logging for testAckedIndexing
  [GEO] Deprecate field parameter in GeoBoundingBoxQueryBuilder
  [GEO] Add WKT Support to GeoBoundingBoxQueryBuilder
  Avoid doing redundant work when checking for self references. (#26927)
  Fix casts in HotThreads. (#27578)
  Ignore the `-snapshot` suffix when comparing the Lucene version in the build and the docs. (#27927)
  Allow update of `eager_global_ordinals` on `_parent`. (#28014)
  Painless: Add whitelist extensions (#28161)
  Fix daitch_mokotoff phonetic filter to use the dedicated Lucene filter (#28225)
  Fix NPE on composite aggregation with sub-aggregations that need scores (#28129)
  #28045 restore removed import after backport
  Fix synonym phrase query expansion for cross_fields parsing (#28045)
  Introduce elasticsearch-core jar (#28191)
  upgrade to lucene 7.2.1 (#28218)
  [Docs] Fix an error in painless-types.asciidoc (#28221)
  Consistent updates of IndexShardSnapshotStatus (#28130)
  ...

Author: martijnvg

build.gradle

@@ -183,6 +183,7 @@ subprojects {
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':server',
     "org.elasticsearch:elasticsearch-cli:${version}": ':server:cli',
+    "org.elasticsearch:elasticsearch-core:${version}": ':libs:elasticsearch-core',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 6.2.0
-lucene            = 7.2.0
+lucene            = 7.2.1
 
 # optional dependencies
 spatial4j         = 0.6

client/rest/build.gradle

@@ -72,6 +72,7 @@ forbiddenApisTest {
 }
 
 // JarHell is part of es server, which we don't want to pull in
+// TODO: Not anymore. Now in elasticsearch-core
 jarHell.enabled=false
 
 namingConventions {

client/sniffer/build.gradle

@@ -75,6 +75,7 @@ dependencyLicenses {
 }
 
 // JarHell is part of es server, which we don't want to pull in
+// TODO: Not anymore. Now in elasticsearch-core
 jarHell.enabled=false
 
 namingConventions {

client/test/build.gradle

@@ -49,6 +49,7 @@ forbiddenApisTest {
 }
 
 // JarHell is part of es server, which we don't want to pull in
+// TODO: Not anymore. Now in elasticsearch-core
 jarHell.enabled=false
 
 // TODO: should we have licenses for our test deps?

docs/Versions.asciidoc

@@ -1,6 +1,6 @@
 :version:               6.2.0
 :major-version:         6.x
-:lucene_version:        7.2.0
+:lucene_version:        7.2.1
 :lucene_version_path:   7_2_0
 :branch:                6.x
 :jdk:                   1.8.0_131

docs/painless/painless-types.asciidoc

@@ -311,7 +311,7 @@ to floating point types.
 | int   | explicit | explicit | explicit |          | implicit | implicit | implicit
 | long  | explicit | explicit | explicit | explicit |          | implicit | implicit
 | float | explicit | explicit | explicit | explicit | explicit |          | implicit
-| float | explicit | explicit | explicit | explicit | explicit | explicit |
+| double | explicit | explicit | explicit | explicit | explicit | explicit |
 |====
 
 
@@ -376,7 +376,7 @@ cast would normally be required between the non-def types.
 def x;          // Declare def variable x and set it to null
 x = 3;          // Set the def variable x to the literal 3 with an implicit
                 //   cast from int to def
-double a = x;   // Declare double variable y and set it to def variable x,
+double a = x;   // Declare double variable a and set it to def variable x,
                 //   which contains a double
 int b = x;      // ERROR: Results in a run-time error because an explicit cast is
                 //   required to cast from a double to an int

docs/reference/cluster/tasks.asciidoc

@@ -195,3 +195,71 @@ The following command will change the grouping to parent tasks:
 GET _tasks?group_by=parents
 --------------------------------------------------
 // CONSOLE
+
+The grouping can be disabled by specifying `none` as a `group_by` parameter:
+
+[source,js]
+--------------------------------------------------
+GET _tasks?group_by=none
+--------------------------------------------------
+// CONSOLE
+
+[float]
+=== Identifying running tasks
+
+The `X-Opaque-Id` header, when provided on the HTTP request header, is going to be returned as a header in the response as well as
+in the `headers` field for in the task information. This allows to track certain calls, or associate certain tasks with
+a the client that started them:
+
+[source,sh]
+--------------------------------------------------
+curl -i -H "X-Opaque-Id: 123456" "http://localhost:9200/_tasks?group_by=parents"
+--------------------------------------------------
+// NOTCONSOLE
+
+The result will look similar to the following:
+
+[source,js]
+--------------------------------------------------
+HTTP/1.1 200 OK
+X-Opaque-Id: 123456 <1>
+content-type: application/json; charset=UTF-8
+content-length: 831
+
+{
+  "tasks" : {
+    "u5lcZHqcQhu-rUoFaqDphA:45" : {
+      "node" : "u5lcZHqcQhu-rUoFaqDphA",
+      "id" : 45,
+      "type" : "transport",
+      "action" : "cluster:monitor/tasks/lists",
+      "start_time_in_millis" : 1513823752749,
+      "running_time_in_nanos" : 293139,
+      "cancellable" : false,
+      "headers" : {
+        "X-Opaque-Id" : "123456" <2>
+      },
+      "children" : [
+        {
+          "node" : "u5lcZHqcQhu-rUoFaqDphA",
+          "id" : 46,
+          "type" : "direct",
+          "action" : "cluster:monitor/tasks/lists[n]",
+          "start_time_in_millis" : 1513823752750,
+          "running_time_in_nanos" : 92133,
+          "cancellable" : false,
+          "parent_task_id" : "u5lcZHqcQhu-rUoFaqDphA:45",
+          "headers" : {
+            "X-Opaque-Id" : "123456" <3>
+          }
+        }
+      ]
+    }
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+<1> id as a part of the response header
+<2> id for the tasks that was initiated by the REST request
+<3> the child task of the task initiated by the REST request

docs/reference/index-modules.asciidoc

@@ -186,7 +186,7 @@ specific index module:
 
      The maximum number of characters that will be analyzed for a highlight request.
      This setting is only applicable when highlighting is requested on a text that was indexed without offsets or term vectors.
-     Defaults to `10000`.
+     By default this settings is unset in 6.x, defaults to `-1`.
 
  `index.max_terms_count`::
 

docs/reference/ingest.asciidoc

@@ -3,26 +3,27 @@
 
 [partintro]
 --
-You can use ingest node to pre-process documents before the actual indexing takes place.
-This pre-processing happens by an ingest node that intercepts bulk and index requests, applies the
-transformations, and then passes the documents back to the index or bulk APIs.
+Use an ingest node to pre-process documents before the actual document indexing happens. 
+The ingest node intercepts bulk and index requests, it applies transformations, and it then
+passes the documents back to the index or bulk APIs.
 
-You can enable ingest on any node or even have dedicated ingest nodes. Ingest is enabled by default
-on all nodes. To disable ingest on a node, configure the following setting in the `elasticsearch.yml` file:
+All nodes enable ingest by default, so any node can handle ingest tasks. You can also create
+dedicated ingest nodes. To disable ingest for a node, configure the following setting in the
+elasticsearch.yml file:
 
 [source,yaml]
 --------------------------------------------------
 node.ingest: false
 --------------------------------------------------
 
-To pre-process documents before indexing, you <<pipeline,define a pipeline>> that specifies
-a series of <<ingest-processors,processors>>. Each processor transforms the document in some way.
-For example, you may have a pipeline that consists of one processor that removes a field from
-the document followed by another processor that renames a field. Configured pipelines are then stored
-in the <<cluster-state,cluster state>>.
+To pre-process documents before indexing, <<pipeline,define a pipeline>> that specifies a series of
+<<ingest-processors,processors>>. Each processor transforms the document in some specific way. For example, a
+pipeline might have one processor that removes a field from the document, followed by
+another processor that renames a field. The <<cluster-state,cluster state>> then stores
+the configured pipelines.
 
-To use a pipeline, you simply specify the `pipeline` parameter on an index or bulk request to
-tell the ingest node which pipeline to use. For example:
+To use a pipeline, simply specify the `pipeline` parameter on an index or bulk request. This
+way, the ingest node knows which pipeline to use. For example:
 
 [source,js]
 --------------------------------------------------

docs/reference/migration/migrate_6_0/analysis.asciidoc

@@ -16,6 +16,8 @@ created in 5.x.
 Highlighting a text that was indexed without offsets or term vectors,
 requires analysis of this text in memory real time during the search request.
 For large texts this analysis may take substantial amount of time and memory.
-To protect against this, the maximum number of characters that will be analyzed has been
-limited to 10000. This default limit can be changed
-for a particular index with the index setting `index.highlight.max_analyzed_offset`.
+To protect against this, the maximum number of characters that to be analyzed will be
+limited to 10000 in the next major Elastic version. For this version, by default the limit
+is not set. A deprecation warning will be issued when an analyzed text exceeds 10000.
+ The limit can be set for a particular index with the index setting
+`index.highlight.max_analyzed_offset`.

docs/reference/query-dsl/geo-bounding-box-query.asciidoc

@@ -180,6 +180,31 @@ GET /_search
 --------------------------------------------------
 // CONSOLE
 
+[float]
+===== Bounding Box as Well-Known Text (WKT)
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "wkt" : "BBOX (-74.1, -71.12, 40.73, 40.01)"
+                    }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
 [float]
 ===== Geohash
 

docs/reference/search/request/highlighting.asciidoc

@@ -106,9 +106,9 @@ needs highlighting. The `plain` highlighter always uses plain highlighting.
 
 [WARNING]
 Plain highlighting for large texts may require substantial amount of time and memory.
-To protect against this, the maximum number of text characters that will be analyzed has been
-limited to 10000. This default limit can be changed
-for a particular index with the index setting `index.highlight.max_analyzed_offset`.
+To protect against this, the maximum number of text characters to be analyzed will be
+limited to 10000 in the next major Elastic version. The default limit is not set for this version,
+but can be set for a particular index with the index setting `index.highlight.max_analyzed_offset`.
 
 [[highlighting-settings]]
 ==== Highlighting Settings

libs/elasticsearch-core/build.gradle

@@ -0,0 +1,81 @@
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.build'
+apply plugin: 'nebula.optional-base'
+apply plugin: 'nebula.maven-base-publish'
+apply plugin: 'nebula.maven-scm'
+
+archivesBaseName = 'elasticsearch-core'
+
+publishing {
+    publications {
+        nebula {
+            artifactId = archivesBaseName
+        }
+    }
+}
+
+dependencies {
+    compile "org.apache.logging.log4j:log4j-api:${versions.log4j}"
+
+    testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+    testCompile "junit:junit:${versions.junit}"
+    testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+
+    if (isEclipse == false || project.path == ":libs:elasticsearch-core-tests") {
+        testCompile("org.elasticsearch.test:framework:${version}") {
+            exclude group: 'org.elasticsearch', module: 'elasticsearch-core'
+        }
+    }
+}
+
+forbiddenApisMain {
+    // elasticsearch-core does not depend on server
+    // TODO: Need to decide how we want to handle for forbidden signatures with the changes to core
+    signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+
+if (isEclipse) {
+    // in eclipse the project is under a fake root, we need to change around the source sets
+    sourceSets {
+        if (project.path == ":libs:elasticsearch-core") {
+            main.java.srcDirs = ['java']
+            main.resources.srcDirs = ['resources']
+        } else {
+            test.java.srcDirs = ['java']
+            test.resources.srcDirs = ['resources']
+        }
+    }
+}
+
+thirdPartyAudit.excludes = [
+        // from log4j
+        'org/osgi/framework/AdaptPermission',
+        'org/osgi/framework/AdminPermission',
+        'org/osgi/framework/Bundle',
+        'org/osgi/framework/BundleActivator',
+        'org/osgi/framework/BundleContext',
+        'org/osgi/framework/BundleEvent',
+        'org/osgi/framework/SynchronousBundleListener',
+        'org/osgi/framework/wiring/BundleWire',
+        'org/osgi/framework/wiring/BundleWiring'
+]

libs/elasticsearch-core/licenses/log4j-api-2.9.1.jar.sha1

@@ -0,0 +1 @@
+7a2999229464e7a324aa503c0a52ec0f05efe7bd