Merge branch '6.2' of github.com:elastic/elasticsearch into 6.2

Author: Sue-Gallagher

buildSrc/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 6.2.4
+elasticsearch     = 6.2.5
 lucene            = 7.2.1
 
 # optional dependencies

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -707,10 +707,13 @@ private void installPlugin(Terminal terminal, boolean isBatch, Path tmpRoot,
         final PluginInfo info = loadPluginInfo(terminal, tmpRoot, isBatch, env);
         // read optional security policy (extra permissions), if it exists, confirm or warn the user
         Path policy = tmpRoot.resolve(PluginInfo.ES_PLUGIN_POLICY);
+        final Set<String> permissions;
         if (Files.exists(policy)) {
-            Set<String> permissions = PluginSecurity.parsePermissions(policy, env.tmpFile());
-            PluginSecurity.confirmPolicyExceptions(terminal, permissions, info.hasNativeController(), isBatch);
+            permissions = PluginSecurity.parsePermissions(policy, env.tmpFile());
+        } else {
+            permissions = Collections.emptySet();
         }
+        PluginSecurity.confirmPolicyExceptions(terminal, permissions, info.hasNativeController(), isBatch);
 
         final Path destination = env.pluginsFile().resolve(info.getName());
         deleteOnFailure.add(destination);

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -1191,6 +1191,59 @@ private Function<byte[], String> checksumAndString(final MessageDigest digest, f
         return bytes -> MessageDigests.toHexString(digest.digest(bytes)) + s;
     }
 
+    // checks the plugin requires a policy confirmation, and does not install when that is rejected by the user
+    // the plugin is installed after this method completes
+    private void assertPolicyConfirmation(Tuple<Path, Environment> env, String pluginZip, String... warnings) throws Exception {
+        for (int i = 0; i < warnings.length; ++i) {
+            String warning = warnings[i];
+            for (int j = 0; j < i; ++j) {
+                terminal.addTextInput("y"); // accept warnings we have already tested
+            }
+            // default answer, does not install
+            terminal.addTextInput("");
+            UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+            assertEquals("installation aborted by user", e.getMessage());
+
+            assertThat(terminal.getOutput(), containsString("WARNING: " + warning));
+            try (Stream<Path> fileStream = Files.list(env.v2().pluginsFile())) {
+                assertThat(fileStream.collect(Collectors.toList()), empty());
+            }
+
+            // explicitly do not install
+            terminal.reset();
+            for (int j = 0; j < i; ++j) {
+                terminal.addTextInput("y"); // accept warnings we have already tested
+            }
+            terminal.addTextInput("n");
+            e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
+            assertEquals("installation aborted by user", e.getMessage());
+            assertThat(terminal.getOutput(), containsString("WARNING: " + warning));
+            try (Stream<Path> fileStream = Files.list(env.v2().pluginsFile())) {
+                assertThat(fileStream.collect(Collectors.toList()), empty());
+            }
+        }
+
+        // allow installation
+        terminal.reset();
+        for (int j = 0; j < warnings.length; ++j) {
+            terminal.addTextInput("y");
+        }
+        installPlugin(pluginZip, env.v1());
+        for (String warning : warnings) {
+            assertThat(terminal.getOutput(), containsString("WARNING: " + warning));
+        }
+    }
+
+    public void testPolicyConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        writePluginSecurityPolicy(pluginDir, "setAccessible", "setFactory");
+        String pluginZip = createPluginUrl("fake", pluginDir);
+
+        assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions");
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
     public void testMetaPluginPolicyConfirmation() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path metaDir = createPluginDir(temp);
@@ -1204,32 +1257,60 @@ public void testMetaPluginPolicyConfirmation() throws Exception {
         writePlugin("fake2", fake2Dir);
         String pluginZip = createMetaPluginUrl("meta-plugin", metaDir);
 
-        // default answer, does not install
-        terminal.addTextInput("");
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertEquals("installation aborted by user", e.getMessage());
-        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
-        try (Stream<Path> fileStream = Files.list(env.v2().pluginsFile())) {
-            assertThat(fileStream.collect(Collectors.toList()), empty());
-        }
+        assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions");
+        assertMetaPlugin("meta-plugin", "fake1", metaDir, env.v2());
+        assertMetaPlugin("meta-plugin", "fake2", metaDir, env.v2());
+    }
 
-        // explicitly do not install
-        terminal.reset();
-        terminal.addTextInput("n");
-        e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertEquals("installation aborted by user", e.getMessage());
-        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
-        try (Stream<Path> fileStream = Files.list(env.v2().pluginsFile())) {
-            assertThat(fileStream.collect(Collectors.toList()), empty());
-        }
+    public void testNativeControllerConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPluginUrl("fake", pluginDir, "has.native.controller", "true");
 
-        // allow installation
-        terminal.reset();
-        terminal.addTextInput("y");
-        installPlugin(pluginZip, env.v1());
-        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
+        assertPolicyConfirmation(env, pluginZip, "plugin forks a native controller");
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
+    public void testMetaPluginNativeControllerConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path metaDir = createPluginDir(temp);
+        Path fake1Dir = metaDir.resolve("fake1");
+        Files.createDirectory(fake1Dir);
+        writePlugin("fake1", fake1Dir, "has.native.controller", "true");
+        Path fake2Dir = metaDir.resolve("fake2");
+        Files.createDirectory(fake2Dir);
+        writePlugin("fake2", fake2Dir);
+        String pluginZip = createMetaPluginUrl("meta-plugin", metaDir);
+
+        assertPolicyConfirmation(env, pluginZip, "plugin forks a native controller");
         assertMetaPlugin("meta-plugin", "fake1", metaDir, env.v2());
         assertMetaPlugin("meta-plugin", "fake2", metaDir, env.v2());
     }
 
+    public void testNativeControllerAndPolicyConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        writePluginSecurityPolicy(pluginDir, "setAccessible", "setFactory");
+        String pluginZip = createPluginUrl("fake", pluginDir, "has.native.controller", "true");
+
+        assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions", "plugin forks a native controller");
+        assertPlugin("fake", pluginDir, env.v2());
+    }
+
+    public void testMetaPluginNativeControllerAndPolicyConfirmation() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path metaDir = createPluginDir(temp);
+        Path fake1Dir = metaDir.resolve("fake1");
+        Files.createDirectory(fake1Dir);
+        writePluginSecurityPolicy(fake1Dir, "setAccessible", "setFactory");
+        writePlugin("fake1", fake1Dir);
+        Path fake2Dir = metaDir.resolve("fake2");
+        Files.createDirectory(fake2Dir);
+        writePlugin("fake2", fake2Dir, "has.native.controller", "true");
+        String pluginZip = createMetaPluginUrl("meta-plugin", metaDir);
+
+        assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions", "plugin forks a native controller");
+        assertMetaPlugin("meta-plugin", "fake1", metaDir, env.v2());
+        assertMetaPlugin("meta-plugin", "fake2", metaDir, env.v2());
+    }
 }

docs/Versions.asciidoc

@@ -1,4 +1,4 @@
-:version:               6.2.3
+:version:               6.2.4
 :major-version:         6.x
 :lucene_version:        7.2.1
 :lucene_version_path:   7_2_0

docs/plugins/analysis.asciidoc

@@ -53,6 +53,7 @@ A number of analysis plugins have been contributed by our community:
 * https://github.com/duydo/elasticsearch-analysis-vietnamese[Vietnamese Analysis Plugin] (by Duy Do)
 * https://github.com/ofir123/elasticsearch-network-analysis[Network Addresses Analysis Plugin] (by Ofir123)
 * https://github.com/medcl/elasticsearch-analysis-string2int[String2Integer Analysis Plugin] (by Medcl)
+* https://github.com/ZarHenry96/elasticsearch-dandelion-plugin[Dandelion Analysis Plugin] (by ZarHenry96)
 
 include::analysis-icu.asciidoc[]
 

docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc

@@ -27,11 +27,13 @@ POST /sales/_search?size=0
 // CONSOLE
 // TEST[setup:sales]
 
-Available expressions for interval: `year`, `quarter`, `month`, `week`, `day`, `hour`, `minute`, `second`
+Available expressions for interval: `year` (`1y`), `quarter` (`1q`), `month` (`1M`), `week` (`1w`),
+`day` (`1d`), `hour` (`1h`), `minute` (`1m`), `second` (`1s`)
 
 Time values can also be specified via abbreviations supported by <<time-units,time units>> parsing.
 Note that fractional time values are not supported, but you can address this by shifting to another
-time unit (e.g., `1.5h` could instead be specified as `90m`).
+time unit (e.g., `1.5h` could instead be specified as `90m`). Also note that time intervals larger than
+than days do not support arbitrary values but can only be one unit large (e.g. `1y` is valid, `2y` is not).
 
 [source,js]
 --------------------------------------------------

docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc

@@ -5,8 +5,7 @@ A `single-value` metrics aggregation that calculates an approximate count of
 distinct values. Values can be extracted either from specific fields in the
 document or generated by a script.
 
-Assume you are indexing books and would like to count the unique authors that
-match a query:
+Assume you are indexing store sales and would like to count the unique number of sold products that match a query:
 
 [source,js]
 --------------------------------------------------

docs/reference/cluster/tasks.asciidoc

@@ -64,7 +64,7 @@ It is also possible to retrieve information for a particular task:
 
 [source,js]
 --------------------------------------------------
-GET _tasks/task_id:1 <1>
+GET _tasks/node_id:1 <1>
 --------------------------------------------------
 // CONSOLE
 // TEST[catch:missing]

docs/reference/docs/delete-by-query.asciidoc

@@ -284,9 +284,12 @@ executed again in order to conform to `requests_per_second`.
 
 `failures`::
 
-Array of all indexing failures. If this is non-empty then the request aborted
-because of those failures. See `conflicts` for how to prevent version conflicts
-from aborting the operation.
+Array of failures if there were any unrecoverable errors during the process. If
+this is non-empty then the request aborted because of those failures.
+Delete-by-query is implemented using batches and any failure causes the entire
+process to abort but all failures in the current batch are collected into the
+array. You can use the `conflicts` option to prevent reindex from aborting on
+version conflicts.
 
 
 [float]

docs/reference/docs/delete.asciidoc

@@ -39,11 +39,14 @@ The result of the above delete operation is:
 [[delete-versioning]]
 === Versioning
 
-Each document indexed is versioned. When deleting a document, the
-`version` can be specified to make sure the relevant document we are
-trying to delete is actually being deleted and it has not changed in the
-meantime. Every write operation executed on a document, deletes included,
-causes its version to be incremented.
+Each document indexed is versioned. When deleting a document, the `version` can
+be specified to make sure the relevant document we are trying to delete is
+actually being deleted and it has not changed in the meantime. Every write
+operation executed on a document, deletes included, causes its version to be
+incremented. The version number of a deleted document remains available for a
+short time after deletion to allow for control of concurrent operations. The
+length of time for which a deleted document's version remains available is
+determined by the `index.gc_deletes` index setting and defaults to 60 seconds.
 
 [float]
 [[delete-routing]]

docs/reference/docs/index_.asciidoc

@@ -229,14 +229,14 @@ The result of the above index operation is:
     },
     "_index" : "twitter",
     "_type" : "_doc",
-    "_id" : "6a8ca01c-7896-48e9-81cc-9f70661fcb32",
+    "_id" : "W0tpsmIBdwcYyG50zbta",
     "_version" : 1,
     "_seq_no" : 0,
     "_primary_term" : 1,
     "result": "created"
 }
 --------------------------------------------------
-// TESTRESPONSE[s/6a8ca01c-7896-48e9-81cc-9f70661fcb32/$body._id/ s/"successful" : 2/"successful" : 1/]
+// TESTRESPONSE[s/W0tpsmIBdwcYyG50zbta/$body._id/ s/"successful" : 2/"successful" : 1/]
 
 [float]
 [[index-routing]]

docs/reference/docs/reindex.asciidoc

@@ -161,12 +161,12 @@ POST _reindex
 
 `index` and `type` in `source` can both be lists, allowing you to copy from
 lots of sources in one request. This will copy documents from the `_doc` and
-`post` types in the `twitter` and `blog` index. The copied documents would include the 
-`post` type in the `twitter` index and the `_doc` type in the `blog` index. For more 
+`post` types in the `twitter` and `blog` index. The copied documents would include the
+`post` type in the `twitter` index and the `_doc` type in the `blog` index. For more
 specific parameters, you can use `query`.
 
-The Reindex API makes no effort to handle ID collisions. For such issues, the target index 
-will remain valid, but it's not easy to predict which document will survive because 
+The Reindex API makes no effort to handle ID collisions. For such issues, the target index
+will remain valid, but it's not easy to predict which document will survive because
 the iteration order isn't well defined.
 
 [source,js]
@@ -666,9 +666,11 @@ executed again in order to conform to `requests_per_second`.
 
 `failures`::
 
-Array of all indexing failures. If this is non-empty then the request aborted
-because of those failures. See `conflicts` for how to prevent version conflicts
-from aborting the operation.
+Array of failures if there were any unrecoverable errors during the process. If
+this is non-empty then the request aborted because of those failures. Reindex
+is implemented using batches and any failure causes the entire process to abort
+but all failures in the current batch are collected into the array. You can use
+the `conflicts` option to prevent reindex from aborting on version conflicts.
 
 [float]
 [[docs-reindex-task-api]]
@@ -1004,7 +1006,7 @@ number for most indices. If slicing manually or otherwise tuning
 automatic slicing, use these guidelines.
 
 Query performance is most efficient when the number of `slices` is equal to the
-number of shards in the index. If that number is large (e.g. 500), 
+number of shards in the index. If that number is large (e.g. 500),
 choose a lower number as too many `slices` will hurt performance. Setting
 `slices` higher than the number of shards generally does not improve efficiency
 and adds overhead.
@@ -1018,7 +1020,7 @@ documents being reindexed and cluster resources.
 [float]
 === Reindex daily indices
 
-You can use `_reindex` in combination with <<modules-scripting-painless, Painless>> 
+You can use `_reindex` in combination with <<modules-scripting-painless, Painless>>
 to reindex daily indices to apply a new template to the existing documents.
 
 Assuming you have indices consisting of documents as follows:

docs/reference/docs/update-by-query.asciidoc

@@ -338,9 +338,13 @@ executed again in order to conform to `requests_per_second`.
 
 `failures`::
 
-Array of all indexing failures. If this is non-empty then the request aborted
-because of those failures. See `conflicts` for how to prevent version conflicts
-from aborting the operation.
+Array of failures if there were any unrecoverable errors during the process. If
+this is non-empty then the request aborted because of those failures.
+Update-by-query is implemented using batches and any failure causes the entire
+process to abort but all failures in the current batch are collected into the
+array. You can use the `conflicts` option to prevent reindex from aborting on
+version conflicts.
+
 
 
 [float]

docs/reference/glossary.asciidoc

@@ -61,6 +61,15 @@
   `object`. The mapping also allows you to define (amongst other things)
   how the value for a field should be analyzed.
 
+[[glossary-filter]] filter ::
+
+  A filter is a non-scoring <<glossary-query,query>>, meaning that it does not score documents.
+  It is only concerned about answering the question - "Does this document match?". 
+  The answer is always a simple, binary yes or no. This kind of query is said to be made 
+  in a <<query-filter-context,filter context>>, 
+  hence it is called a filter. Filters are simple checks for set inclusion or exclusion. 
+  In most cases, the goal of filtering is to reduce the number of documents that have to be examined.
+
 [[glossary-index]] index ::
 
   An index is like a _table_ in a relational database. It has a
@@ -105,6 +114,16 @@
   +
   See also <<glossary-routing,routing>>
 
+[[glossary-query]] query ::
+
+  A query is the basic component of a search. A search can be defined by one or more queries 
+  which can be mixed and matched in endless combinations. While <<glossary-filter,filters>> are
+  queries that only determine if a document matches, those queries that also calculate how well
+  the document matches are known as "scoring queries". Those queries assign it a score, which is 
+  later used to sort matched documents. Scoring queries take more resources than <<glossary-filter,non scoring queries>> 
+  and their query results are not cacheable. As a general rule, use query clauses for full-text 
+  search or for any condition that requires scoring, and use filters for everything else.
+
  [[glossary-replica-shard]] replica shard ::
 
   Each <<glossary-primary-shard,primary shard>> can have zero or more
@@ -161,8 +180,9 @@
 
   A term is an exact value that is indexed in Elasticsearch. The terms
   `foo`, `Foo`, `FOO` are NOT equivalent. Terms (i.e. exact values) can
-  be searched for using _term_ queries. +
-   See also <<glossary-text,text>> and <<glossary-analysis,analysis>>.
+  be searched for using _term_ queries.
+  +
+  See also <<glossary-text,text>> and <<glossary-analysis,analysis>>.
 
 [[glossary-text]] text ::
 

docs/reference/how-to/search-speed.asciidoc

@@ -377,3 +377,19 @@ criteria called <<search-adaptive-replica,adaptive replica selection>> to select
 the best copy of the data based on response time, service time, and queue size
 of the node containing each copy of the shard. This can improve query throughput
 and reduce latency for search-heavy applications.
+
+=== Tune your queries with the Profile API
+
+You can also analyse how expensive each component of your queries and 
+aggregations are using the {ref}/search-profile.html[Profile API]. This might 
+allow you to tune your queries to be less expensive, resulting in a positive 
+performance result and reduced load. Also note that Profile API payloads can be 
+easily visualised for better readability in the 
+{kibana-ref}/xpack-profiler.html[Search Profiler], which is a Kibana dev tools 
+UI available in all X-Pack licenses, including the free X-Pack Basic license.
+
+Some caveats to the Profile API are that:
+
+ - the Profile API as a debugging tool adds significant overhead to search execution and can also have a very verbose output
+ - given the added overhead, the resulting took times are not reliable indicators of actual took time, but can be used comparatively between clauses for relative timing differences
+ - the Profile API is best for exploring possible reasons behind the most costly clauses of a query but isn't intended for accurately measuring absolute timings of each clause