Merge branch 'master' into feature/searchable-snapshots

Author: tlrx

TESTING.asciidoc

@@ -340,7 +340,8 @@ zip, and deb packages and all the plugins.  It will then run the tests on every
 available system. This will take a very long time.
 +
 Fortunately, the various systems under test have their own Gradle tasks under
-`qa/os`. To find out what packaging combinations can be tested on a system, run
+`qa/os`. To find the systems tested, do a listing of the `qa/os` directory.
+To find out what packaging combinations can be tested on a system, run
 the `tasks` task.  For example:
 +
 ----------------------------------
@@ -360,28 +361,7 @@ will remain running and you'll have to stop them manually with `./gradlew --stop
 
 All the regular vagrant commands should just work so you can get a shell in a
 VM running trusty by running
-`vagrant up ubuntu-1604 --provider virtualbox && vagrant ssh ubuntu-1604`.
-
-These are the linux flavors supported, all of which we provide images for
-
-* ubuntu-1604 aka xenial
-* ubuntu-1804 aka bionic beaver
-* debian-8 aka jessie
-* debian-9 aka stretch, the current debian stable distribution
-* centos-6
-* centos-7
-* rhel-8
-* fedora-28
-* fedora-29
-* oel-6 aka Oracle Enterprise Linux 6
-* oel-7 aka Oracle Enterprise Linux 7
-* sles-12
-* opensuse-42 aka Leap
-
-We're missing the following from the support matrix because there aren't high
-quality boxes available in vagrant atlas:
-
-* sles-11
+`vagrant up ubuntu-1804 --provider virtualbox && vagrant ssh ubuntu-1804`.
 
 === Testing packaging on Windows
 

buildSrc/src/main/java/org/elasticsearch/gradle/tool/DockerUtils.java

@@ -146,17 +146,7 @@ public static void assertDockerIsAvailable(Project project, List<String> tasks)
             throwDockerRequiredException(message);
         }
 
-        if (availability.version == null) {
-            final String message = String.format(
-                Locale.ROOT,
-                "Docker is required to run the following task%s, but it doesn't appear to be running: \n%s",
-                tasks.size() > 1 ? "s" : "",
-                String.join("\n", tasks)
-            );
-            throwDockerRequiredException(message);
-        }
-
-        if (availability.isVersionHighEnough == false) {
+        if (availability.lastCommand.isSuccess() && availability.isVersionHighEnough == false) {
             final String message = String.format(
                 Locale.ROOT,
                 "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]",
@@ -168,9 +158,10 @@ public static void assertDockerIsAvailable(Project project, List<String> tasks)
         // Some other problem, print the error
         final String message = String.format(
             Locale.ROOT,
-            "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n"
-                + "the problem is that Docker exited with exit code [%d] with standard error output [%s]",
+            "a problem occurred while using Docker from [%s]%s yet it is required to run the following task%s: \n%s\n"
+                + "the problem is that Docker exited with exit code [%d] with standard error output:\n%s",
             availability.path,
+            availability.version == null ? "" : " v" + availability.version,
             tasks.size() > 1 ? "s" : "",
             String.join("\n", tasks),
             availability.lastCommand.exitCode,
@@ -213,8 +204,8 @@ private static Result runCommand(Project project, String... args) {
             spec.setCommandLine((Object[]) args);
             spec.setStandardOutput(stdout);
             spec.setErrorOutput(stderr);
+            spec.setIgnoreExitValue(true);
         });
-
         return new Result(execResult.getExitValue(), stdout.toString(), stderr.toString());
     }
 

buildSrc/src/main/resources/checkstyle.xml

@@ -18,7 +18,7 @@
     <property name="message" value="Empty javadoc comments are forbidden"/>
   </module>
 
-  <!-- 
+  <!--
     We include snippets that are wrapped in `// tag` and `// end` into the
     docs, stripping the leading spaces. If the context is wider than 76
     characters then it'll need to scroll. This fails the build if it sees
@@ -103,5 +103,31 @@
       <property name="ignoreComments" value="true" />
     </module>
     <!-- end Orwellian suppression of Serializable -->
+
+    <!-- Forbid equality comparisons with `true` -->
+    <module name="DescendantToken">
+      <property name="tokens" value="EQUAL"/>
+      <property name="limitedTokens" value="LITERAL_TRUE"/>
+      <property name="maximumNumber" value="0"/>
+      <property name="maximumDepth" value="1"/>
+      <message key="descendant.token.max" value="Do not check for equality with 'true', since it is implied"/>
+    </module>
+
+    <!-- Forbid using '!' for logical negations in favour of checking
+         against 'false' explicitly. -->
+    <!-- This is disabled for now because there are many, many violations,
+         hence the rule is reporting at the "warning" severity. -->
+
+    <!--
+    <module name="DescendantToken">
+      <property name="severity" value="warning"/>
+      <property name="tokens" value="EXPR"/>
+      <property name="limitedTokens" value="LNOT"/>
+      <property name="maximumNumber" value="0"/>
+      <property name="maximumDepth" value="1"/>
+      <message key="descendant.token.max" value="Do not negate boolean expressions with '!', but check explicitly with '== false' as it is more explicit"/>
+    </module>
+    -->
+
   </module>
 </module>

buildSrc/src/main/resources/forbidden/jdk-signatures.txt

@@ -59,7 +59,7 @@ java.net.MulticastSocket#<init>(int)
 java.net.ServerSocket#<init>(int)
 java.net.ServerSocket#<init>(int,int)
 
-@defaultMessage use NetworkAddress format/formatAddress to print IP or IP+ports
+@defaultMessage use NetworkAddress format() to print IP or IP+ports
 java.net.InetAddress#toString()
 java.net.InetAddress#getHostAddress()
 java.net.Inet4Address#getHostAddress()

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -1242,7 +1242,7 @@ public void testFieldCaps() throws IOException {
         assertEquals(1, fieldResponse.size());
 
         FieldCapabilities expectedTextCapabilities = new FieldCapabilities(
-            "field", "text", true, false, Collections.emptyMap());
+            "field", "text", true, false, null, null, null, Collections.emptyMap());
         assertEquals(expectedTextCapabilities, fieldResponse.get("text"));
     }
 

docs/reference/cat/nodes.asciidoc

@@ -87,7 +87,7 @@ Used disk space, such as `259.8gb`.
 Available disk space, such as `198.4gb`.
 
 `disk.used_percent`, `dup`, `diskUsedPercent`::
-Used disk space percentage, such as `198.4gb`.
+Used disk space percentage, such as `47`.
 
 `heap.current`, `hc`, `heapCurrent`::
 Used heap, such as `311.2mb`.

docs/reference/cluster/nodes-reload-secure-settings.asciidoc

@@ -4,60 +4,59 @@
 <titleabbrev>Nodes reload secure settings</titleabbrev>
 ++++
 
+Reloads the keystore on nodes in the cluster.
 
-The cluster nodes reload secure settings API is used to re-load the keystore on each node.
+[[cluster-nodes-reload-secure-settings-api-request]]
+==== {api-request-title}
 
-[source,console]
---------------------------------------------------
-POST _nodes/reload_secure_settings
-POST _nodes/nodeId1,nodeId2/reload_secure_settings
---------------------------------------------------
-// TEST[setup:node]
-// TEST[s/nodeId1,nodeId2/*/]
+`POST _nodes/reload_secure_settings` +
+`POST _nodes/<nodes/reload_secure_settings`
 
-The first command reloads the keystore on each node. The seconds allows
-to selectively target `nodeId1` and `nodeId2`. The node selection options are
-detailed <<cluster-nodes,here>>.
+[[cluster-nodes-reload-secure-settings-api-desc]]
+==== {api-description-title}
 
-NOTE: {es} requires consistent secure settings across the cluster nodes, but this consistency is not enforced.
-Hence, reloading specific nodes is not standard. It is only justifiable when retrying failed reload operations.
+<<secure-settings,Secure settings>> are stored in an on-disk keystore. Certain
+of these settings are <<reloadable-secure-settings,reloadable>>. That is, you
+can change them on disk and reload them without restarting any nodes in the
+cluster. When you have updated reloadable secure settings in your keystore, you
+can use this API to reload those settings on each node.
 
-==== Reload Password Protected Secure Settings
+When the {es} keystore is password protected and not simply obfuscated, you must
+provide the password for the keystore when you reload the secure settings.
+Reloading the settings for the whole cluster assumes that all nodes' keystores
+are protected with the same password; this method is allowed only when
+<<tls-transport,inter-node communications are encrypted>>. Alternatively, you can
+reload the secure settings on each node by locally accessing the API and passing
+the node-specific {es} keystore password.
 
-When the {es} keystore is password protected and not simply obfuscated, the password for the keystore needs
-to be provided in the request to reload the secure settings.
-Reloading the settings for the whole cluster assumes that all nodes' keystores are protected with the same password
-and is only allowed when {ref}/configuring-tls.html#tls-transport[node to node communications are encrypted]
+[[cluster-nodes-reload-secure-settings-path-params]]
+==== {api-path-parms-title}
 
-[source,js]
---------------------------------------------------
-POST _nodes/reload_secure_settings
-{
-  "reload_secure_settings": "s3cr3t" <1>
-}
---------------------------------------------------
-// NOTCONSOLE
+`<nodes>`::
+    (Optional, string) The names of particular nodes in the cluster to target.
+    For example, `nodeId1,nodeId2`. For node selection options, see
+    <<cluster-nodes>>.
+    
+NOTE: {es} requires consistent secure settings across the cluster nodes, but
+this consistency is not enforced. Hence, reloading specific nodes is not
+standard. It is justifiable only when retrying failed reload operations.
 
-<1> The common password that the {es} keystore is encrypted with in every node of the cluster.
+[[cluster-nodes-reload-secure-settings-api-request-body]]
+==== {api-request-body-title}
 
-Alternatively the secure settings can be reloaded on a per node basis, locally accessing the API and passing the
-node-specific {es} keystore password.
+`reload_secure_settings`::
+  (Optional, string) The password for the {es} keystore.
 
-[source,js]
+[[cluster-nodes-reload-secure-settings-api-example]]
+==== {api-examples-title}
+
+[source,console]
 --------------------------------------------------
-POST _nodes/_local/reload_secure_settings
-{
-  "reload_secure_settings": "s3cr3t" <1>
-}
+POST _nodes/reload_secure_settings
+POST _nodes/nodeId1,nodeId2/reload_secure_settings
 --------------------------------------------------
-// NOTCONSOLE
-
-<1> The password that the {es} keystore is encrypted with on the local node.
-
-
-[float]
-[[rest-reload-secure-settings]]
-==== REST Reload Secure Settings Response
+// TEST[setup:node]
+// TEST[s/nodeId1,nodeId2/*/]
 
 The response contains the `nodes` object, which is a map, keyed by the
 node id. Each value has the node `name` and an optional `reload_exception`
@@ -82,3 +81,27 @@ that was thrown during the reload process, if any.
 --------------------------------------------------
 // TESTRESPONSE[s/"my_cluster"/$body.cluster_name/]
 // TESTRESPONSE[s/"pQHNt5rXTTWNvUgOrdynKg"/\$node_name/]
+
+The following example uses a common password for the {es} keystore on every
+node of the cluster:
+
+[source,js]
+--------------------------------------------------
+POST _nodes/reload_secure_settings
+{
+  "reload_secure_settings": "s3cr3t"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+The following example uses a password for the {es} keystore on the local node:
+
+[source,js]
+--------------------------------------------------
+POST _nodes/_local/reload_secure_settings
+{
+  "reload_secure_settings": "s3cr3t"
+}
+--------------------------------------------------
+// NOTCONSOLE
+

docs/reference/cluster/stats.asciidoc

@@ -438,6 +438,19 @@ The API returns the following response:
          "fixed_bit_set_memory_in_bytes": 0,
          "max_unsafe_auto_id_timestamp" : -9223372036854775808,
          "file_sizes": {}
+      },
+      "mappings": {
+        "field_types": []
+      },
+      "analysis": {
+        "char_filter_types": [],
+        "tokenizer_types": [],
+        "filter_types": [],
+        "analyzer_types": [],
+        "built_in_char_filters": [],
+        "built_in_tokenizers": [],
+        "built_in_filters": [],
+        "built_in_analyzers": []
       }
    },
    "nodes": {
@@ -554,6 +567,7 @@ The API returns the following response:
 // TESTRESPONSE[s/"processor_stats": \{[^\}]*\}/"processor_stats": $body.$_path/]
 // TESTRESPONSE[s/"count": \{[^\}]*\}/"count": $body.$_path/]
 // TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/]
+// TESTRESPONSE[s/"field_types": \[[^\]]*\]/"field_types": $body.$_path/]
 // TESTRESPONSE[s/: true|false/: $body.$_path/]
 // TESTRESPONSE[s/: (\-)?[0-9]+/: $body.$_path/]
 // TESTRESPONSE[s/: "[^"]*"/: $body.$_path/]

docs/reference/how-to/search-speed.asciidoc

@@ -414,7 +414,7 @@ queries, this can speed up queries significantly.
 [[faster-prefix-queries]]
 === Faster prefix queries with `index_prefixes`
 
-The <<text,`text`>> field has an <<index-phrases,`index_prefixes`>> option that
+The <<text,`text`>> field has an <<index-prefixes,`index_prefixes`>> option that
 indexes prefixes of all terms and is automatically leveraged by query parsers to
 run prefix queries. If your use-case involves running lots of prefix queries,
 this can speed up queries significantly.

docs/reference/ingest/processors/csv.asciidoc

@@ -21,7 +21,7 @@ include::common-options.asciidoc[]
 {
   "csv": {
     "field": "my_field",
-    "target_fields": ["field1, field2"],
+    "target_fields": ["field1, field2"]
   }
 }
 --------------------------------------------------

docs/reference/setup/secure-settings.asciidoc

@@ -25,7 +25,7 @@ are node-specific settings that must have the same value on every node.
 Just like the settings values in `elasticsearch.yml`, changes to the keystore
 contents are not automatically applied to the running {es} node. Re-reading
 settings requires a node restart. However, certain secure settings are marked as
-*reloadable*. Such settings can be re-read and applied on a running node.
+*reloadable*. Such settings can be <<cluster-nodes-reload-secure-settings, re-read and applied on a running node>>.
 
 The values of all secure settings, *reloadable* or not, must be identical
 across all cluster nodes. After making the desired secure settings changes,
@@ -50,8 +50,8 @@ dependent on these settings have been changed. Everything should look as if the
 settings had the new value from the start.
 
 When changing multiple *reloadable* secure settings, modify all of them on each
-cluster node, then issue a `reload_secure_settings` call instead of reloading
-after each modification.
+cluster node, then issue a <<cluster-nodes-reload-secure-settings, `reload_secure_settings`>>
+call instead of reloading after each modification.
 
 There are reloadable secure settings for:
 

docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -143,14 +143,14 @@ protected boolean preserveTemplatesUponCompletion() {
 
     protected boolean isSLMTest() {
         String testName = getTestName();
-        return testName != null && (testName.contains("/slm/") || testName.contains("\\slm\\") ||
+        return testName != null && (testName.contains("/slm/") || testName.contains("\\slm\\") || (testName.contains("\\slm/")) ||
             // TODO: Remove after backport of https://github.com/elastic/elasticsearch/pull/48705 which moves SLM docs to correct folder
-            testName.contains("/ilm/") || testName.contains("\\ilm\\"));
+            testName.contains("/ilm/") || testName.contains("\\ilm\\") || testName.contains("\\ilm/"));
     }
 
     protected boolean isILMTest() {
         String testName = getTestName();
-        return testName != null && (testName.contains("/ilm/") || testName.contains("\\ilm\\"));
+        return testName != null && (testName.contains("/ilm/") || testName.contains("\\ilm\\") || testName.contains("\\ilm/"));
     }
 
     protected boolean isMachineLearningTest() {

libs/grok/src/main/resources/patterns/grok-patterns

@@ -49,7 +49,7 @@ URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
 URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
 
 # Months: January, Feb, 3, 03, 12, December
-MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b
+MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y|i)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b
 MONTHNUM (?:0?[1-9]|1[0-2])
 MONTHNUM2 (?:0[1-9]|1[0-2])
 MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])