Merge remote-tracking branch 'elastic/master' into reduce-metadata-writes-master

Author: ywelsch

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -498,6 +498,13 @@ private static boolean shouldRunDockerTests(Project project) {
                 return true;
 
             case LINUX:
+                // We don't attempt to check the current flavor and version of Linux unless we're
+                // running in CI, because we don't want to stop people running the Docker tests in
+                // their own environments if they really want to.
+                if (BuildParams.isCi() == false) {
+                    return true;
+                }
+
                 // Only some hosts in CI are configured with Docker. We attempt to work out the OS
                 // and version, so that we know whether to expect to find Docker. We don't attempt
                 // to probe for whether Docker is available, because that doesn't tell us whether

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -179,6 +179,11 @@ public void keystore(String key, FileSupplier valueSupplier) {
         nodes.all(each -> each.keystore(key, valueSupplier));
     }
 
+    @Override
+    public void cliSetup(String binTool, CharSequence... args) {
+        nodes.all(each -> each.cliSetup(binTool, args));
+    }
+
     @Override
     public void setting(String key, String value) {
         nodes.all(each -> each.setting(key, value));

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -124,6 +124,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final LazyPropertyMap<String, CharSequence> settings = new LazyPropertyMap<>("Settings", this);
     private final LazyPropertyMap<String, CharSequence> keystoreSettings = new LazyPropertyMap<>("Keystore", this);
     private final LazyPropertyMap<String, File> keystoreFiles = new LazyPropertyMap<>("Keystore files", this, FileEntry::new);
+    private final LazyPropertyList<CliEntry> cliSetup = new LazyPropertyList<>("CLI setup commands", this);
     private final LazyPropertyMap<String, CharSequence> systemProperties = new LazyPropertyMap<>("System properties", this);
     private final LazyPropertyMap<String, CharSequence> environment = new LazyPropertyMap<>("Environment", this);
     private final LazyPropertyList<CharSequence> jvmArgs = new LazyPropertyList<>("JVM arguments", this);
@@ -301,6 +302,11 @@ public void keystore(String key, FileSupplier valueSupplier) {
         keystoreFiles.put(key, valueSupplier);
     }
 
+    @Override
+    public void cliSetup(String binTool, CharSequence... args) {
+        cliSetup.add(new CliEntry(binTool, args));
+    }
+
     @Override
     public void setting(String key, String value) {
         settings.put(key, value);
@@ -475,6 +481,14 @@ public synchronized void start() {
             ));
         }
 
+        if (cliSetup.isEmpty() == false) {
+            logToProcessStdout("Running " + cliSetup.size() + " setup commands");
+
+            for (CliEntry entry : cliSetup) {
+                runElasticsearchBinScript(entry.executable, entry.args);
+            }
+        }
+
         logToProcessStdout("Starting Elasticsearch process");
         startElasticsearchProcess();
     }
@@ -623,7 +637,7 @@ public void user(Map<String, String> userSpec) {
         credentials.add(cred);
     }
 
-    private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
+    private void runElasticsearchBinScriptWithInput(String input, String tool, CharSequence... args) {
         if (
             Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false &&
                 Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false
@@ -642,12 +656,12 @@ private void runElasticsearchBinScriptWithInput(String input, String tool, Strin
                         .supply()
                 );
                 spec.args(
-                    OS.<List<String>>conditional()
+                    OS.<List<CharSequence>>conditional()
                         .onWindows(() -> {
-                            ArrayList<String> result = new ArrayList<>();
+                            ArrayList<CharSequence> result = new ArrayList<>();
                             result.add("/c");
                             result.add("bin\\" + tool + ".bat");
-                            for (String arg : args) {
+                            for (CharSequence arg : args) {
                                 result.add(arg);
                             }
                             return result;
@@ -663,7 +677,7 @@ private void runElasticsearchBinScriptWithInput(String input, String tool, Strin
         }
     }
 
-    private void runElasticsearchBinScript(String tool, String... args) {
+    private void runElasticsearchBinScript(String tool, CharSequence... args) {
         runElasticsearchBinScriptWithInput("", tool, args);
     }
 
@@ -1205,6 +1219,11 @@ public List<?> getKeystoreFiles() {
         return keystoreFiles.getNormalizedCollection();
     }
 
+    @Nested
+    public List<?> getCliSetup() {
+        return cliSetup.getNormalizedCollection();
+    }
+
     @Nested
     public List<?> getSettings() {
         return settings.getNormalizedCollection();
@@ -1376,4 +1395,24 @@ public File getFile() {
             return file;
         }
     }
+
+    private static class CliEntry {
+        private String executable;
+        private CharSequence[] args;
+
+        CliEntry(String executable, CharSequence[] args) {
+            this.executable = executable;
+            this.args = args;
+        }
+
+        @Input
+        public String getExecutable() {
+            return executable;
+        }
+
+        @Input
+        public CharSequence[] getArgs() {
+            return args;
+        }
+    }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -58,6 +58,8 @@ public interface TestClusterConfiguration {
 
     void keystore(String key, FileSupplier valueSupplier);
 
+    void cliSetup(String binTool, CharSequence... args);
+
     void setting(String key, String value);
 
     void setting(String key, String value, PropertyNormalization normalization);

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -2168,8 +2168,8 @@ public void testGetTrainedModels() throws Exception {
             GetTrainedModelsResponse getTrainedModelsResponse = execute(
                 GetTrainedModelsRequest.getAllTrainedModelConfigsRequest(),
                 machineLearningClient::getTrainedModels, machineLearningClient::getTrainedModelsAsync);
-            assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(numberOfModels));
-            assertThat(getTrainedModelsResponse.getCount(), equalTo(5L));
+            assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(numberOfModels + 1));
+            assertThat(getTrainedModelsResponse.getCount(), equalTo(5L + 1));
         }
         {
             GetTrainedModelsResponse getTrainedModelsResponse = execute(
@@ -2192,7 +2192,7 @@ public void testGetTrainedModels() throws Exception {
 
     public void testGetTrainedModelsStats() throws Exception {
         MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
-        String modelIdPrefix = "get-trained-model-stats-";
+        String modelIdPrefix = "a-get-trained-model-stats-";
         int numberOfModels = 5;
         for (int i = 0; i < numberOfModels; ++i) {
             String modelId = modelIdPrefix + i;
@@ -2224,8 +2224,8 @@ public void testGetTrainedModelsStats() throws Exception {
             GetTrainedModelsStatsResponse getTrainedModelsStatsResponse = execute(
                 GetTrainedModelsStatsRequest.getAllTrainedModelStatsRequest(),
                 machineLearningClient::getTrainedModelsStats, machineLearningClient::getTrainedModelsStatsAsync);
-            assertThat(getTrainedModelsStatsResponse.getTrainedModelStats(), hasSize(numberOfModels));
-            assertThat(getTrainedModelsStatsResponse.getCount(), equalTo(5L));
+            assertThat(getTrainedModelsStatsResponse.getTrainedModelStats(), hasSize(numberOfModels + 1));
+            assertThat(getTrainedModelsStatsResponse.getCount(), equalTo(5L + 1));
             assertThat(getTrainedModelsStatsResponse.getTrainedModelStats().get(0).getPipelineCount(), equalTo(1));
             assertThat(getTrainedModelsStatsResponse.getTrainedModelStats().get(1).getPipelineCount(), equalTo(0));
         }

docs/reference/aggregations/bucket/composite-aggregation.asciidoc

@@ -287,6 +287,72 @@ Time zones may either be specified as an ISO 8601 UTC offset (e.g. `+01:00` or
 `-08:00`)  or as a timezone id, an identifier used in the TZ database like
 `America/Los_Angeles`.
 
+*Offset*
+
+include::datehistogram-aggregation.asciidoc[tag=offset-explanation]
+
+[source,console,id=composite-aggregation-datehistogram-offset-example]
+----
+PUT my_index/_doc/1?refresh
+{
+  "date": "2015-10-01T05:30:00Z"
+}
+
+PUT my_index/_doc/2?refresh
+{
+  "date": "2015-10-01T06:30:00Z"
+}
+
+GET my_index/_search?size=0
+{
+  "aggs": {
+    "my_buckets": {
+      "composite" : {
+        "sources" : [
+          {
+            "date": {
+              "date_histogram" : {
+                "field": "date",
+                "calendar_interval": "day",
+                "offset": "+6h",
+                "format": "iso8601"
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}
+----
+
+include::datehistogram-aggregation.asciidoc[tag=offset-result-intro]
+
+[source,console-result]
+----
+{
+  ...
+  "aggregations": {
+    "my_buckets": {
+      "after_key": { "date": "2015-10-01T06:00:00.000Z" },
+      "buckets": [
+        {
+          "key": { "date": "2015-09-30T06:00:00.000Z" },
+          "doc_count": 1
+        },
+        {
+          "key": { "date": "2015-10-01T06:00:00.000Z" },
+          "doc_count": 1
+        }
+      ]
+    }
+  }
+}
+----
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+include::datehistogram-aggregation.asciidoc[tag=offset-note]
+
 ===== Mixing different values source
 
 The `sources` parameter accepts an array of values source.

docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc

@@ -461,16 +461,19 @@ the bucket covering that day will only hold data for 23 hours instead of the usu
 where you'll have only a 11h bucket on the morning of 27 March when the DST shift
 happens.
 
+[[search-aggregations-bucket-datehistogram-offset]]
 ===== Offset
 
+// tag::offset-explanation[]
 Use the `offset` parameter to change the start value of each bucket by the
 specified positive (`+`) or negative offset (`-`) duration, such as `1h` for
 an hour, or `1d` for a day. See <<time-units>> for more possible time
 duration options.
 
 For example, when using an interval of `day`, each bucket runs from midnight
-to midnight.  Setting the `offset` parameter to `+6h` changes each bucket
+to midnight. Setting the `offset` parameter to `+6h` changes each bucket
 to run from 6am to 6am:
+// end::offset-explanation[]
 
 [source,console,id=datehistogram-aggregation-offset-example]
 -----------------------------
@@ -498,8 +501,10 @@ GET my_index/_search?size=0
 }
 -----------------------------
 
+// tag::offset-result-intro[]
 Instead of a single bucket starting at midnight, the above request groups the
 documents into buckets starting at 6am:
+// end::offset-result-intro[]
 
 [source,console-result]
 -----------------------------
@@ -525,8 +530,10 @@ documents into buckets starting at 6am:
 -----------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
+// tag::offset-note[]
 NOTE: The start `offset` of each bucket is calculated after `time_zone`
 adjustments have been made.
+// end::offset-note[]
 
 ===== Keyed Response
 

docs/reference/analysis/tokenfilters/reverse-tokenfilter.asciidoc

@@ -4,4 +4,90 @@
 <titleabbrev>Reverse</titleabbrev>
 ++++
 
-A token filter of type `reverse` that simply reverses each token.
+Reverses each token in a stream. For example, you can use the `reverse` filter
+to change `cat` to `tac`.
+
+Reversed tokens are useful for suffix-based searches,
+such as finding words that end in `-ion` or searching file names by their
+extension.
+
+This filter uses Lucene's
+https://lucene.apache.org/core/{lucene_version_path}/analyzers-common/org/apache/lucene/analysis/reverse/ReverseStringFilter.html[ReverseStringFilter].
+
+[[analysis-reverse-tokenfilter-analyze-ex]]
+==== Example
+
+The following <<indices-analyze,analyze API>> request uses the `reverse`
+filter to reverse each token in `quick fox jumps`:
+
+[source,console]
+--------------------------------------------------
+GET _analyze
+{
+  "tokenizer" : "standard",
+  "filter" : ["reverse"],
+  "text" : "quick fox jumps"
+}
+--------------------------------------------------
+
+The filter produces the following tokens:
+
+[source,text]
+--------------------------------------------------
+[ kciuq, xof, spmuj ]
+--------------------------------------------------
+
+/////////////////////
+[source,console-result]
+--------------------------------------------------
+{
+  "tokens" : [
+    {
+      "token" : "kciuq",
+      "start_offset" : 0,
+      "end_offset" : 5,
+      "type" : "<ALPHANUM>",
+      "position" : 0
+    },
+    {
+      "token" : "xof",
+      "start_offset" : 6,
+      "end_offset" : 9,
+      "type" : "<ALPHANUM>",
+      "position" : 1
+    },
+    {
+      "token" : "spmuj",
+      "start_offset" : 10,
+      "end_offset" : 15,
+      "type" : "<ALPHANUM>",
+      "position" : 2
+    }
+  ]
+}
+--------------------------------------------------
+/////////////////////
+
+[[analysis-reverse-tokenfilter-analyzer-ex]]
+==== Add to an analyzer
+
+The following <<indices-create-index,create index API>> request uses the
+`reverse` filter to configure a new
+<<analysis-custom-analyzer,custom analyzer>>.
+
+[source,console]
+--------------------------------------------------
+PUT reverse_example
+{
+  "settings" : {
+    "analysis" : {
+      "analyzer" : {
+        "whitespace_reverse" : {
+          "tokenizer" : "whitespace",
+          "filter" : ["reverse"]
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------