Merge branch 'master' into ccr

* master:
  Cross-cluster search: preserve cluster alias in shard failures (#32608)
  Handle AlreadyClosedException when bumping primary term
  [TEST] Allow to run in FIPS JVM (#32607)
  [Test] Add ckb to the list of unsupported languages (#32611)
  SCRIPTING: Move Aggregation Scripts to their own context (#32068)
  Painless: Use LocalMethod Map For Lookup at Runtime (#32599)
  [TEST] Enhance failure message when bulk updates have failures
  [ML] Add ML result classes to protocol library (#32587)
  Suppress LicensingDocumentationIT.testPutLicense in release builds (#32613)
  [Rollup] Update wire version check after backport
  Suppress Wildfly test in FIPS JVMs (#32543)
  [Rollup] Improve ID scheme for rollup documents (#32558)
  ingest: doc: move Dot Expander Processor doc to correct position (#31743)
  [ML] Add some ML config classes to protocol library (#32502)
  [TEST]Split transport verification mode none tests (#32488)
  Core: Move helper date formatters over to java time (#32504)
  [Rollup] Remove builders from DateHistogramGroupConfig (#32555)
  [TEST} unmutes SearchAsyncActionTests and adds debugging info
  [ML] Add Detector config classes to protocol library (#32495)
  [Rollup] Remove builders from MetricConfig (#32536)
  Tests: Add rolling upgrade tests for watcher (#32428)
  Fix race between replica reset and primary promotion (#32442)

Author: dnhatn

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java

@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client.documentation;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
+import org.elasticsearch.Build;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
@@ -41,10 +41,10 @@
  * Documentation for Licensing APIs in the high level java client.
  * Code wrapped in {@code tag} and {@code end} tags is included in the docs.
  */
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32580")
 public class LicensingDocumentationIT extends ESRestHighLevelClientTestCase {
 
     public void testPutLicense() throws Exception {
+        assumeTrue("License is only valid when tested against snapshot/test builds", Build.CURRENT.isSnapshot());
         RestHighLevelClient client = highLevelClient();
         String license = "{\"license\": {\"uid\":\"893361dc-9749-4997-93cb-802e3d7fa4a8\",\"type\":\"gold\"," +
             "\"issue_date_in_millis\":1411948800000,\"expiry_date_in_millis\":1914278399999,\"max_nodes\":1,\"issued_to\":\"issued_to\"," +

docs/reference/ingest/ingest-node.asciidoc

@@ -1049,6 +1049,125 @@ understands this to mean `2016-04-01` as is explained in the <<date-math-index-n
 | `index_name_format`    | no        | yyyy-MM-dd                   | The format to be used when printing the parsed date into the index name. An valid Joda pattern is expected here.
 |======
 
+[[dot-expand-processor]]
+=== Dot Expander Processor
+
+Expands a field with dots into an object field. This processor allows fields
+with dots in the name to be accessible by other processors in the pipeline.
+Otherwise these <<accessing-data-in-pipelines,fields>> can't be accessed by any processor.
+
+[[dot-expender-options]]
+.Dot Expand Options
+[options="header"]
+|======
+| Name     | Required  | Default  | Description
+| `field`  | yes       | -        | The field to expand into an object field
+| `path`   | no        | -        | The field that contains the field to expand. Only required if the field to expand is part another object field, because the `field` option can only understand leaf fields.
+|======
+
+[source,js]
+--------------------------------------------------
+{
+  "dot_expander": {
+    "field": "foo.bar"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+For example the dot expand processor would turn this document:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo.bar" : "value"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+into:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo" : {
+    "bar" : "value"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+If there is already a `bar` field nested under `foo` then
+this processor merges the `foo.bar` field into it. If the field is
+a scalar value then it will turn that field into an array field.
+
+For example, the following document:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo.bar" : "value2",
+  "foo" : {
+    "bar" : "value1"
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+is transformed by the `dot_expander` processor into:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo" : {
+    "bar" : ["value1", "value2"]
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+If any field outside of the leaf field conflicts with a pre-existing field of the same name,
+then that field needs to be renamed first.
+
+Consider the following document:
+
+[source,js]
+--------------------------------------------------
+{
+  "foo": "value1",
+  "foo.bar": "value2"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+Then the `foo` needs to be renamed first before the `dot_expander`
+processor is applied. So in order for the `foo.bar` field to properly
+be expanded into the `bar` field under the `foo` field the following
+pipeline should be used:
+
+[source,js]
+--------------------------------------------------
+{
+  "processors" : [
+    {
+      "rename" : {
+        "field" : "foo",
+        "target_field" : "foo.bar""
+      }
+    },
+    {
+      "dot_expander": {
+        "field": "foo.bar"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+The reason for this is that Ingest doesn't know how to automatically cast
+a scalar field to an object field.
+
 [[fail-processor]]
 === Fail Processor
 Raises an exception. This is useful for when
@@ -2058,125 +2177,6 @@ Converts a string to its uppercase equivalent.
 --------------------------------------------------
 // NOTCONSOLE
 
-[[dot-expand-processor]]
-=== Dot Expander Processor
-
-Expands a field with dots into an object field. This processor allows fields
-with dots in the name to be accessible by other processors in the pipeline.
-Otherwise these <<accessing-data-in-pipelines,fields>> can't be accessed by any processor.
-
-[[dot-expender-options]]
-.Dot Expand Options
-[options="header"]
-|======
-| Name     | Required  | Default  | Description
-| `field`  | yes       | -        | The field to expand into an object field
-| `path`   | no        | -        | The field that contains the field to expand. Only required if the field to expand is part another object field, because the `field` option can only understand leaf fields.
-|======
-
-[source,js]
---------------------------------------------------
-{
-  "dot_expander": {
-    "field": "foo.bar"
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-For example the dot expand processor would turn this document:
-
-[source,js]
---------------------------------------------------
-{
-  "foo.bar" : "value"
-}
---------------------------------------------------
-// NOTCONSOLE
-
-into:
-
-[source,js]
---------------------------------------------------
-{
-  "foo" : {
-    "bar" : "value"
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-If there is already a `bar` field nested under `foo` then
-this processor merges the `foo.bar` field into it. If the field is
-a scalar value then it will turn that field into an array field.
-
-For example, the following document:
-
-[source,js]
---------------------------------------------------
-{
-  "foo.bar" : "value2",
-  "foo" : {
-    "bar" : "value1"
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-is transformed by the `dot_expander` processor into:
-
-[source,js]
---------------------------------------------------
-{
-  "foo" : {
-    "bar" : ["value1", "value2"]
-  }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-If any field outside of the leaf field conflicts with a pre-existing field of the same name,
-then that field needs to be renamed first.
-
-Consider the following document:
-
-[source,js]
---------------------------------------------------
-{
-  "foo": "value1",
-  "foo.bar": "value2"
-}
---------------------------------------------------
-// NOTCONSOLE
-
-Then the `foo` needs to be renamed first before the `dot_expander`
-processor is applied. So in order for the `foo.bar` field to properly
-be expanded into the `bar` field under the `foo` field the following
-pipeline should be used:
-
-[source,js]
---------------------------------------------------
-{
-  "processors" : [
-    {
-      "rename" : {
-        "field" : "foo",
-        "target_field" : "foo.bar""
-      }
-    },
-    {
-      "dot_expander": {
-        "field": "foo.bar"
-      }
-    }
-  ]
-}
---------------------------------------------------
-// NOTCONSOLE
-
-The reason for this is that Ingest doesn't know how to automatically cast
-a scalar field to an object field.
-
 [[urldecode-processor]]
 === URL Decode Processor
 URL-decodes a string

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java

@@ -34,10 +34,12 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.index.mapper.GeoPointFieldMapper.GeoPointFieldType;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.GeoPointFieldMapper.GeoPointFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.script.BucketAggregationScript;
+import org.elasticsearch.script.BucketAggregationSelectorScript;
 import org.elasticsearch.script.ClassPermission;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.FilterScript;
@@ -54,6 +56,7 @@
 import java.security.PrivilegedAction;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -112,6 +115,17 @@ protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundE
         } else if (context.instanceClazz.equals(ExecutableScript.class)) {
             ExecutableScript.Factory factory = (p) -> new ExpressionExecutableScript(expr, p);
             return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(BucketAggregationScript.class)) {
+            return context.factoryClazz.cast(newBucketAggregationScriptFactory(expr));
+        } else if (context.instanceClazz.equals(BucketAggregationSelectorScript.class)) {
+            BucketAggregationScript.Factory factory = newBucketAggregationScriptFactory(expr);
+            BucketAggregationSelectorScript.Factory wrappedFactory = parameters -> new BucketAggregationSelectorScript(parameters) {
+                @Override
+                public boolean execute() {
+                    return factory.newInstance(getParams()).execute() == 1.0;
+                }
+            };
+            return context.factoryClazz.cast(wrappedFactory);
         } else if (context.instanceClazz.equals(FilterScript.class)) {
             FilterScript.Factory factory = (p, lookup) -> newFilterScript(expr, lookup, p);
             return context.factoryClazz.cast(factory);
@@ -122,6 +136,37 @@ protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundE
         throw new IllegalArgumentException("expression engine does not know how to handle script context [" + context.name + "]");
     }
 
+    private static BucketAggregationScript.Factory newBucketAggregationScriptFactory(Expression expr) {
+        return parameters -> {
+            ReplaceableConstDoubleValues[] functionValuesArray =
+                new ReplaceableConstDoubleValues[expr.variables.length];
+            Map<String, ReplaceableConstDoubleValues> functionValuesMap = new HashMap<>();
+            for (int i = 0; i < expr.variables.length; ++i) {
+                functionValuesArray[i] = new ReplaceableConstDoubleValues();
+                functionValuesMap.put(expr.variables[i], functionValuesArray[i]);
+            }
+            return new BucketAggregationScript(parameters) {
+                @Override
+                public double execute() {
+                    getParams().forEach((name, value) -> {
+                        ReplaceableConstDoubleValues placeholder = functionValuesMap.get(name);
+                        if (placeholder == null) {
+                            throw new IllegalArgumentException("Error using " + expr + ". " +
+                                "The variable [" + name + "] does not exist in the executable expressions script.");
+                        } else if (value instanceof Number == false) {
+                            throw new IllegalArgumentException("Error using " + expr + ". " +
+                                "Executable expressions scripts can only process numbers." +
+                                "  The variable [" + name + "] is not a number.");
+                        } else {
+                            placeholder.setValue(((Number) value).doubleValue());
+                        }
+                    });
+                    return expr.evaluate(functionValuesArray);
+                }
+            };
+        };
+    }
+
     private SearchScript.LeafFactory newSearchScript(Expression expr, SearchLookup lookup, @Nullable Map<String, Object> vars) {
         MapperService mapper = lookup.doc().mapperService();
         // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
@@ -267,7 +312,7 @@ public void setDocument(int docid) {
             };
         };
     }
-    
+
     private ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLookup lookup, @Nullable Map<String, Object> vars) {
         SearchScript.LeafFactory searchLeafFactory = newSearchScript(expr, lookup, vars);
         return new ScoreScript.LeafFactory() {
@@ -284,17 +329,17 @@ public ScoreScript newInstance(LeafReaderContext ctx) throws IOException {
                     public double execute() {
                         return script.runAsDouble();
                     }
-    
+
                     @Override
                     public void setDocument(int docid) {
                         script.setDocument(docid);
                     }
-    
+
                     @Override
                     public void setScorer(Scorer scorer) {
                         script.setScorer(scorer);
                     }
-    
+
                     @Override
                     public double get_score() {
                         return script.getScore();