Move runtime fields to server (#69223)

This commit makes a start on moving runtime fields to server.

The runtime field types are now built-in. The dynamic fields builder extension (needed to make dynamic:runtime work) is removed: further simplifications are needed but are left for a follow-up. It is still possible to plug in custom runtime field types through the same extension point that the xpack plugin used to use.

The runtime fields script contexts are now also moved to server, and the corresponding whitelists are part of painless-lang with the exception of grok and dissect which will require additional work.

The runtime fields xpack plugin still exists to hold the grok and dissect extensions needed, which depend on grok and dissect lib and we will address as a follow-up. Also, the qa tests and yaml tests are yet to be moved.

Despite the need for additional work, runtime fields are fully functional and users should not be impacted by this change.

Author: javanna

modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java

@@ -37,6 +37,13 @@
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.runtimefields.mapper.BooleanFieldScript;
+import org.elasticsearch.runtimefields.mapper.DateFieldScript;
+import org.elasticsearch.runtimefields.mapper.DoubleFieldScript;
+import org.elasticsearch.runtimefields.mapper.GeoPointFieldScript;
+import org.elasticsearch.runtimefields.mapper.IpFieldScript;
+import org.elasticsearch.runtimefields.mapper.LongFieldScript;
+import org.elasticsearch.runtimefields.mapper.StringFieldScript;
 import org.elasticsearch.script.IngestScript;
 import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.ScriptContext;
@@ -88,6 +95,15 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens
         ingest.add(ingestWhitelist);
         map.put(IngestScript.CONTEXT, ingest);
 
+        // Functions available to runtime fields
+        map.put(BooleanFieldScript.CONTEXT, getRuntimeFieldWhitelist("boolean"));
+        map.put(DateFieldScript.CONTEXT, getRuntimeFieldWhitelist("date"));
+        map.put(DoubleFieldScript.CONTEXT, getRuntimeFieldWhitelist("double"));
+        map.put(LongFieldScript.CONTEXT, getRuntimeFieldWhitelist("long"));
+        map.put(StringFieldScript.CONTEXT, getRuntimeFieldWhitelist("string"));
+        map.put(GeoPointFieldScript.CONTEXT, getRuntimeFieldWhitelist("geopoint"));
+        map.put(IpFieldScript.CONTEXT, getRuntimeFieldWhitelist("ip"));
+
         // Execute context gets everything
         List<Whitelist> test = new ArrayList<>(Whitelist.BASE_WHITELISTS);
         test.add(movFnWhitelist);
@@ -99,6 +115,14 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens
         whitelists = map;
     }
 
+    private static List<Whitelist> getRuntimeFieldWhitelist(String fieldType) {
+        List<Whitelist> scriptField = new ArrayList<>(Whitelist.BASE_WHITELISTS);
+        Whitelist whitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class,
+            "org.elasticsearch.runtimefields." + fieldType + ".txt");
+        scriptField.add(whitelist);
+        return scriptField;
+    }
+
     private final SetOnce<PainlessScriptEngine> painlessScriptEngine = new SetOnce<>();
 
     @Override

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.boolean.txt

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for boolean-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.BooleanFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.BooleanFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.BooleanFieldScript, boolean) bound_to org.elasticsearch.runtimefields.mapper.BooleanFieldScript$Emit
+}
+

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.date.txt

@@ -0,0 +1,25 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for date-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.DateFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.DateFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.DateFieldScript, long) bound_to org.elasticsearch.runtimefields.mapper.DateFieldScript$Emit
+    # Parse a value from the source to millis since epoch
+    long parse(org.elasticsearch.runtimefields.mapper.DateFieldScript, def) bound_to org.elasticsearch.runtimefields.mapper.DateFieldScript$Parse
+}
+
+
+

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.double.txt

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for double-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.DoubleFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.DoubleFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.DoubleFieldScript, double) bound_to org.elasticsearch.runtimefields.mapper.DoubleFieldScript$Emit
+}
+

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.geopoint.txt

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for ip-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.GeoPointFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.GeoPointFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.GeoPointFieldScript, double, double) bound_to org.elasticsearch.runtimefields.mapper.GeoPointFieldScript$Emit
+}

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.ip.txt

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for ip-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.IpFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.IpFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.IpFieldScript, String) bound_to org.elasticsearch.runtimefields.mapper.IpFieldScript$Emit
+}

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.long.txt

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for long-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.LongFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.LongFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.LongFieldScript, long) bound_to org.elasticsearch.runtimefields.mapper.LongFieldScript$Emit
+}

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.string.txt

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for string-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.StringFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.StringFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.StringFieldScript, String) bound_to org.elasticsearch.runtimefields.mapper.StringFieldScript$Emit
+}

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -8,14 +8,6 @@
 
 package org.elasticsearch.index.mapper;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.function.Function;
-
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
@@ -31,6 +23,14 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.SearchExecutionContext;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Function;
+
 /** A parser for documents, given mappings from a DocumentMapper */
 final class DocumentParser {
 
@@ -39,11 +39,10 @@ final class DocumentParser {
     private final DynamicRuntimeFieldsBuilder dynamicRuntimeFieldsBuilder;
 
     DocumentParser(NamedXContentRegistry xContentRegistry,
-                   Function<DateFormatter, Mapper.TypeParser.ParserContext> dateParserContext,
-                   DynamicRuntimeFieldsBuilder dynamicRuntimeFieldsBuilder) {
+                   Function<DateFormatter, Mapper.TypeParser.ParserContext> dateParserContext) {
         this.xContentRegistry = xContentRegistry;
         this.dateParserContext = dateParserContext;
-        this.dynamicRuntimeFieldsBuilder = dynamicRuntimeFieldsBuilder;
+        this.dynamicRuntimeFieldsBuilder = org.elasticsearch.runtimefields.mapper.DynamicRuntimeFieldsBuilder.INSTANCE;
     }
 
     ParsedDocument parseDocument(SourceToParse source,

server/src/main/java/org/elasticsearch/index/mapper/DynamicRuntimeFieldsBuilder.java

@@ -9,7 +9,6 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.plugins.MapperPlugin;
 
 import static org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
 
@@ -18,7 +17,6 @@
  * Plugins that provide runtime field implementations can also plug in their implementation of this interface
  * to define how leaf fields of each supported type can be dynamically created in dynamic runtime mode.
  *
- * @see MapperPlugin#getDynamicRuntimeFieldsBuilder()
  * @see Dynamic
  */
 public interface DynamicRuntimeFieldsBuilder {

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -49,7 +49,6 @@ class ParserContext {
             private final Function<String, SimilarityProvider> similarityLookupService;
             private final Function<String, TypeParser> typeParsers;
             private final Function<String, RuntimeFieldType.Parser> runtimeTypeParsers;
-            private final boolean supportsDynamicRuntimeMappings;
             private final Version indexVersionCreated;
             private final Supplier<SearchExecutionContext> searchExecutionContextSupplier;
             private final DateFormatter dateFormatter;
@@ -67,8 +66,7 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
                                  ScriptService scriptService,
                                  IndexAnalyzers indexAnalyzers,
                                  IndexSettings indexSettings,
-                                 BooleanSupplier idFieldDataEnabled,
-                                 boolean supportsDynamicRuntimeMappings) {
+                                 BooleanSupplier idFieldDataEnabled) {
                 this.similarityLookupService = similarityLookupService;
                 this.typeParsers = typeParsers;
                 this.runtimeTypeParsers = runtimeTypeParsers;
@@ -79,7 +77,6 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
                 this.indexAnalyzers = indexAnalyzers;
                 this.indexSettings = indexSettings;
                 this.idFieldDataEnabled = idFieldDataEnabled;
-                this.supportsDynamicRuntimeMappings = supportsDynamicRuntimeMappings;
             }
 
             public IndexAnalyzers getIndexAnalyzers() {
@@ -110,10 +107,6 @@ public RuntimeFieldType.Parser runtimeFieldTypeParser(String type) {
                 return runtimeTypeParsers.apply(type);
             }
 
-            public boolean supportsDynamicRuntimeMappings() {
-                return supportsDynamicRuntimeMappings;
-            }
-
             public Version indexVersionCreated() {
                 return indexVersionCreated;
             }
@@ -159,7 +152,7 @@ private static class MultiFieldParserContext extends ParserContext {
                 MultiFieldParserContext(ParserContext in) {
                     super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated,
                         in.searchExecutionContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings,
-                        in.idFieldDataEnabled, in.supportsDynamicRuntimeMappings);
+                        in.idFieldDataEnabled);
                 }
 
                 @Override
@@ -170,7 +163,7 @@ private static class DynamicTemplateParserContext extends ParserContext {
                 DynamicTemplateParserContext(ParserContext in) {
                     super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated,
                         in.searchExecutionContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings,
-                        in.idFieldDataEnabled, in.supportsDynamicRuntimeMappings);
+                        in.idFieldDataEnabled);
                 }
 
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -110,8 +110,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
         Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction =
             dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get,
                 mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, searchExecutionContextSupplier, dateFormatter,
-                scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled, mapperRegistry.getDynamicRuntimeFieldsBuilder() != null);
-        this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction, mapperRegistry.getDynamicRuntimeFieldsBuilder());
+                scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled);
+        this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction);
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers =
             mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated());
         this.parserContextSupplier = () -> parserContextFunction.apply(null);

server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

@@ -219,10 +219,6 @@ protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, O
                 if (value.equalsIgnoreCase("strict")) {
                     builder.dynamic(Dynamic.STRICT);
                 }  else if (value.equalsIgnoreCase("runtime")) {
-                    if (parserContext.supportsDynamicRuntimeMappings() == false) {
-                        throw new IllegalArgumentException("unable to set dynamic:runtime as there is " +
-                            "no registered dynamic runtime fields builder");
-                    }
                     builder.dynamic(Dynamic.RUNTIME);
                 } else {
                     boolean dynamic = XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".dynamic");

server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java

@@ -15,7 +15,6 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
-import org.elasticsearch.plugins.MapperPlugin;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -685,7 +684,6 @@ public final <T> T parseExternalValue(Class<T> clazz) {
 
     /**
      * Retrieve the builder for dynamically created runtime fields
-     * @see MapperPlugin#getDynamicRuntimeFieldsBuilder()
      */
     public abstract DynamicRuntimeFieldsBuilder getDynamicRuntimeFieldsBuilder();
 }