RuntimeField to expose more than one MappedFieldType (#73900)

Up until now, there was a 1:1 relationship between a RuntimeField and a MappedFieldType. Actually, the two were only recently split to support emitting multiple fields from a single runtime script. The next step is to change the signature of asMappedFieldType to make it return multiple sub-fields. The leaf fields that are supported to-date will return a collection with a single item, but the upcoming runtime "object" field will return as many subfields as are listed in its definition.

Together with this, we are introducing two consistency checks:
- sub-fields exposed by a RuntimeField either have its same name, or belong to its namespace (e.g. object.subfield)
- there can't be two mapped field types with the same name as part of the same runtime section, overrides happen defining the same field in two different sections (index mappings and search request)

Author: javanna

server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java

@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.time.ZoneId;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -58,8 +59,8 @@ abstract class AbstractScriptFieldType<LeafFactory> extends MappedFieldType impl
     }
 
     @Override
-    public final MappedFieldType asMappedFieldType() {
-        return this;
+    public final Collection<MappedFieldType> asMappedFieldTypes() {
+        return Collections.singleton(this);
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -885,9 +885,11 @@ private static Mapper getLeafMapper(final ParseContext context,
         // if a leaf field is not mapped, and is defined as a runtime field, then we
         // don't create a dynamic mapping for it and don't index it.
         String fieldPath = context.path().pathAsText(fieldName);
-        RuntimeField runtimeField = context.root().getRuntimeField(fieldPath);
-        if (runtimeField != null) {
-            return new NoOpFieldMapper(subfields[subfields.length - 1], runtimeField.asMappedFieldType().name());
+        MappedFieldType fieldType = context.mappingLookup().getFieldType(fieldPath);
+        if (fieldType != null) {
+            //we haven't found a mapper with this name above, which means if a field type is found it is for sure a runtime field.
+            assert fieldType.hasDocValues() == false && fieldType.isAggregatable() && fieldType.isSearchable();
+            return new NoOpFieldMapper(subfields[subfields.length - 1], fieldType.name());
         }
         return null;
     }

server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java

@@ -80,10 +80,9 @@ final class FieldTypeLookup {
             }
         }
 
-        for (RuntimeField runtimeField : runtimeFields) {
-            MappedFieldType runtimeFieldType = runtimeField.asMappedFieldType();
+        for (MappedFieldType fieldType : RuntimeField.collectFieldTypes(runtimeFields).values()) {
             //this will override concrete fields with runtime fields that have the same name
-            fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType);
+            fullNameToFieldType.put(fieldType.name(), fieldType);
         }
     }
 

server/src/main/java/org/elasticsearch/index/mapper/RuntimeField.java

@@ -12,12 +12,14 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Definition of a runtime field that can be defined as part of the runtime section of the index mappings
@@ -51,10 +53,10 @@ default XContentBuilder toXContent(XContentBuilder builder, Params params) throw
     String typeName();
 
     /**
-     * Exposes the {@link MappedFieldType} backing this runtime field, used to execute queries, run aggs etc.
-     * @return the {@link MappedFieldType} backing this runtime field
+     * Exposes the {@link MappedFieldType}s backing this runtime field, used to execute queries, run aggs etc.
+     * @return the {@link MappedFieldType}s backing this runtime field
      */
-    MappedFieldType asMappedFieldType();
+    Collection<MappedFieldType> asMappedFieldTypes();
 
     /**
      *  For runtime fields the {@link RuntimeField.Parser} returns directly the {@link MappedFieldType}.
@@ -175,4 +177,31 @@ static Map<String, RuntimeField> parseRuntimeFields(Map<String, Object> node,
         }
         return Collections.unmodifiableMap(runtimeFields);
     }
+
+    /**
+     * Collect and return all {@link MappedFieldType} exposed by the provided {@link RuntimeField}s.
+     * Note that validation is performed to make sure that there are no name clashes among the collected runtime fields.
+     * This is because runtime fields with the same name are not accepted as part of the same section.
+     * @param runtimeFields the runtime to extract the mapped field types from
+     * @return the collected mapped field types
+     */
+    static Map<String, MappedFieldType> collectFieldTypes(Collection<RuntimeField> runtimeFields) {
+        return runtimeFields.stream()
+            .flatMap(runtimeField -> {
+                List<String> names = runtimeField.asMappedFieldTypes().stream().map(MappedFieldType::name)
+                    .filter(name -> name.equals(runtimeField.name()) == false
+                        && (name.startsWith(runtimeField.name() + ".") == false
+                        || name.length() > runtimeField.name().length() + 1 == false))
+                    .collect(Collectors.toList());
+                if (names.isEmpty() == false) {
+                    throw new IllegalStateException("Found sub-fields with name not belonging to the parent field they are part of "
+                        + names);
+                }
+                return runtimeField.asMappedFieldTypes().stream();
+            })
+            .collect(Collectors.toMap(MappedFieldType::name, mappedFieldType -> mappedFieldType,
+                (t, t2) -> {
+                    throw new IllegalArgumentException("Found two runtime fields with same name [" + t.name() + "]");
+                }));
+    }
 }

server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java

@@ -663,12 +663,7 @@ private static Map<String, MappedFieldType> parseRuntimeMappings(Map<String, Obj
         }
         Map<String, RuntimeField> runtimeFields = RuntimeField.parseRuntimeFields(new HashMap<>(runtimeMappings),
             mapperService.parserContext(), false);
-        Map<String, MappedFieldType> runtimeFieldTypes = new HashMap<>();
-        for (RuntimeField runtimeField : runtimeFields.values()) {
-            MappedFieldType fieldType = runtimeField.asMappedFieldType();
-            runtimeFieldTypes.put(fieldType.name(), fieldType);
-        }
-        return Collections.unmodifiableMap(runtimeFieldTypes);
+        return RuntimeField.collectFieldTypes(runtimeFields.values());
     }
 
     /**

server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java

@@ -22,6 +22,7 @@
 import static java.util.Collections.singletonList;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.hasSize;
@@ -55,6 +56,7 @@ public void testAddFieldAlias() {
     }
 
     public void testGetMatchingFieldNames() {
+        FlattenedFieldMapper flattened = createFlattenedMapper("flattened");
         MockFieldMapper field1 = new MockFieldMapper("foo");
         MockFieldMapper field2 = new MockFieldMapper("bar");
         MockFieldMapper field3 = new MockFieldMapper("baz");
@@ -63,18 +65,24 @@ public void testGetMatchingFieldNames() {
         FieldAliasMapper alias2 = new FieldAliasMapper("barometer", "barometer", "bar");
 
         TestRuntimeField runtimeField = new TestRuntimeField("baz", "type");
+        TestRuntimeField multi = new TestRuntimeField("flat", "multi",
+            List.of(new TestRuntimeField.TestRuntimeFieldType("flat.first", "first"),
+                new TestRuntimeField.TestRuntimeFieldType("flat.second", "second")));
 
-        FieldTypeLookup lookup = new FieldTypeLookup("_doc", List.of(field1, field2, field3), List.of(alias1, alias2),
-            List.of(runtimeField));
-
+        FieldTypeLookup lookup = new FieldTypeLookup("_doc", List.of(field1, field2, field3, flattened), List.of(alias1, alias2),
+            List.of(runtimeField, multi));
         {
             Collection<String> names = lookup.getMatchingFieldNames("*");
-            assertThat(names, containsInAnyOrder("foo", "food", "bar", "baz", "barometer"));
+            assertThat(names, containsInAnyOrder("foo", "food", "bar", "baz", "barometer", "flattened", "flat.first", "flat.second"));
         }
         {
             Collection<String> names = lookup.getMatchingFieldNames("b*");
             assertThat(names, containsInAnyOrder("bar", "baz", "barometer"));
         }
+        {
+            Collection<String> names = lookup.getMatchingFieldNames("fl*");
+            assertThat(names, containsInAnyOrder("flattened", "flat.first", "flat.second"));
+        }
         {
             Collection<String> names = lookup.getMatchingFieldNames("baro.any*");
             assertThat(names, hasSize(0));
@@ -83,6 +91,18 @@ public void testGetMatchingFieldNames() {
             Collection<String> names = lookup.getMatchingFieldNames("foo*");
             assertThat(names, containsInAnyOrder("foo", "food"));
         }
+        {
+            Collection<String> names = lookup.getMatchingFieldNames("flattened.anything");
+            assertThat(names, containsInAnyOrder("flattened.anything"));
+        }
+        {
+            Collection<String> names = lookup.getMatchingFieldNames("flat.first");
+            assertThat(names, containsInAnyOrder("flat.first"));
+        }
+        {
+            Collection<String> names = lookup.getMatchingFieldNames("flat.second");
+            assertThat(names, containsInAnyOrder("flat.second"));
+        }
     }
 
     public void testSourcePathWithMultiFields() {
@@ -123,28 +143,48 @@ public void testTypeLookup() {
 
     public void testRuntimeFieldsLookup() {
         MockFieldMapper concrete = new MockFieldMapper("concrete");
-        TestRuntimeField runtime = new TestRuntimeField("runtime", "type");
+        TestRuntimeField runtimeLong = new TestRuntimeField("multi.outside", "date");
+        TestRuntimeField runtime = new TestRuntimeField("string", "type");
+        TestRuntimeField multi = new TestRuntimeField("multi", "multi", List.of(
+            new TestRuntimeField.TestRuntimeFieldType("multi.string", "string"),
+            new TestRuntimeField.TestRuntimeFieldType("multi.long", "long")));
 
-        FieldTypeLookup fieldTypeLookup = new FieldTypeLookup("_doc", List.of(concrete), emptyList(), List.of(runtime));
+        FieldTypeLookup fieldTypeLookup = new FieldTypeLookup("_doc", List.of(concrete), emptyList(), List.of(runtime, runtimeLong, multi));
         assertThat(fieldTypeLookup.get("concrete"), instanceOf(MockFieldMapper.FakeFieldType.class));
-        assertThat(fieldTypeLookup.get("runtime"), instanceOf(TestRuntimeField.class));
+        assertThat(fieldTypeLookup.get("string"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
+        assertThat(fieldTypeLookup.get("string").typeName(), equalTo("type"));
+        assertThat(fieldTypeLookup.get("multi"), nullValue());
+        assertThat(fieldTypeLookup.get("multi.string"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
+        assertThat(fieldTypeLookup.get("multi.string").typeName(), equalTo("string"));
+        assertThat(fieldTypeLookup.get("multi.long"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
+        assertThat(fieldTypeLookup.get("multi.long").typeName(), equalTo("long"));
+        assertThat(fieldTypeLookup.get("multi.outside"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
+        assertThat(fieldTypeLookup.get("multi.outside").typeName(), equalTo("date"));
+        assertThat(fieldTypeLookup.get("multi.anything"), nullValue());
     }
 
     public void testRuntimeFieldsOverrideConcreteFields() {
         FlattenedFieldMapper flattened = createFlattenedMapper("flattened");
         MockFieldMapper field = new MockFieldMapper("field");
         MockFieldMapper subfield = new MockFieldMapper("object.subfield");
         MockFieldMapper concrete = new MockFieldMapper("concrete");
-        TestRuntimeField fieldOverride = new TestRuntimeField("field", "type");
-        TestRuntimeField subfieldOverride = new TestRuntimeField("object.subfield", "type");
+        TestRuntimeField fieldOverride = new TestRuntimeField("field", "string");
+        TestRuntimeField subfieldOverride = new TestRuntimeField("object", "multi",
+            Collections.singleton(new TestRuntimeField.TestRuntimeFieldType("object.subfield", "leaf")));
         TestRuntimeField runtime = new TestRuntimeField("runtime", "type");
+        TestRuntimeField flattenedRuntime = new TestRuntimeField("flattened.runtime", "type");
 
         FieldTypeLookup fieldTypeLookup = new FieldTypeLookup("_doc", List.of(field, concrete, subfield, flattened), emptyList(),
-            List.of(fieldOverride, runtime, subfieldOverride));
-        assertThat(fieldTypeLookup.get("field"), instanceOf(TestRuntimeField.class));
-        assertThat(fieldTypeLookup.get("object.subfield"), instanceOf(TestRuntimeField.class));
+            List.of(fieldOverride, runtime, subfieldOverride, flattenedRuntime));
+        assertThat(fieldTypeLookup.get("field"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
+        assertThat(fieldTypeLookup.get("field").typeName(), equalTo("string"));
+        assertThat(fieldTypeLookup.get("object.subfield"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
+        assertThat(fieldTypeLookup.get("object.subfield").typeName(), equalTo("leaf"));
         assertThat(fieldTypeLookup.get("concrete"), instanceOf(MockFieldMapper.FakeFieldType.class));
-        assertThat(fieldTypeLookup.get("runtime"), instanceOf(TestRuntimeField.class));
+        assertThat(fieldTypeLookup.get("runtime"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
+        assertThat(fieldTypeLookup.get("runtime").typeName(), equalTo("type"));
+        assertThat(fieldTypeLookup.get("flattened.anything"), instanceOf(FlattenedFieldMapper.KeyedFlattenedFieldType.class));
+        assertThat(fieldTypeLookup.get("flattened.runtime"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
     }
 
     public void testRuntimeFieldsSourcePaths() {
@@ -283,6 +323,54 @@ public void testMaxDynamicKeyDepth() {
         }
     }
 
+    public void testRuntimeFieldNameClashes() {
+        {
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new FieldTypeLookup(
+                "_doc", Collections.emptySet(), Collections.emptySet(),
+                List.of(new TestRuntimeField("field", "type"), new TestRuntimeField("field", "long"))));
+            assertEquals(iae.getMessage(), "Found two runtime fields with same name [field]");
+        }
+        {
+            TestRuntimeField multi = new TestRuntimeField("multi", "multi",
+                Collections.singleton(new TestRuntimeField.TestRuntimeFieldType("multi.first", "leaf")));
+            TestRuntimeField runtime = new TestRuntimeField("multi.first", "runtime");
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new FieldTypeLookup(
+                "_doc", Collections.emptySet(), Collections.emptySet(), List.of(multi, runtime)));
+            assertEquals(iae.getMessage(), "Found two runtime fields with same name [multi.first]");
+        }
+        {
+            TestRuntimeField multi = new TestRuntimeField("multi", "multi",
+                List.of(new TestRuntimeField.TestRuntimeFieldType("multi", "leaf"),
+                    new TestRuntimeField.TestRuntimeFieldType("multi", "leaf")));
+
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> new FieldTypeLookup(
+                "_doc", Collections.emptySet(), Collections.emptySet(), List.of(multi)));
+            assertEquals(iae.getMessage(), "Found two runtime fields with same name [multi]");
+        }
+    }
+
+    public void testRuntimeFieldNameOutsideContext() {
+        {
+            TestRuntimeField multi = new TestRuntimeField("multi", "multi",
+                List.of(new TestRuntimeField.TestRuntimeFieldType("first", "leaf"),
+                    new TestRuntimeField.TestRuntimeFieldType("second", "leaf"),
+                    new TestRuntimeField.TestRuntimeFieldType("multi.third", "leaf")));
+            IllegalStateException ise = expectThrows(IllegalStateException.class, () -> new FieldTypeLookup(
+                "_doc", Collections.emptySet(), Collections.emptySet(), Collections.singletonList(multi)));
+            assertEquals("Found sub-fields with name not belonging to the parent field they are part of [first, second]",
+                ise.getMessage());
+        }
+        {
+            TestRuntimeField multi = new TestRuntimeField("multi", "multi",
+                List.of(new TestRuntimeField.TestRuntimeFieldType("multi.", "leaf"),
+                    new TestRuntimeField.TestRuntimeFieldType("multi.f", "leaf")));
+            IllegalStateException ise = expectThrows(IllegalStateException.class, () -> new FieldTypeLookup(
+                "_doc", Collections.emptySet(), Collections.emptySet(), Collections.singletonList(multi)));
+            assertEquals("Found sub-fields with name not belonging to the parent field they are part of [multi.]",
+                ise.getMessage());
+        }
+    }
+
     private static FlattenedFieldMapper createFlattenedMapper(String fieldName) {
         return new FlattenedFieldMapper.Builder(fieldName).build(new ContentPath());
     }

server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java

@@ -48,7 +48,7 @@ public void testOnlyRuntimeField() {
         assertEquals(0, size(mappingLookup.fieldMappers()));
         assertEquals(0, mappingLookup.objectMappers().size());
         assertNull(mappingLookup.getMapper("test"));
-        assertThat(mappingLookup.fieldTypesLookup().get("test"), instanceOf(TestRuntimeField.class));
+        assertThat(mappingLookup.fieldTypesLookup().get("test"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
     }
 
     public void testRuntimeFieldLeafOverride() {
@@ -58,7 +58,7 @@ public void testRuntimeFieldLeafOverride() {
         assertThat(mappingLookup.getMapper("test"), instanceOf(MockFieldMapper.class));
         assertEquals(1, size(mappingLookup.fieldMappers()));
         assertEquals(0, mappingLookup.objectMappers().size());
-        assertThat(mappingLookup.fieldTypesLookup().get("test"), instanceOf(TestRuntimeField.class));
+        assertThat(mappingLookup.fieldTypesLookup().get("test"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
     }
 
     public void testSubfieldOverride() {
@@ -73,7 +73,7 @@ public void testSubfieldOverride() {
         assertThat(mappingLookup.getMapper("object.subfield"), instanceOf(MockFieldMapper.class));
         assertEquals(1, size(mappingLookup.fieldMappers()));
         assertEquals(1, mappingLookup.objectMappers().size());
-        assertThat(mappingLookup.fieldTypesLookup().get("object.subfield"), instanceOf(TestRuntimeField.class));
+        assertThat(mappingLookup.fieldTypesLookup().get("object.subfield"), instanceOf(TestRuntimeField.TestRuntimeFieldType.class));
     }
 
     public void testAnalyzers() throws IOException {