Pull the flattened field type into a mapper plugin. (#43250)

This PR pulls `FlatObjectFieldMapper` into its own `MapperPlugin`. To do so it
introduces a new interface `DynamicKeyFieldMapper` with the method
`keyedFieldType(String key)`, which gives the opportunity to return a special
field type for a subfield.

Author: jtibshirani

modules/mapper-flattened/build.gradle

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+    description 'Module for the flattened field type, which allows JSON objects to be flattened into a single field.'
+    classname 'org.elasticsearch.plugin.mapper.FlattenedMapperPlugin'
+}

server/src/main/java/org/elasticsearch/index/mapper/FlatObjectFieldMapper.java renamed to modules/mapper-flattened/src/main/java/org/elasticsearch/index/mapper/FlatObjectFieldMapper.java

@@ -88,7 +88,7 @@
  * "key\0some value" and "key2.key3\0true". Note that \0 is used as a reserved separator
  *  character (see {@link FlatObjectFieldParser#SEPARATOR}).
  */
-public final class FlatObjectFieldMapper extends FieldMapper {
+public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper {
 
     public static final String CONTENT_TYPE = "flattened";
     private static final String KEYED_FIELD_SUFFIX = "._keyed";
@@ -160,7 +160,7 @@ public Builder splitQueriesOnWhitespace(boolean splitQueriesOnWhitespace) {
         }
 
         @Override
-        public Builder addMultiField(Mapper.Builder mapperBuilder) {
+        public Builder addMultiField(Mapper.Builder<?, ?> mapperBuilder) {
             throw new UnsupportedOperationException("[fields] is not supported for [" + CONTENT_TYPE + "] fields.");
         }
 
@@ -189,7 +189,7 @@ public FlatObjectFieldMapper build(BuilderContext context) {
     public static class TypeParser implements Mapper.TypeParser {
         @Override
         public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
-            FlatObjectFieldMapper.Builder builder = new FlatObjectFieldMapper.Builder(name);
+            Builder builder = new Builder(name);
             parseField(builder, name, node, parserContext);
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
@@ -424,6 +424,11 @@ public OrdinalMap getOrdinalMap() {
                 + delegate.getFieldName() + "] does not allow access to the underlying ordinal map.");
         }
 
+        @Override
+        public boolean supportsGlobalOrdinalsMapping() {
+            return false;
+        }
+
         @Override
         public Index index() {
             return delegate.index();
@@ -534,7 +539,7 @@ private FlatObjectFieldMapper(String simpleName,
                                   int ignoreAbove,
                                   int depthLimit,
                                   Settings indexSettings) {
-        super(simpleName, fieldType, defaultFieldType, indexSettings, MultiFields.empty(), CopyTo.empty());
+        super(simpleName, fieldType, defaultFieldType, indexSettings, CopyTo.empty());
         assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0;
 
         this.depthLimit = depthLimit;
@@ -564,6 +569,7 @@ public RootFlatObjectFieldType fieldType() {
         return (RootFlatObjectFieldType) super.fieldType();
     }
 
+    @Override
     public KeyedFlatObjectFieldType keyedFieldType(String key) {
         return new KeyedFlatObjectFieldType(keyedFieldName(), key, fieldType());
     }

server/src/main/java/org/elasticsearch/index/mapper/FlatObjectFieldParser.java renamed to modules/mapper-flattened/src/main/java/org/elasticsearch/index/mapper/FlatObjectFieldParser.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugin.mapper;
+
+import org.elasticsearch.index.mapper.FlatObjectFieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+
+public class FlattenedMapperPlugin extends Plugin implements MapperPlugin, SearchPlugin {
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        return singletonMap(FlatObjectFieldMapper.CONTENT_TYPE, new FlatObjectFieldMapper.TypeParser());
+    }
+
+}

server/src/main/java/org/elasticsearch/index/mapper/KeyedFlatObjectAtomicFieldData.java renamed to modules/mapper-flattened/src/main/java/org/elasticsearch/index/mapper/KeyedFlatObjectAtomicFieldData.java

@@ -0,0 +1,200 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType;
+import org.elasticsearch.search.lookup.LeafDocLookup;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Function;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class FlatObjectFieldLookupTests extends ESTestCase {
+
+    public void testFieldTypeLookup() {
+        String fieldName = "object1.object2.field";
+        FlatObjectFieldMapper mapper = createFlatObjectMapper(fieldName);
+
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", singletonList(mapper), emptyList());
+        assertEquals(mapper.fieldType(), lookup.get(fieldName));
+
+        String objectKey = "key1.key2";
+        String searchFieldName = fieldName + "." + objectKey;
+
+        MappedFieldType searchFieldType = lookup.get(searchFieldName);
+        assertEquals(mapper.keyedFieldName(), searchFieldType.name());
+        assertThat(searchFieldType, instanceOf(KeyedFlatObjectFieldType.class));
+
+        FlatObjectFieldMapper.KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) searchFieldType;
+        assertEquals(objectKey, keyedFieldType.key());
+    }
+
+    public void testFieldTypeLookupWithAlias() {
+        String fieldName = "object1.object2.field";
+        FlatObjectFieldMapper mapper = createFlatObjectMapper(fieldName);
+
+        String aliasName = "alias";
+        FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, fieldName);
+
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", singletonList(mapper), singletonList(alias));
+        assertEquals(mapper.fieldType(), lookup.get(aliasName));
+
+        String objectKey = "key1.key2";
+        String searchFieldName = aliasName + "." + objectKey;
+
+        MappedFieldType searchFieldType = lookup.get(searchFieldName);
+        assertEquals(mapper.keyedFieldName(), searchFieldType.name());
+        assertThat(searchFieldType, instanceOf(KeyedFlatObjectFieldType.class));
+
+        KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) searchFieldType;
+        assertEquals(objectKey, keyedFieldType.key());
+    }
+
+    public void testFieldTypeLookupWithMultipleFields() {
+        String field1 = "object1.object2.field";
+        String field2 = "object1.field";
+        String field3 = "object2.field";
+
+        FlatObjectFieldMapper mapper1 = createFlatObjectMapper(field1);
+        FlatObjectFieldMapper mapper2 = createFlatObjectMapper(field2);
+        FlatObjectFieldMapper mapper3 = createFlatObjectMapper(field3);
+
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", Arrays.asList(mapper1, mapper2), emptyList());
+        assertNotNull(lookup.get(field1 + ".some.key"));
+        assertNotNull(lookup.get(field2 + ".some.key"));
+
+        lookup = lookup.copyAndAddAll("type", singletonList(mapper3), emptyList());
+        assertNotNull(lookup.get(field1 + ".some.key"));
+        assertNotNull(lookup.get(field2 + ".some.key"));
+        assertNotNull(lookup.get(field3 + ".some.key"));
+    }
+
+    public void testMaxDynamicKeyDepth() {
+        FieldTypeLookup lookup = new FieldTypeLookup();
+        assertEquals(0, lookup.maxKeyedLookupDepth());
+
+        // Add a flattened object field.
+        String flatObjectName = "object1.object2.field";
+        FlatObjectFieldMapper flatObjectField = createFlatObjectMapper(flatObjectName);
+        lookup = lookup.copyAndAddAll("type", singletonList(flatObjectField), emptyList());
+        assertEquals(3, lookup.maxKeyedLookupDepth());
+
+        // Add a short alias to that field.
+        String aliasName = "alias";
+        FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, flatObjectName);
+        lookup = lookup.copyAndAddAll("type", emptyList(), singletonList(alias));
+        assertEquals(3, lookup.maxKeyedLookupDepth());
+
+        // Add a longer alias to that field.
+        String longAliasName = "object1.object2.object3.alias";
+        FieldAliasMapper longAlias = new FieldAliasMapper(longAliasName, longAliasName, flatObjectName);
+        lookup = lookup.copyAndAddAll("type", emptyList(), singletonList(longAlias));
+        assertEquals(4, lookup.maxKeyedLookupDepth());
+
+        // Update the long alias to refer to a non-flattened object field.
+        String fieldName = "field";
+        MockFieldMapper field = new MockFieldMapper(fieldName);
+        longAlias = new FieldAliasMapper(longAliasName, longAliasName, fieldName);
+        lookup = lookup.copyAndAddAll("type", singletonList(field), singletonList(longAlias));
+        assertEquals(3, lookup.maxKeyedLookupDepth());
+    }
+
+    public void testFieldLookupIterator() {
+        MockFieldMapper mapper = new MockFieldMapper("foo");
+        FlatObjectFieldMapper flatObjectMapper = createFlatObjectMapper("object1.object2.field");
+
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", Arrays.asList(mapper, flatObjectMapper), emptyList());
+
+        Set<String> fieldNames = new HashSet<>();
+        for (MappedFieldType fieldType : lookup) {
+            fieldNames.add(fieldType.name());
+        }
+
+        assertThat(fieldNames, containsInAnyOrder(
+            mapper.name(), flatObjectMapper.name(), flatObjectMapper.keyedFieldName()));
+    }
+
+    private FlatObjectFieldMapper createFlatObjectMapper(String fieldName) {
+        Settings settings = Settings.builder()
+            .put("index.version.created", Version.CURRENT)
+            .build();
+        Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
+        return new FlatObjectFieldMapper.Builder(fieldName).build(context);
+    }
+
+    public void testScriptDocValuesLookup() {
+        MapperService mapperService = mock(MapperService.class);
+
+        ScriptDocValues<?> docValues1 = mock(ScriptDocValues.class);
+        IndexFieldData<?> fieldData1 = createFieldData(docValues1);
+
+        ScriptDocValues<?> docValues2 = mock(ScriptDocValues.class);
+        IndexFieldData<?> fieldData2 = createFieldData(docValues2);
+
+        KeyedFlatObjectFieldType fieldType1 = new KeyedFlatObjectFieldType("key1");
+        when(mapperService.fullName("json.key1")).thenReturn(fieldType1);
+
+        KeyedFlatObjectFieldType fieldType2 = new KeyedFlatObjectFieldType( "key2");
+        when(mapperService.fullName("json.key2")).thenReturn(fieldType2);
+
+        Function<MappedFieldType, IndexFieldData<?>> fieldDataSupplier = fieldType -> {
+            KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) fieldType;
+            return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2;
+        };
+
+        SearchLookup searchLookup = new SearchLookup(mapperService, fieldDataSupplier);
+        LeafDocLookup docLookup = searchLookup.doc().getLeafDocLookup(null);
+
+        assertEquals(docValues1, docLookup.get("json.key1"));
+        assertEquals(docValues2, docLookup.get("json.key2"));
+    }
+
+    private IndexFieldData<?> createFieldData(ScriptDocValues<?> scriptDocValues) {
+        AtomicFieldData atomicFieldData = mock(AtomicFieldData.class);
+        doReturn(scriptDocValues).when(atomicFieldData).getScriptValues();
+
+        IndexFieldData<?> fieldData = mock(IndexFieldData.class);
+        when(fieldData.getFieldName()).thenReturn("field");
+        doReturn(atomicFieldData).when(fieldData).load(anyObject());
+
+        return fieldData;
+    }
+}

modules/mapper-flattened/src/main/java/org/elasticsearch/plugin/mapper/FlattenedMapperPlugin.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType;
 import org.elasticsearch.index.mapper.FlatObjectFieldMapper.RootFlatObjectFieldType;
+import org.elasticsearch.plugin.mapper.FlattenedMapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -55,7 +56,7 @@ public void setup() {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class);
+        return pluginList(FlattenedMapperPlugin.class, InternalSettingsPlugin.class);
     }
 
     public void testDefaults() throws Exception {