From dd5f648d77a15238def61e9aa53f19cd950d2224 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 10 Nov 2020 17:46:35 +0100 Subject: [PATCH 01/16] Add support for runtime section in the mappings The runtime section is at the same level as the existing properties section. Its purpose is to hold runtime fields only. With the introduction of the runtime section, a runtime field can be defined by specifying its type (previously called runtime_type) and script. Fields defined in the runtime section can be updated at any time as they are not present in the lucene index. They get replaced entirely when they get updated. Thanks to the introduction of the runtime section, runtime fields can override existing mapped fields defined with the same name, similarly to runtime fields defined in the search request. --- .../TransportGetFieldMappingsIndexAction.java | 4 +- .../index/mapper/FieldMapper.java | 22 +- .../index/mapper/FieldTypeLookup.java | 12 +- .../elasticsearch/index/mapper/Mapper.java | 10 +- .../index/mapper/MapperService.java | 4 +- .../index/mapper/MappingLookup.java | 18 +- .../index/mapper/RootObjectMapper.java | 96 +++- .../index/mapper/RuntimeFieldType.java | 60 +++ .../elasticsearch/indices/IndicesModule.java | 19 +- .../indices/mapper/MapperRegistry.java | 9 +- .../elasticsearch/plugins/MapperPlugin.java | 5 + .../MetadataRolloverServiceTests.java | 2 +- .../MetadataIndexUpgradeServiceTests.java | 2 +- .../index/IndexSortSettingsTests.java | 61 +++ .../elasticsearch/index/codec/CodecTests.java | 3 +- .../mapper/DocumentFieldMapperTests.java | 1 + .../FieldAliasMapperValidationTests.java | 24 +- .../index/mapper/FieldTypeLookupTests.java | 103 ++++- .../index/mapper/MappingLookupTests.java | 74 ++++ .../index/mapper/ParametrizedMapperTests.java | 2 +- .../index/mapper/RootObjectMapperTests.java | 287 +++++++++++- .../index/mapper/TestRuntimeField.java | 52 +++ .../index/mapper/TypeParsersTests.java | 8 +- .../index/query/QueryShardContextTests.java | 2 +- .../index/mapper/MockFieldMapper.java | 6 - .../aggregations/AggregatorTestCase.java | 2 +- .../mapper/FlattenedFieldLookupTests.java | 10 +- .../xpack/runtimefields/RuntimeFields.java | 36 +- .../mapper/AbstractScriptFieldType.java | 159 +++++-- .../mapper/BooleanScriptFieldType.java | 31 +- .../mapper/DateScriptFieldType.java | 75 +++- .../mapper/DoubleScriptFieldType.java | 31 +- .../mapper/GeoPointScriptFieldType.java | 30 +- .../mapper/IpScriptFieldType.java | 37 +- .../mapper/KeywordScriptFieldType.java | 29 +- .../mapper/LongScriptFieldType.java | 31 +- .../mapper/RuntimeFieldMapper.java | 290 ------------ ...bstractNonTextScriptFieldTypeTestCase.java | 18 +- .../AbstractScriptFieldTypeTestCase.java | 264 ++++++++++- .../mapper/BooleanScriptFieldTypeTests.java | 5 +- .../mapper/DateScriptFieldTypeTests.java | 43 +- .../mapper/DoubleScriptFieldTypeTests.java | 5 +- .../mapper/GeoPointScriptFieldTypeTests.java | 10 +- .../mapper/IpScriptFieldTypeTests.java | 9 +- .../mapper/KeywordScriptFieldTypeTests.java | 5 +- .../mapper/LongScriptFieldTypeTests.java | 5 +- .../mapper/RuntimeFieldMapperTests.java | 411 ------------------ .../test/runtime_fields/100_geo_point.yml | 27 +- .../test/runtime_fields/10_keyword.yml | 89 ++-- .../test/runtime_fields/20_long.yml | 55 ++- .../test/runtime_fields/30_double.yml | 37 +- .../test/runtime_fields/40_date.yml | 52 +-- .../test/runtime_fields/50_ip.yml | 27 +- .../test/runtime_fields/60_boolean.yml | 37 +- .../runtime_fields/80_multiple_indices.yml | 34 +- .../test/runtime_fields/90_loops.yml | 56 +-- 56 files changed, 1669 insertions(+), 1167 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java delete mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java delete mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapperTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 78301003508b5..ef100fbc6857a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -39,9 +39,9 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; @@ -155,6 +155,8 @@ private static Map findFieldMappings(Predicate fieldMappings = new HashMap<>(); final MappingLookup allFieldMappers = documentMapper.mappers(); for (String field : request.fields()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 2dcccf398ce77..ddf2ceab1a9dd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -278,15 +278,7 @@ public final FieldMapper merge(Mapper mergeWith) { Conflicts conflicts = new Conflicts(name()); builder.merge((FieldMapper) mergeWith, conflicts); conflicts.check(); - return builder.build(parentPath(name())); - } - - private static ContentPath parentPath(String name) { - int endPos = name.lastIndexOf("."); - if (endPos == -1) { - return new ContentPath(0); - } - return new ContentPath(name.substring(0, endPos)); + return builder.build(Builder.parentPath(name())); } protected void checkIncomingMergeType(FieldMapper mergeWith) { @@ -482,7 +474,7 @@ public List copyToFields() { /** * Serializes a parameter */ - protected interface Serializer { + public interface Serializer { void serialize(XContentBuilder builder, String name, T value) throws IOException; } @@ -931,7 +923,7 @@ protected String buildFullName(ContentPath contentPath) { /** * Writes the current builder parameter values as XContent */ - protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { + public final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { for (Parameter parameter : getParameters()) { parameter.toXContent(builder, includeDefaults); } @@ -1011,6 +1003,14 @@ public final void parse(String name, ParserContext parserContext, Map { + //TODO aliases could be added directly to fullNameToFieldType private final Map fullNameToFieldType = new HashMap<>(); private final Map aliasToConcreteName = new HashMap<>(); @@ -47,8 +49,10 @@ final class FieldTypeLookup implements Iterable { private final Map> fieldToCopiedFields = new HashMap<>(); private final DynamicKeyFieldTypeLookup dynamicKeyLookup; + //TODO ideally the constructor would not take mappers, but MappedFieldTypes and an external method does the conversion FieldTypeLookup(Collection fieldMappers, - Collection fieldAliasMappers) { + Collection fieldAliasMappers, + Collection runtimeFieldTypes) { Map dynamicKeyMappers = new HashMap<>(); for (FieldMapper fieldMapper : fieldMappers) { @@ -70,12 +74,18 @@ final class FieldTypeLookup implements Iterable { } } + //TODO field aliases should probably be defined as runtime fields? for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) { String aliasName = fieldAliasMapper.name(); String path = fieldAliasMapper.path(); aliasToConcreteName.put(aliasName, path); } + for (RuntimeFieldType runtimeFieldType : runtimeFieldTypes) { + //this will override concrete fields with runtime fields that have the same name + fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType); + } + this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index d7cfe22950ff9..2bae141fc9d12 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -59,6 +59,7 @@ class ParserContext { private final Function similarityLookupService; private final Function typeParsers; + private final Function runtimeTypeParsers; private final Version indexVersionCreated; private final Supplier queryShardContextSupplier; private final DateFormatter dateFormatter; @@ -69,6 +70,7 @@ class ParserContext { public ParserContext(Function similarityLookupService, Function typeParsers, + Function runtimeTypeParsers, Version indexVersionCreated, Supplier queryShardContextSupplier, DateFormatter dateFormatter, @@ -78,6 +80,7 @@ public ParserContext(Function similarityLookupServic BooleanSupplier idFieldDataEnabled) { this.similarityLookupService = similarityLookupService; this.typeParsers = typeParsers; + this.runtimeTypeParsers = runtimeTypeParsers; this.indexVersionCreated = indexVersionCreated; this.queryShardContextSupplier = queryShardContextSupplier; this.dateFormatter = dateFormatter; @@ -132,6 +135,8 @@ public DateFormatter getDateFormatter() { protected Function typeParsers() { return typeParsers; } + protected Function runtimeTypeParsers() { return runtimeTypeParsers; } + protected Function similarityLookupService() { return similarityLookupService; } /** @@ -147,8 +152,9 @@ public ParserContext createMultiFieldContext(ParserContext in) { static class MultiFieldParserContext extends ParserContext { MultiFieldParserContext(ParserContext in) { - super(in.similarityLookupService, in.typeParsers, in.indexVersionCreated, in.queryShardContextSupplier, - in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, in.idFieldDataEnabled); + super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated, + in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, + in.idFieldDataEnabled); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 766955f0e32a1..71b6abb737dbf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -125,8 +125,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, this.mapperRegistry = mapperRegistry; Function parserContextFunction = dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get, - indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService, indexAnalyzers, indexSettings, - idFieldDataEnabled); + mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter, + scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled); this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index c9b0ce0d3d7a2..cefe03374fc32 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.stream.Stream; +//TODO does this need to be iterable? It is not easy to track who's relying on that public final class MappingLookup implements Iterable { /** Full field name to mapper */ @@ -51,24 +52,24 @@ public static MappingLookup fromMapping(Mapping mapping) { newFieldMappers.add(metadataMapper); } } - collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers); - return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length); + for (Mapper child : mapping.root) { + collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers); + } + return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, + mapping.root.runtimeFieldTypes(), mapping.metadataMappers.length); } private static void collect(Mapper mapper, Collection objectMappers, Collection fieldMappers, Collection fieldAliasMappers) { - if (mapper instanceof RootObjectMapper) { - // root mapper isn't really an object mapper - } else if (mapper instanceof ObjectMapper) { + if (mapper instanceof ObjectMapper) { objectMappers.add((ObjectMapper)mapper); } else if (mapper instanceof FieldMapper) { fieldMappers.add((FieldMapper)mapper); } else if (mapper instanceof FieldAliasMapper) { fieldAliasMappers.add((FieldAliasMapper) mapper); } else { - throw new IllegalStateException("Unrecognized mapper type [" + - mapper.getClass().getSimpleName() + "]."); + throw new IllegalStateException("Unrecognized mapper type [" + mapper.getClass().getSimpleName() + "]."); } for (Mapper child : mapper) { @@ -79,6 +80,7 @@ private static void collect(Mapper mapper, Collection objectMapper public MappingLookup(Collection mappers, Collection objectMappers, Collection aliasMappers, + Collection runtimeFieldTypes, int metadataFieldCount) { Map fieldMappers = new HashMap<>(); Map indexAnalyzers = new HashMap<>(); @@ -115,7 +117,7 @@ public MappingLookup(Collection mappers, } } - this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers); + this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, runtimeFieldTypes); this.fieldMappers = Collections.unmodifiableMap(fieldMappers); this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index e2d4b197bb741..433829d4d1892 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; @@ -33,16 +34,19 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter; -public class RootObjectMapper extends ObjectMapper { +public final class RootObjectMapper extends ObjectMapper { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RootObjectMapper.class); public static class Defaults { @@ -61,6 +65,7 @@ public static class Builder extends ObjectMapper.Builder { protected Explicit dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false); protected Explicit dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false); protected Explicit numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false); + protected final Map runtimeFieldTypes = new HashMap<>(); public Builder(String name, Version indexCreatedVersion) { super(name, indexCreatedVersion); @@ -82,6 +87,11 @@ public RootObjectMapper.Builder add(Mapper.Builder builder) { return this; } + public RootObjectMapper.Builder addRuntime(RuntimeFieldType runtimeFieldType) { + this.runtimeFieldTypes.put(runtimeFieldType.name(), runtimeFieldType); + return this; + } + @Override public RootObjectMapper build(ContentPath contentPath) { return (RootObjectMapper) super.build(contentPath); @@ -91,7 +101,7 @@ public RootObjectMapper build(ContentPath contentPath) { protected ObjectMapper createMapper(String name, String fullPath, Explicit enabled, Nested nested, Dynamic dynamic, Map mappers, Version indexCreatedVersion) { assert !nested.isNested(); - return new RootObjectMapper(name, enabled, dynamic, mappers, + return new RootObjectMapper(name, enabled, dynamic, mappers, runtimeFieldTypes, dynamicDateTimeFormatters, dynamicTemplates, dateDetection, numericDetection, indexCreatedVersion); @@ -126,7 +136,7 @@ private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean pare } } - public static class TypeParser extends ObjectMapper.TypeParser { + static final class TypeParser extends ObjectMapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { @@ -145,8 +155,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext } @SuppressWarnings("unchecked") - protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, - ParserContext parserContext) { + private boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, ParserContext parserContext) { if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) { if (fieldNode instanceof List) { List formatters = new ArrayList<>(); @@ -189,10 +198,8 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam String templateName = entry.getKey(); Map templateParams = (Map) entry.getValue(); DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams); - if (template != null) { - validateDynamicTemplate(parserContext, template); - templates.add(template); - } + validateDynamicTemplate(parserContext, template); + templates.add(template); } builder.dynamicTemplates(templates); return true; @@ -202,6 +209,13 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam } else if (fieldName.equals("numeric_detection")) { builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true); return true; + } else if (fieldName.equals("runtime")) { + if (fieldNode instanceof Map) { + parseRuntime(builder, (Map) fieldNode, parserContext); + return true; + } else { + throw new ElasticsearchParseException("runtime must be a map type"); + } } return false; } @@ -211,11 +225,14 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam private Explicit dateDetection; private Explicit numericDetection; private Explicit dynamicTemplates; + private final Map runtimeFieldTypes; RootObjectMapper(String name, Explicit enabled, Dynamic dynamic, Map mappers, + Map runtimeFieldTypes, Explicit dynamicDateTimeFormatters, Explicit dynamicTemplates, Explicit dateDetection, Explicit numericDetection, Version indexCreatedVersion) { super(name, name, enabled, Nested.NO, dynamic, mappers, indexCreatedVersion); + this.runtimeFieldTypes = runtimeFieldTypes; this.dynamicTemplates = dynamicTemplates; this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; this.dateDetection = dateDetection; @@ -235,23 +252,26 @@ public ObjectMapper mappingUpdate(Mapper mapper) { return update; } - public boolean dateDetection() { + boolean dateDetection() { return this.dateDetection.value(); } - public boolean numericDetection() { + boolean numericDetection() { return this.numericDetection.value(); } - public DateFormatter[] dynamicDateTimeFormatters() { + DateFormatter[] dynamicDateTimeFormatters() { return dynamicDateTimeFormatters.value(); } - public DynamicTemplate[] dynamicTemplates() { + DynamicTemplate[] dynamicTemplates() { return dynamicTemplates.value(); } - @SuppressWarnings("rawtypes") + Collection runtimeFieldTypes() { + return runtimeFieldTypes.values(); + } + public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) { return findTemplateBuilder(context, name, matchType, null); } @@ -267,7 +287,6 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Dat * @param dateFormat a dateformatter to use if the type is a date, null if not a date or is using the default format * @return a mapper builder, or null if there is no template for such a field */ - @SuppressWarnings("rawtypes") private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) { DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); if (dynamicTemplate == null) { @@ -330,6 +349,8 @@ protected void doMerge(ObjectMapper mergeWith, MergeReason reason) { this.dynamicTemplates = mergeWithObject.dynamicTemplates; } } + + this.runtimeFieldTypes.putAll(mergeWithObject.runtimeFieldTypes); } @Override @@ -360,6 +381,16 @@ protected void doXContent(XContentBuilder builder, ToXContent.Params params) thr if (numericDetection.explicit() || includeDefaults) { builder.field("numeric_detection", numericDetection.value()); } + + if (runtimeFieldTypes.size() > 0) { + builder.startObject("runtime"); + List sortedRuntimeFieldTypes = runtimeFieldTypes.values().stream().sorted( + Comparator.comparing(RuntimeFieldType::name)).collect(Collectors.toList()); + for (RuntimeFieldType fieldType : sortedRuntimeFieldTypes) { + fieldType.toXContent(builder, params); + } + builder.endObject(); + } } private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext parserContext, @@ -470,4 +501,39 @@ private static boolean containsSnippet(List list, String snippet) { } return false; } + + private static void parseRuntime(RootObjectMapper.Builder builder, + Map node, + Mapper.TypeParser.ParserContext parserContext) { + Iterator> iterator = node.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + if (entry.getValue() instanceof Map) { + @SuppressWarnings("unchecked") + Map propNode = (Map) entry.getValue(); + Object typeNode = propNode.get("type"); + String type; + if (typeNode == null) { + throw new MapperParsingException("No type specified for runtime field [" + fieldName + "]"); + } else { + type = typeNode.toString(); + } + RuntimeFieldType.Parser typeParser = parserContext.runtimeTypeParsers().apply(type); + if (typeParser == null) { + throw new MapperParsingException("No handler for type [" + type + + "] declared on runtime field [" + fieldName + "]"); + } + builder.addRuntime(typeParser.parse(fieldName, propNode, parserContext)); + propNode.remove("type"); + DocumentMapperParser.checkNoRemainingFields(fieldName, propNode, parserContext.indexVersionCreated()); + iterator.remove(); + } else { + throw new MapperParsingException("Expected map for runtime field [" + fieldName + "] definition but got a " + + fieldName.getClass().getName()); + } + } + DocumentMapperParser.checkNoRemainingFields(node, parserContext.indexVersionCreated(), + "DocType runtime definition has unsupported parameters: "); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java new file mode 100644 index 0000000000000..d0dc537b4f509 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java @@ -0,0 +1,60 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +/** + * Base implementation for a runtime field that can be defined as part of the runtime section of the index mappings + */ +public abstract class RuntimeFieldType extends MappedFieldType implements ToXContentFragment { + + protected RuntimeFieldType(String name, Map meta) { + super(name, false, false, false, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); + } + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(name()); + builder.field("type", typeName()); + boolean includeDefaults = params.paramAsBoolean("include_defaults", false); + doXContentBody(builder, includeDefaults); + builder.endObject(); + return builder; + } + + /** + * Prints out the parameters that subclasses expose + */ + protected abstract void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException; + + /** + * Parser for a runtime field. Creates the appropriate {@link RuntimeFieldType} for a runtime field, + * as defined in the runtime section of the index mappings. + */ + public interface Parser { + RuntimeFieldType parse(String name, Map node, Mapper.TypeParser.ParserContext parserContext) + throws MapperParsingException; + } +} diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 2ba32df753ea1..ca01bddaaccf8 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -77,8 +78,8 @@ public class IndicesModule extends AbstractModule { private final MapperRegistry mapperRegistry; public IndicesModule(List mapperPlugins) { - this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getMetadataMappers(mapperPlugins), - getFieldFilter(mapperPlugins)); + this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getRuntimeFieldTypes(mapperPlugins), + getMetadataMappers(mapperPlugins), getFieldFilter(mapperPlugins)); } public static List getNamedWriteables() { @@ -134,9 +135,21 @@ public static Map getMappers(List mappe return Collections.unmodifiableMap(mappers); } + private static Map getRuntimeFieldTypes(List mapperPlugins) { + Map runtimeParsers = new LinkedHashMap<>(); + for (MapperPlugin mapperPlugin : mapperPlugins) { + for (Map.Entry entry : mapperPlugin.getRuntimeFieldTypes().entrySet()) { + if (runtimeParsers.put(entry.getKey(), entry.getValue()) != null) { + throw new IllegalArgumentException("Runtime field type [" + entry.getKey() + "] is already registered"); + } + } + } + return Collections.unmodifiableMap(runtimeParsers); + } + private static final Map builtInMetadataMappers = initBuiltInMetadataMappers(); - private static Set builtInMetadataFields = Collections.unmodifiableSet(builtInMetadataMappers.keySet()); + private static final Set builtInMetadataFields = Collections.unmodifiableSet(builtInMetadataMappers.keySet()); private static Map initBuiltInMetadataMappers() { Map builtInMetadataMappers; diff --git a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java index e5c1132267624..8e95ac9851cd7 100644 --- a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java +++ b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.NestedPathFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.plugins.MapperPlugin; import java.util.Collections; @@ -37,14 +38,16 @@ public final class MapperRegistry { private final Map mapperParsers; + private final Map runtimeFieldTypeParsers; private final Map metadataMapperParsers; private final Map metadataMapperParsers7x; private final Function> fieldFilter; - public MapperRegistry(Map mapperParsers, + public MapperRegistry(Map mapperParsers, Map runtimeFieldTypeParsers, Map metadataMapperParsers, Function> fieldFilter) { this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers)); + this.runtimeFieldTypeParsers = runtimeFieldTypeParsers; this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers)); Map metadata7x = new LinkedHashMap<>(metadataMapperParsers); metadata7x.remove(NestedPathFieldMapper.NAME); @@ -60,6 +63,10 @@ public Map getMapperParsers() { return mapperParsers; } + public Map getRuntimeFieldTypeParsers() { + return runtimeFieldTypeParsers; + } + /** * Return a map of the meta mappers that have been registered. The * returned map uses the name of the field as a key. diff --git a/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java b/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java index 5edf994b32ea4..8544c75a575ee 100644 --- a/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import java.util.Collections; import java.util.Map; @@ -43,6 +44,10 @@ default Map getMappers() { return Collections.emptyMap(); } + default Map getRuntimeFieldTypes() { + return Collections.emptyMap(); + } + /** * Returns additional metadata mapper implementations added by this plugin. *

diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index b56b179305d4c..2d065e3cc0b5b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -561,7 +561,7 @@ protected String contentType() { } }; MappingLookup mappingLookup = - new MappingLookup(List.of(mockedTimestampField, dateFieldMapper), List.of(), List.of(), 0); + new MappingLookup(List.of(mockedTimestampField, dateFieldMapper), List.of(), List.of(), List.of(), 0); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); Environment env = mock(Environment.class); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java index 227436f56ec9c..590285fa0ca88 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java @@ -159,7 +159,7 @@ private MetadataIndexUpgradeService getMetadataIndexUpgradeService() { return new MetadataIndexUpgradeService( Settings.EMPTY, xContentRegistry(), - new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), + new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new SystemIndices(Map.of("system-plugin", List.of(new SystemIndexDescriptor(".system", "a system index")))), null diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index ffc08f9e1d689..553349428f90a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -19,15 +19,30 @@ package org.elasticsearch.index; +import org.apache.lucene.search.Query; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.RuntimeFieldType; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; +import java.util.Collections; +import java.util.function.Supplier; + import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class IndexSortSettingsTests extends ESTestCase { private static IndexSettings indexSettings(Settings settings) { @@ -129,4 +144,50 @@ public void testInvalidMissing() { assertThat(exc.getMessage(), containsString("Illegal missing value:[default]," + " must be one of [_last, _first]")); } + + public void testIndexSorting() { + IndexSettings indexSettings = indexSettings(Settings.builder().put("index.sort.field", "field").build()); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null); + NoneCircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); + final IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, cache, circuitBreakerService, null); + MappedFieldType fieldType = new RuntimeFieldType("field", Collections.emptyMap()) { + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + + @Override + public String typeName() { + return null; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + searchLookup.get(); + return null; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new UnsupportedOperationException(); + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) { + throw new UnsupportedOperationException(); + } + }; + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> config.buildIndexSort( + field -> fieldType, + (ft, searchLookupSupplier) -> indexFieldDataService.getForField(ft, "index", searchLookupSupplier) + ) + ); + assertEquals("docvalues not found for index sort field:[field]", iae.getMessage()); + assertThat(iae.getCause(), instanceOf(UnsupportedOperationException.class)); + assertEquals("index sorting not supported on runtime field [field]", iae.getCause().getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index aa86888cdefe9..f01bcc169b191 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -92,7 +92,8 @@ private CodecService createCodecService() throws IOException { IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); SimilarityService similarityService = new SimilarityService(settings, null, Collections.emptyMap()); IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; - MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); + MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), + MapperPlugin.NOOP_FIELD_FILTER); MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry, () -> null, () -> false, null); return new CodecService(service, LogManager.getLogger("test")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 829550a5439e2..2b386472f8816 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -119,6 +119,7 @@ public void testAnalyzers() throws IOException { Arrays.asList(fieldMapper1, fieldMapper2), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), 0); assertAnalyzes(mappingLookup.indexAnalyzer(), "field1", "index1"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index 89ac4d29beb78..475d9b252fbad 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -40,7 +40,9 @@ public void testDuplicateFieldAliasAndObject() { new MappingLookup( Collections.emptyList(), singletonList(objectMapper), - singletonList(aliasMapper), 0)); + singletonList(aliasMapper), + emptyList(), + 0)); assertEquals("Alias [some.path] is defined both as an object and an alias", e.getMessage()); } @@ -53,7 +55,9 @@ public void testDuplicateFieldAliasAndConcreteField() { new MappingLookup( Arrays.asList(field, invalidField), emptyList(), - singletonList(invalidAlias), 0)); + singletonList(invalidAlias), + emptyList(), + 0)); assertEquals("Alias [invalid] is defined both as an alias and a concrete field", e.getMessage()); } @@ -66,7 +70,9 @@ public void testAliasThatRefersToAlias() { MappingLookup mappers = new MappingLookup( singletonList(field), emptyList(), - Arrays.asList(alias, invalidAlias), 0); + Arrays.asList(alias, invalidAlias), + emptyList(), + 0); alias.validate(mappers); MapperParsingException e = expectThrows(MapperParsingException.class, () -> { @@ -84,7 +90,9 @@ public void testAliasThatRefersToItself() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0); + singletonList(invalidAlias), + emptyList(), + 0); invalidAlias.validate(mappers); }); @@ -99,7 +107,9 @@ public void testAliasWithNonExistentPath() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0); + singletonList(invalidAlias), + emptyList(), + 0); invalidAlias.validate(mappers); }); @@ -115,6 +125,7 @@ public void testFieldAliasWithNestedScope() { singletonList(createFieldMapper("nested", "field")), singletonList(objectMapper), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); } @@ -127,6 +138,7 @@ public void testFieldAliasWithDifferentObjectScopes() { List.of(createFieldMapper("object1", "field")), List.of(createObjectMapper("object1"), createObjectMapper("object2")), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); } @@ -140,6 +152,7 @@ public void testFieldAliasWithNestedTarget() { singletonList(createFieldMapper("nested", "field")), Collections.singletonList(objectMapper), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); }); @@ -158,6 +171,7 @@ public void testFieldAliasWithDifferentNestedScopes() { singletonList(createFieldMapper("nested1", "field")), List.of(createNestedObjectMapper("nested1"), createNestedObjectMapper("nested2")), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index ca5f330815d55..961b77a2ba0f7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -25,15 +25,17 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.List; import java.util.Set; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.hamcrest.CoreMatchers.instanceOf; public class FieldTypeLookupTests extends ESTestCase { public void testEmpty() { - FieldTypeLookup lookup = new FieldTypeLookup(Collections.emptyList(), Collections.emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); assertNull(lookup.get("foo")); Collection names = lookup.simpleMatchToFullName("foo"); assertNotNull(names); @@ -45,7 +47,7 @@ public void testEmpty() { public void testAddNewField() { MockFieldMapper f = new MockFieldMapper("foo"); - FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f), emptyList(), Collections.emptyList()); assertNull(lookup.get("bar")); assertEquals(f.fieldType(), lookup.get("foo")); assertEquals(1, size(lookup.iterator())); @@ -55,7 +57,8 @@ public void testAddFieldAlias() { MockFieldMapper field = new MockFieldMapper("foo"); FieldAliasMapper alias = new FieldAliasMapper("alias", "alias", "foo"); - FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(field), Collections.singletonList(alias)); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(field), Collections.singletonList(alias), + Collections.emptyList()); MappedFieldType aliasType = lookup.get("alias"); assertEquals(field.fieldType(), aliasType); @@ -68,7 +71,7 @@ public void testSimpleMatchToFullName() { FieldAliasMapper alias1 = new FieldAliasMapper("food", "food", "foo"); FieldAliasMapper alias2 = new FieldAliasMapper("barometer", "barometer", "bar"); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field1, field2), Arrays.asList(alias1, alias2)); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field1, field2), Arrays.asList(alias1, alias2), Collections.emptyList()); Collection names = lookup.simpleMatchToFullName("b*"); @@ -85,7 +88,7 @@ public void testSourcePathWithMultiFields() { .addMultiField(new MockFieldMapper.Builder("field.subfield2")) .build(new ContentPath()); - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(field), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(field), emptyList(), emptyList()); assertEquals(Set.of("field"), lookup.sourcePaths("field")); assertEquals(Set.of("field"), lookup.sourcePaths("field.subfield1")); @@ -101,7 +104,7 @@ public void testSourcePathsWithCopyTo() { .copyTo("field") .build(new ContentPath()); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field, otherField), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field, otherField), emptyList(), emptyList()); assertEquals(Set.of("other_field", "field"), lookup.sourcePaths("field")); assertEquals(Set.of("other_field", "field"), lookup.sourcePaths("field.subfield1")); @@ -109,20 +112,86 @@ public void testSourcePathsWithCopyTo() { public void testIteratorImmutable() { MockFieldMapper f1 = new MockFieldMapper("foo"); - FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f1), emptyList()); - - try { - Iterator itr = lookup.iterator(); - assertTrue(itr.hasNext()); - assertEquals(f1.fieldType(), itr.next()); - itr.remove(); - fail("remove should have failed"); - } catch (UnsupportedOperationException e) { - // expected + FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f1), emptyList(), emptyList()); + + Iterator itr = lookup.iterator(); + assertTrue(itr.hasNext()); + assertEquals(f1.fieldType(), itr.next()); + expectThrows(UnsupportedOperationException.class, itr::remove); + } + + public void testRuntimeFieldsLookup() { + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField runtime = new TestRuntimeField("runtime"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(concrete), emptyList(), List.of(runtime)); + assertThat(fieldTypeLookup.get("concrete"), instanceOf(MockFieldMapper.FakeFieldType.class)); + assertThat(fieldTypeLookup.get("runtime"), instanceOf(TestRuntimeField.class)); + assertEquals(2, size(fieldTypeLookup.iterator())); + } + + public void testRuntimeFieldOverrides() { + MockFieldMapper field = new MockFieldMapper("field"); + MockFieldMapper subfield = new MockFieldMapper("object.subfield"); + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField fieldOverride = new TestRuntimeField("field"); + TestRuntimeField subfieldOverride = new TestRuntimeField("object.subfield"); + TestRuntimeField runtime = new TestRuntimeField("runtime"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(field, concrete, subfield), emptyList(), + List.of(fieldOverride, runtime, subfieldOverride)); + assertThat(fieldTypeLookup.get("field"), instanceOf(TestRuntimeField.class)); + assertThat(fieldTypeLookup.get("object.subfield"), instanceOf(TestRuntimeField.class)); + assertThat(fieldTypeLookup.get("concrete"), instanceOf(MockFieldMapper.FakeFieldType.class)); + assertThat(fieldTypeLookup.get("runtime"), instanceOf(TestRuntimeField.class)); + assertEquals(4, size(fieldTypeLookup.iterator())); + } + + public void testRuntimeFieldsSimpleMatchToFullName() { + MockFieldMapper field1 = new MockFieldMapper("field1"); + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField field2 = new TestRuntimeField("field2"); + TestRuntimeField subfield = new TestRuntimeField("object.subfield"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(field1, concrete), emptyList(), List.of(field2, subfield)); + { + Set matches = fieldTypeLookup.simpleMatchToFullName("fie*"); + assertEquals(2, matches.size()); + assertTrue(matches.contains("field1")); + assertTrue(matches.contains("field2")); + } + { + Set matches = fieldTypeLookup.simpleMatchToFullName("object.sub*"); + assertEquals(1, matches.size()); + assertTrue(matches.contains("object.subfield")); + } + } + + public void testRuntimeFieldsSourcePaths() { + MockFieldMapper field1 = new MockFieldMapper("field1"); + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField field2 = new TestRuntimeField("field2"); + TestRuntimeField subfield = new TestRuntimeField("object.subfield"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(field1, concrete), emptyList(), List.of(field2, subfield)); + { + Set sourcePaths = fieldTypeLookup.sourcePaths("field1"); + assertEquals(1, sourcePaths.size()); + assertTrue(sourcePaths.contains("field1")); + } + { + Set sourcePaths = fieldTypeLookup.sourcePaths("field2"); + assertEquals(1, sourcePaths.size()); + assertTrue(sourcePaths.contains("field2")); + } + { + Set sourcePaths = fieldTypeLookup.sourcePaths("object.subfield"); + assertEquals(1, sourcePaths.size()); + assertTrue(sourcePaths.contains("object.subfield")); } } - private int size(Iterator iterator) { + private static int size(Iterator iterator) { if (iterator == null) { throw new NullPointerException("iterator"); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java new file mode 100644 index 0000000000000..4ce8cc4892fa5 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.Iterator; + +import static org.hamcrest.CoreMatchers.instanceOf; + +public class MappingLookupTests extends ESTestCase { + + public void testOnlyRuntimeField() { + MappingLookup mappingLookup = new MappingLookup(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.singletonList(new TestRuntimeField("test")), 0); + assertEquals(0, size(mappingLookup.iterator())); + assertEquals(0, mappingLookup.objectMappers().size()); + assertNull(mappingLookup.getMapper("test")); + assertThat(mappingLookup.fieldTypes().get("test"), instanceOf(TestRuntimeField.class)); + } + + public void testRuntimeFieldLeafOverride() { + MockFieldMapper fieldMapper = new MockFieldMapper("test"); + MappingLookup mappingLookup = new MappingLookup(Collections.singletonList(fieldMapper), Collections.emptyList(), + Collections.emptyList(), Collections.singletonList(new TestRuntimeField("test")), 0); + assertThat(mappingLookup.getMapper("test"), instanceOf(MockFieldMapper.class)); + assertEquals(1, size(mappingLookup.iterator())); + assertEquals(0, mappingLookup.objectMappers().size()); + assertThat(mappingLookup.fieldTypes().get("test"), instanceOf(TestRuntimeField.class)); + assertEquals(1, size(mappingLookup.fieldTypes().iterator())); + } + + public void testSubfieldOverride() { + MockFieldMapper fieldMapper = new MockFieldMapper("object.subfield"); + ObjectMapper objectMapper = new ObjectMapper("object", "object", new Explicit<>(true, true), ObjectMapper.Nested.NO, + ObjectMapper.Dynamic.TRUE, Collections.singletonMap("object.subfield", fieldMapper), Version.CURRENT); + MappingLookup mappingLookup = new MappingLookup(Collections.singletonList(fieldMapper), Collections.singletonList(objectMapper), + Collections.emptyList(), Collections.singletonList(new TestRuntimeField("object.subfield")), 0); + assertThat(mappingLookup.getMapper("object.subfield"), instanceOf(MockFieldMapper.class)); + assertEquals(1, size(mappingLookup.iterator())); + assertEquals(1, mappingLookup.objectMappers().size()); + assertThat(mappingLookup.fieldTypes().get("object.subfield"), instanceOf(TestRuntimeField.class)); + assertEquals(1, size(mappingLookup.fieldTypes().iterator())); + } + + private static int size(Iterator iterator) { + int count = 0; + while (iterator.hasNext()) { + iterator.next(); + count++; + } + return count; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 0c41711bdc064..21989bde8f4cb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -212,7 +212,7 @@ private static TestMapper fromMapping(String mapping, Version version) { return BinaryFieldMapper.PARSER; } return null; - }, version, () -> null, null, null, + }, name -> null, version, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index 3e52680c74679..f0d7f903d8fc4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -19,25 +19,68 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -public class RootObjectMapperTests extends ESSingleNodeTestCase { +public class RootObjectMapperTests extends ESTestCase { + + private static MapperService createMapperService() { + return createMapperService(Version.CURRENT); + } + + private static MapperService createMapperService(Version indexCreatedVersion) { + return createMapperService(indexCreatedVersion, Collections.emptyList()); + } + + private static MapperService createMapperService(List mapperPlugins) { + return createMapperService(Version.CURRENT, mapperPlugins); + } + + private static MapperService createMapperService(Version indexCreatedVersion, List mapperPlugins) { + Settings settings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0) + .put("index.version.created", indexCreatedVersion).build(); + IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + IndexAnalyzers indexAnalyzers = new IndexAnalyzers( + Map.of("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())), Map.of(), Map.of()); + IndicesModule indicesModule = new IndicesModule(mapperPlugins); + SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); + return new MapperService(indexSettings, indexAnalyzers, NamedXContentRegistry.EMPTY, similarityService, + indicesModule.getMapperRegistry(), () -> { + throw new UnsupportedOperationException(); + }, () -> true, null); + } public void testNumericDetection() throws Exception { MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); @@ -47,7 +90,7 @@ public void testNumericDetection() throws Exception { .field("numeric_detection", false) .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); assertEquals(mapping, mapper.mappingSource().toString()); @@ -79,7 +122,7 @@ public void testDateDetection() throws Exception { .field("date_detection", true) .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); assertEquals(mapping, mapper.mappingSource().toString()); @@ -111,7 +154,7 @@ public void testDateFormatters() throws Exception { .field("dynamic_date_formats", Arrays.asList("yyyy-MM-dd")) .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); assertEquals(mapping, mapper.mappingSource().toString()); @@ -150,7 +193,7 @@ public void testDynamicTemplates() throws Exception { .endArray() .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); assertEquals(mapping, mapper.mappingSource().toString()); @@ -166,7 +209,7 @@ public void testDynamicTemplates() throws Exception { String mapping3 = Strings.toString(XContentFactory.jsonBuilder() .startObject() .startObject("type") - .field("dynamic_templates", Arrays.asList()) + .field("dynamic_templates", Collections.emptyList()) .endObject() .endObject()); mapper = mapperService.merge("type", new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); @@ -194,7 +237,7 @@ public void testDynamicTemplatesForIndexTemplate() throws IOException { .endObject() .endArray() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent(mapping), MergeReason.INDEX_TEMPLATE); // There should be no update if templates are not set. @@ -266,7 +309,7 @@ public void testIllegalFormatField() throws Exception { .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); for (String m : Arrays.asList(mapping, dynamicMapping)) { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> mapperService.parse("type", new CompressedXContent(m))); @@ -283,7 +326,7 @@ public void testIllegalDynamicTemplates() throws Exception { .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapperService.parse("type", new CompressedXContent(mapping))); assertEquals("Dynamic template syntax error. An array of named objects is expected.", e.getMessage()); @@ -309,7 +352,7 @@ public void testIllegalDynamicTemplateUnknownFieldType() throws Exception { mapping.endObject(); } mapping.endObject(); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE)); assertThat(e.getRootCause(), instanceOf(IllegalArgumentException.class)); @@ -337,7 +380,7 @@ public void testIllegalDynamicTemplateUnknownAttribute() throws Exception { mapping.endObject(); } mapping.endObject(); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE)); assertThat(e.getRootCause(), instanceOf(MapperParsingException.class)); @@ -366,7 +409,7 @@ public void testIllegalDynamicTemplateInvalidAttribute() throws Exception { mapping.endObject(); } mapping.endObject(); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE)); assertThat(e.getRootCause(), instanceOf(IllegalArgumentException.class)); @@ -401,7 +444,7 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { mapping.endObject(); } mapping.endObject(); - mapperService = createIndex("test").mapperService(); + mapperService = createMapperService(); DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); assertThat(mapper.mappingSource().toString(), containsString("\"index_phrases\":true")); @@ -439,11 +482,6 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { } } - @Override - protected boolean forbidPrivateIndexSettings() { - return false; - } - public void testIllegalDynamicTemplate7DotXIndex() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); @@ -465,13 +503,216 @@ public void testIllegalDynamicTemplate7DotXIndex() throws Exception { } mapping.endObject(); Version createdVersion = randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_7_0); - Settings indexSettings = Settings.builder() - .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), createdVersion) - .build(); - MapperService mapperService = createIndex("test", indexSettings).mapperService(); + MapperService mapperService = createMapperService(createdVersion); DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); assertThat(mapper.mappingSource().toString(), containsString("\"type\":\"string\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{\"type\":" + "\"string\"}}], caused by [No mapper found for type [string]]"); } + + public void testRuntimeSection() throws IOException { + MapperService mapperService = createMapperService(Collections.singletonList(new RuntimeFieldPlugin())); + MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("runtime") + .startObject("field1") + .field("type", "test") + .field("prop1", "value1") + .endObject() + .startObject("field2") + .field("type", "test") + .field("prop2", "value2") + .endObject() + .startObject("field3") + .field("type", "test") + .endObject() + .endObject() + .endObject() + .endObject()); + DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); + assertEquals(mapping, mapper.mappingSource().toString()); + } + + public void testRuntimeSectionMerge() throws IOException { + MapperService mapperService = createMapperService(Collections.singletonList(new RuntimeFieldPlugin())); + MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); + { + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("runtime") + .startObject("field1") + .field("type", "test") + .field("prop1", "first version") + .endObject() + .startObject("field2") + .field("type", "test") + .endObject() + .endObject() + .endObject() + .endObject()); + DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); + assertEquals(mapping, mapper.mappingSource().toString()); + } + { + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("runtime") + .startObject("field1") + .field("type", "test") + .field("prop2", "second version") + .endObject() + .endObject() + .endObject() + .endObject()); + mapperService.merge("type", new CompressedXContent(mapping), reason); + RuntimeField field1 = (RuntimeField)mapperService.fieldType("field1"); + assertNull(field1.prop1); + assertEquals("second version", field1.prop2); + RuntimeField field2 = (RuntimeField)mapperService.fieldType("field2"); + assertNull(field2.prop1); + assertNull(field2.prop2); + } + } + + public void testRuntimeSectionNonRuntimeType() throws IOException { + MapperService mapperService = createMapperService(); + MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("runtime") + .startObject("field") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .endObject()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> mapperService.merge("type", new CompressedXContent(mapping), reason)); + assertEquals("Failed to parse mapping: No handler for type [keyword] declared on runtime field [field]", e.getMessage()); + } + + public void testRuntimeSectionHandlerNotFound() throws IOException { + MapperService mapperService = createMapperService(Collections.singletonList(new RuntimeFieldPlugin())); + MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("runtime") + .startObject("field") + .field("type", "unknown") + .endObject() + .endObject() + .endObject() + .endObject()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> mapperService.merge("type", new CompressedXContent(mapping), reason)); + assertEquals("Failed to parse mapping: No handler for type [unknown] declared on runtime field [field]", e.getMessage()); + } + + public void testRuntimeSectionMissingType() throws IOException { + MapperService mapperService = createMapperService(Collections.singletonList(new RuntimeFieldPlugin())); + MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("runtime") + .startObject("field") + .endObject() + .endObject() + .endObject() + .endObject()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> mapperService.merge("type", new CompressedXContent(mapping), reason)); + assertEquals("Failed to parse mapping: No type specified for runtime field [field]", e.getMessage()); + } + + public void testRuntimeSectionWrongFormat() throws IOException { + MapperService mapperService = createMapperService(Collections.singletonList(new RuntimeFieldPlugin())); + MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("runtime") + .field("field", "value") + .endObject() + .endObject() + .endObject()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> mapperService.merge("type", new CompressedXContent(mapping), reason)); + assertEquals("Failed to parse mapping: Expected map for runtime field [field] definition but got a java.lang.String", + e.getMessage()); + } + + public void testRuntimeSectionRemainingField() throws IOException { + MapperService mapperService = createMapperService(Collections.singletonList(new RuntimeFieldPlugin())); + MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("runtime") + .startObject("field") + .field("type", "test") + .field("unsupported", "value") + .endObject() + .endObject() + .endObject() + .endObject()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> mapperService.merge("type", new CompressedXContent(mapping), reason)); + assertEquals("Failed to parse mapping: Mapping definition for [field] has unsupported parameters: " + + "[unsupported : value]", e.getMessage()); + } + + private static class RuntimeFieldPlugin extends Plugin implements MapperPlugin { + @Override + public Map getRuntimeFieldTypes() { + return Collections.singletonMap("test", (name, node, parserContext) -> { + Object prop1 = node.remove("prop1"); + Object prop2 = node.remove("prop2"); + return new RuntimeField(name, prop1 == null ? null : prop1.toString(), prop2 == null ? null : prop2.toString()); + }); + } + } + + private static final class RuntimeField extends RuntimeFieldType { + private final String prop1; + private final String prop2; + + protected RuntimeField(String name, String prop1, String prop2) { + super(name, Collections.emptyMap()); + this.prop1 = prop1; + this.prop2 = prop2; + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + return null; + } + + @Override + public String typeName() { + return "test"; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException { + if (prop1 != null) { + builder.field("prop1", prop1); + } + if (prop2 != null) { + builder.field("prop2", prop2); + } + } + + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java new file mode 100644 index 0000000000000..ccc52f502d993 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.search.Query; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.util.Collections; + +public final class TestRuntimeField extends RuntimeFieldType { + TestRuntimeField(String name) { + super(name, Collections.emptyMap()); + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) { + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + return null; + } + + @Override + public String typeName() { + return null; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index daf34df975157..5ff679432dcbb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -83,8 +83,8 @@ public void testMultiFieldWithinMultiField() throws IOException { MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); Version olderVersion = VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0); - Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext(null, type -> typeParser, olderVersion, null, - null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { + Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext(null, type -> typeParser, type -> null, + olderVersion, null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); @@ -100,8 +100,8 @@ public void testMultiFieldWithinMultiField() throws IOException { BytesReference.bytes(mapping), true, mapping.contentType()).v2(); Version version = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.CURRENT); - Mapper.TypeParser.ParserContext context = new Mapper.TypeParser.ParserContext(null, type -> typeParser, version, null, null, - null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { + Mapper.TypeParser.ParserContext context = new Mapper.TypeParser.ParserContext(null, type -> typeParser, type -> null, version, + null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index c4d63e051230d..27fa444ac8891 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -312,7 +312,7 @@ private static QueryShardContext createQueryShardContext(String indexUuid, Strin when(mapperService.index()).thenReturn(indexMetadata.getIndex()); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); Map typeParserMap = IndicesModule.getMappers(Collections.emptyList()); - Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(name -> null, typeParserMap::get, + Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(name -> null, typeParserMap::get, type -> null, Version.CURRENT, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 78c3330b23aa1..08bfae61d0cca 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -19,9 +19,6 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -30,9 +27,6 @@ // this sucks how much must be overridden just do get a dummy field mapper... public class MockFieldMapper extends FieldMapper { - static Settings DEFAULT_SETTINGS = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id) - .build(); public MockFieldMapper(String fullName) { this(new FakeFieldType(fullName)); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 86684785f6f69..e7d6f3fa7e624 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -785,7 +785,7 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd private static class MockParserContext extends Mapper.TypeParser.ParserContext { MockParserContext() { - super(null, null, null, null, null, null, null, null, null); + super(null, null, null, null, null, null, null, null, null, null); } @Override diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java index 28161418dfe42..32eeb72d8e454 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java @@ -39,7 +39,7 @@ public void testFieldTypeLookup() { String fieldName = "object1.object2.field"; FlattenedFieldMapper mapper = createFlattenedMapper(fieldName); - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList(), emptyList()); assertEquals(mapper.fieldType(), lookup.get(fieldName)); String objectKey = "key1.key2"; @@ -60,7 +60,7 @@ public void testFieldTypeLookupWithAlias() { String aliasName = "alias"; FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, fieldName); - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias)); + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias), emptyList()); assertEquals(mapper.fieldType(), lookup.get(aliasName)); String objectKey = "key1.key2"; @@ -83,11 +83,11 @@ public void testFieldTypeLookupWithMultipleFields() { FlattenedFieldMapper mapper2 = createFlattenedMapper(field2); FlattenedFieldMapper mapper3 = createFlattenedMapper(field3); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList(), emptyList()); assertNotNull(lookup.get(field1 + ".some.key")); assertNotNull(lookup.get(field2 + ".some.key")); - lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList()); + lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList(), emptyList()); assertNotNull(lookup.get(field1 + ".some.key")); assertNotNull(lookup.get(field2 + ".some.key")); assertNotNull(lookup.get(field3 + ".some.key")); @@ -124,7 +124,7 @@ public void testFieldLookupIterator() { MockFieldMapper mapper = new MockFieldMapper("foo"); FlattenedFieldMapper flattenedMapper = createFlattenedMapper("object1.object2.field"); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flattenedMapper), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flattenedMapper), emptyList(), emptyList()); Set fieldNames = new HashSet<>(); for (MappedFieldType fieldType : lookup) { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 62cb58fd94567..a07c314442a5f 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -6,29 +6,55 @@ package org.elasticsearch.xpack.runtimefields; -import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.xpack.runtimefields.mapper.BooleanFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.BooleanScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.DateFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.DateScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.DoubleFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.DoubleScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.IpFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.IpScriptFieldType; +import org.elasticsearch.xpack.runtimefields.mapper.KeywordScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.LongFieldScript; -import org.elasticsearch.xpack.runtimefields.mapper.RuntimeFieldMapper; +import org.elasticsearch.xpack.runtimefields.mapper.LongScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.StringFieldScript; -import java.util.Collections; import java.util.List; import java.util.Map; public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { @Override - public Map getMappers() { - return Collections.singletonMap(RuntimeFieldMapper.CONTENT_TYPE, RuntimeFieldMapper.PARSER); + public Map getRuntimeFieldTypes() { + return Map.of( + BooleanFieldMapper.CONTENT_TYPE, + BooleanScriptFieldType.PARSER, + NumberFieldMapper.NumberType.LONG.typeName(), + LongScriptFieldType.PARSER, + NumberFieldMapper.NumberType.DOUBLE.typeName(), + DoubleScriptFieldType.PARSER, + IpFieldMapper.CONTENT_TYPE, + IpScriptFieldType.PARSER, + DateFieldMapper.CONTENT_TYPE, + DateScriptFieldType.PARSER, + KeywordFieldMapper.CONTENT_TYPE, + KeywordScriptFieldType.PARSER, + GeoPointFieldMapper.CONTENT_TYPE, + GeoPointScriptFieldType.PARSER + ); } @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java index 0feb612f82018..e188431557ff5 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java @@ -12,52 +12,58 @@ import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocValueFetcher; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.lookup.SearchLookup; +import java.io.IOException; import java.time.ZoneId; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; /** * Abstract base {@linkplain MappedFieldType} for scripted fields. */ -abstract class AbstractScriptFieldType extends MappedFieldType { +abstract class AbstractScriptFieldType extends RuntimeFieldType { protected final Script script; private final TriFunction, SearchLookup, LeafFactory> factory; + private final CheckedBiConsumer toXContent; + + AbstractScriptFieldType(String name, TriFunction, SearchLookup, LeafFactory> factory, Builder builder) { + this(name, factory, builder.script.getValue(), builder.meta.getValue(), builder::toXContent); + } AbstractScriptFieldType( String name, - Script script, TriFunction, SearchLookup, LeafFactory> factory, - Map meta + Script script, + Map meta, + CheckedBiConsumer toXContent ) { - super(name, false, false, false, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); - this.script = script; + super(name, meta); this.factory = factory; - } - - protected abstract String runtimeType(); - - @Override - public final String typeName() { - return RuntimeFieldMapper.CONTENT_TYPE; - } - - @Override - public final String familyTypeName() { - return runtimeType(); + this.script = script; + this.toXContent = toXContent; } @Override @@ -101,8 +107,8 @@ public final Query rangeQuery( QueryShardContext context ) { if (relation == ShapeRelation.DISJOINT) { - String message = "Field [%s] of type [%s] with runtime type [%s] does not support DISJOINT ranges"; - throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName(), runtimeType())); + String message = "Runtime field [%s] of type [%s] does not support DISJOINT ranges"; + throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName())); } return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context); } @@ -174,46 +180,115 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew private String unsupported(String query, String supported) { return String.format( Locale.ROOT, - "Can only use %s queries on %s fields - not on [%s] which is of type [%s] with runtime_type [%s]", + "Can only use %s queries on %s fields - not on [%s] which is a runtime field of type [%s]", query, supported, name(), - RuntimeFieldMapper.CONTENT_TYPE, - runtimeType() + typeName() ); } protected final void checkAllowExpensiveQueries(QueryShardContext context) { if (context.allowExpensiveQueries() == false) { throw new ElasticsearchException( - "queries cannot be executed against [" - + RuntimeFieldMapper.CONTENT_TYPE - + "] fields while [" - + ALLOW_EXPENSIVE_QUERIES.getKey() - + "] is set to [false]." + "queries cannot be executed against runtime fields while [" + ALLOW_EXPENSIVE_QUERIES.getKey() + "] is set to [false]." ); } } - /** - * The format that this field should use. The default implementation is - * {@code null} because most fields don't support formats. - */ - protected String format() { - return null; + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup lookup, String format) { + return new DocValueFetcher(docValueFormat(format, null), lookup.doc().getForField(this)); + } + + @Override + protected final void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException { + toXContent.accept(builder, includeDefaults); } /** - * The locale that this field's format should use. The default - * implementation is {@code null} because most fields don't - * support formats. + * For runtime fields the {@link RuntimeFieldType.Parser} returns directly the {@link org.elasticsearch.index.mapper.MappedFieldType}. + * Internally we still create a {@link Builder} so we reuse the {@link FieldMapper.Parameter} infrastructure, + * but {@link Builder#init(FieldMapper)} and {@link Builder#build(ContentPath)} are never called as + * {@link RuntimeFieldTypeParser#parse(String, Map, Mapper.TypeParser.ParserContext)} calls + * {@link Builder#parse(String, Mapper.TypeParser.ParserContext, Map)} and returns the corresponding + * {@link org.elasticsearch.index.mapper.MappedFieldType}. */ - protected Locale formatLocale() { - return null; + abstract static class Builder extends FieldMapper.Builder { + final FieldMapper.Parameter> meta = FieldMapper.Parameter.metaParam(); + final FieldMapper.Parameter