Skip to content

Commit cb1f603

Browse files
javannajavanna
authored
Introduce dynamic runtime setting (#65489)
The dynamic:runtime setting is similar to dynamic:true in that it dynamically defines fields based on values parsed from incoming documents. Though instead of defining leaf fields under properties, it defines them as runtime fields under the runtime section. This is useful in scenarios where search speed can be traded for storage costs, given that runtime fields are loaded at runtime rather than indexed.
1 parent b125033 commit cb1f603

35 files changed

+1194
-323
lines changed

‎modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1119,7 +1119,7 @@ private void duelRun(PercolateQuery.QueryStore queryStore, MemoryIndex memoryInd
11191119
}
11201120

11211121
private void addQuery(Query query, List<ParseContext.Document> docs) {
1122-
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
1122+
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
11231123
fieldMapper.processQuery(query, parseContext);
11241124
ParseContext.Document queryDocument = parseContext.doc();
11251125
// Add to string representation of the query to make debugging easier:

‎modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ public void testExtractTerms() throws Exception {
181181

182182
DocumentMapper documentMapper = mapperService.documentMapper("doc");
183183
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
184-
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
184+
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
185185
fieldMapper.processQuery(bq.build(), parseContext);
186186
ParseContext.Document document = parseContext.doc();
187187

@@ -202,7 +202,7 @@ public void testExtractTerms() throws Exception {
202202
bq.add(termQuery1, Occur.MUST);
203203
bq.add(termQuery2, Occur.MUST);
204204

205-
parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
205+
parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
206206
fieldMapper.processQuery(bq.build(), parseContext);
207207
document = parseContext.doc();
208208

@@ -231,7 +231,7 @@ public void testExtractRanges() throws Exception {
231231

232232
DocumentMapper documentMapper = mapperService.documentMapper("doc");
233233
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
234-
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
234+
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
235235
fieldMapper.processQuery(bq.build(), parseContext);
236236
ParseContext.Document document = parseContext.doc();
237237

@@ -256,7 +256,7 @@ public void testExtractRanges() throws Exception {
256256
.rangeQuery(15, 20, true, true, null, null, null, context);
257257
bq.add(rangeQuery2, Occur.MUST);
258258

259-
parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
259+
parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
260260
fieldMapper.processQuery(bq.build(), parseContext);
261261
document = parseContext.doc();
262262

@@ -279,7 +279,7 @@ public void testExtractTermsAndRanges_failed() throws Exception {
279279
TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true);
280280
DocumentMapper documentMapper = mapperService.documentMapper("doc");
281281
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
282-
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
282+
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
283283
fieldMapper.processQuery(query, parseContext);
284284
ParseContext.Document document = parseContext.doc();
285285

@@ -293,7 +293,7 @@ public void testExtractTermsAndRanges_partial() throws Exception {
293293
PhraseQuery phraseQuery = new PhraseQuery("field", "term");
294294
DocumentMapper documentMapper = mapperService.documentMapper("doc");
295295
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
296-
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
296+
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
297297
fieldMapper.processQuery(phraseQuery, parseContext);
298298
ParseContext.Document document = parseContext.doc();
299299

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

Lines changed: 68 additions & 185 deletions
Large diffs are not rendered by default.

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

Lines changed: 322 additions & 0 deletions
Large diffs are not rendered by default.

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicRuntimeFieldsBuilder.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.index.mapper;
21+
22+
import org.elasticsearch.common.time.DateFormatter;
23+
import org.elasticsearch.plugins.MapperPlugin;
24+
25+
import static org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
26+
27+
/**
28+
* Defines how runtime fields are dynamically created. Used when objects are mapped with dynamic:runtime.
29+
* Plugins that provide runtime field implementations can also plug in their implementation of this interface
30+
* to define how leaf fields of each supported type can be dynamically created in dynamic runtime mode.
31+
*
32+
* @see MapperPlugin#getDynamicRuntimeFieldsBuilder()
33+
* @see Dynamic
34+
*/
35+
public interface DynamicRuntimeFieldsBuilder {
36+
/**
37+
* Dynamically creates a runtime field from a parsed string value
38+
*/
39+
RuntimeFieldType newDynamicStringField(String name);
40+
/**
41+
* Dynamically creates a runtime field from a parsed long value
42+
*/
43+
RuntimeFieldType newDynamicLongField(String name);
44+
/**
45+
* Dynamically creates a runtime field from a parsed double value
46+
*/
47+
RuntimeFieldType newDynamicDoubleField(String name);
48+
/**
49+
* Dynamically creates a runtime field from a parsed boolean value
50+
*/
51+
RuntimeFieldType newDynamicBooleanField(String name);
52+
/**
53+
* Dynamically creates a runtime field from a parsed date value
54+
*/
55+
RuntimeFieldType newDynamicDateField(String name, DateFormatter dateFormatter);
56+
}

‎server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ class ParserContext {
6060
private final Function<String, SimilarityProvider> similarityLookupService;
6161
private final Function<String, TypeParser> typeParsers;
6262
private final Function<String, RuntimeFieldType.Parser> runtimeTypeParsers;
63+
private final boolean supportsDynamicRuntimeMappings;
6364
private final Version indexVersionCreated;
6465
private final Supplier<QueryShardContext> queryShardContextSupplier;
6566
private final DateFormatter dateFormatter;
@@ -77,7 +78,8 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
7778
ScriptService scriptService,
7879
IndexAnalyzers indexAnalyzers,
7980
IndexSettings indexSettings,
80-
BooleanSupplier idFieldDataEnabled) {
81+
BooleanSupplier idFieldDataEnabled,
82+
boolean supportsDynamicRuntimeMappings) {
8183
this.similarityLookupService = similarityLookupService;
8284
this.typeParsers = typeParsers;
8385
this.runtimeTypeParsers = runtimeTypeParsers;
@@ -88,6 +90,7 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
8890
this.indexAnalyzers = indexAnalyzers;
8991
this.indexSettings = indexSettings;
9092
this.idFieldDataEnabled = idFieldDataEnabled;
93+
this.supportsDynamicRuntimeMappings = supportsDynamicRuntimeMappings;
9194
}
9295

9396
public IndexAnalyzers getIndexAnalyzers() {
@@ -118,6 +121,10 @@ public RuntimeFieldType.Parser runtimeFieldTypeParser(String type) {
118121
return runtimeTypeParsers.apply(type);
119122
}
120123

124+
public boolean supportsDynamicRuntimeMappings() {
125+
return supportsDynamicRuntimeMappings;
126+
}
127+
121128
public Version indexVersionCreated() {
122129
return indexVersionCreated;
123130
}
@@ -154,7 +161,7 @@ static class MultiFieldParserContext extends ParserContext {
154161
MultiFieldParserContext(ParserContext in) {
155162
super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated,
156163
in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings,
157-
in.idFieldDataEnabled);
164+
in.idFieldDataEnabled, in.supportsDynamicRuntimeMappings);
158165
}
159166

160167
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,8 +146,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
146146
Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction =
147147
dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get,
148148
mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter,
149-
scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled);
150-
this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction);
149+
scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled, mapperRegistry.getDynamicRuntimeFieldsBuilder() != null);
150+
this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction, mapperRegistry.getDynamicRuntimeFieldsBuilder());
151151
Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers =
152152
mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated());
153153
this.parserContextSupplier = () -> parserContextFunction.apply(null);

‎server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,8 @@ public void validate(MappingLookup mappers) {
8585
/**
8686
* Generate a mapping update for the given root object mapper.
8787
*/
88-
public Mapping mappingUpdate(Mapper rootObjectMapper) {
89-
return new Mapping((RootObjectMapper) rootObjectMapper, metadataMappers, meta);
88+
public Mapping mappingUpdate(RootObjectMapper rootObjectMapper) {
89+
return new Mapping(rootObjectMapper, metadataMappers, meta);
9090
}
9191

9292
/** Get the root mapper with the given class. */

‎server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,27 @@ public class ObjectMapper extends Mapper implements Cloneable {
5353
public static class Defaults {
5454
public static final boolean ENABLED = true;
5555
public static final Nested NESTED = Nested.NO;
56-
public static final Dynamic DYNAMIC = null; // not set, inherited from root
5756
}
5857

5958
public enum Dynamic {
60-
TRUE,
59+
TRUE {
60+
@Override
61+
DynamicFieldsBuilder getDynamicFieldsBuilder() {
62+
return DynamicFieldsBuilder.DYNAMIC_TRUE;
63+
}
64+
},
6165
FALSE,
62-
STRICT
66+
STRICT,
67+
RUNTIME {
68+
@Override
69+
DynamicFieldsBuilder getDynamicFieldsBuilder() {
70+
return DynamicFieldsBuilder.DYNAMIC_RUNTIME;
71+
}
72+
};
73+
74+
DynamicFieldsBuilder getDynamicFieldsBuilder() {
75+
throw new UnsupportedOperationException("Cannot create dynamic fields when dynamic is set to [" + this + "]");
76+
};
6377
}
6478

6579
public static class Nested {
@@ -139,7 +153,7 @@ public static class Builder extends Mapper.Builder {
139153

140154
protected Nested nested = Defaults.NESTED;
141155

142-
protected Dynamic dynamic = Defaults.DYNAMIC;
156+
protected Dynamic dynamic;
143157

144158
protected final List<Mapper.Builder> mappersBuilders = new ArrayList<>();
145159
protected final Version indexCreatedVersion;
@@ -198,7 +212,7 @@ public static class TypeParser implements Mapper.TypeParser {
198212
@Override
199213
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
200214
ObjectMapper.Builder builder = new Builder(name, parserContext.indexVersionCreated());
201-
parseNested(name, node, builder, parserContext);
215+
parseNested(name, node, builder);
202216
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
203217
Map.Entry<String, Object> entry = iterator.next();
204218
String fieldName = entry.getKey();
@@ -216,6 +230,12 @@ protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, O
216230
String value = fieldNode.toString();
217231
if (value.equalsIgnoreCase("strict")) {
218232
builder.dynamic(Dynamic.STRICT);
233+
} else if (value.equalsIgnoreCase("runtime")) {
234+
if (parserContext.supportsDynamicRuntimeMappings() == false) {
235+
throw new IllegalArgumentException("unable to set dynamic:runtime as there is " +
236+
"no registered dynamic runtime fields builder");
237+
}
238+
builder.dynamic(Dynamic.RUNTIME);
219239
} else {
220240
boolean dynamic = XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".dynamic");
221241
builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE);
@@ -241,8 +261,7 @@ protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, O
241261
return false;
242262
}
243263

244-
protected static void parseNested(String name, Map<String, Object> node, ObjectMapper.Builder builder,
245-
ParserContext parserContext) {
264+
protected static void parseNested(String name, Map<String, Object> node, ObjectMapper.Builder builder) {
246265
boolean nested = false;
247266
Explicit<Boolean> nestedIncludeInParent = new Explicit<>(false, false);
248267
Explicit<Boolean> nestedIncludeInRoot = new Explicit<>(false, false);
@@ -383,13 +402,18 @@ protected ObjectMapper clone() {
383402
return clone;
384403
}
385404

405+
ObjectMapper copyAndReset() {
406+
ObjectMapper copy = clone();
407+
// reset the sub mappers
408+
copy.mappers = new CopyOnWriteHashMap<>();
409+
return copy;
410+
}
411+
386412
/**
387413
* Build a mapping update with the provided sub mapping update.
388414
*/
389-
public ObjectMapper mappingUpdate(Mapper mapper) {
390-
ObjectMapper mappingUpdate = clone();
391-
// reset the sub mappers
392-
mappingUpdate.mappers = new CopyOnWriteHashMap<>();
415+
final ObjectMapper mappingUpdate(Mapper mapper) {
416+
ObjectMapper mappingUpdate = copyAndReset();
393417
mappingUpdate.putMapper(mapper);
394418
return mappingUpdate;
395419
}
@@ -566,5 +590,4 @@ public int compare(Mapper o1, Mapper o2) {
566590
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
567591

568592
}
569-
570593
}

0 commit comments

Comments
 (0)