Skip to content

Commit 1d88fe6

Browse files
javannajavanna
authored
Dynamic runtime to not dynamically create objects (#74234)
When we introduced dynamic:runtime (#65489) we decided to have it create objects dynamically under properties, as the runtime section did not (and still does not) support object fields. That proved to be a poor choice, because the runtime section is flat, supports dots in field names, and does not really need objects. Also, these end up causing unnecessary mapping conflicts. With this commit we adapt dynamic:runtime to not dynamically create objects. Closes #70268
1 parent 331a44b commit 1d88fe6

File tree

7 files changed

+134
-31
lines changed

7 files changed

+134
-31
lines changed

‎docs/reference/mapping/dynamic/field-mapping.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@ h| JSON data type h| `"dynamic":"true"` h| `"dynamic":"runtime"`
2222
|`true` or `false` 2*| `boolean`
2323
|`double` | `float` | `double`
2424
|`integer` 2*| `long`
25-
|`object`^1^ 2*| `object`
25+
|`object` | `object` | No field added
2626
|`array` 2*| Depends on the first non-`null` value in the array
2727
|`string` that passes <<date-detection,date detection>> 2*| `date`
2828
|`string` that passes <<numeric-detection,numeric detection>> | `float` or `long` | `double` or `long`
2929
|`string` that doesn't pass `date` detection or `numeric` detection | `text` with a `.keyword` sub-field | `keyword`
30-
3+| ^1^Objects are always mapped as part of the `properties` section, even when the `dynamic` parameter is set to `runtime`. | |
30+
3+|
3131
|===
3232
// end::dynamic-field-mapping-types-tag[]
3333

‎server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,17 @@
2020
import org.elasticsearch.cluster.metadata.MappingMetadata;
2121
import org.elasticsearch.cluster.service.ClusterService;
2222
import org.elasticsearch.common.Randomness;
23+
import org.elasticsearch.common.Strings;
2324
import org.elasticsearch.common.geo.GeoPoint;
2425
import org.elasticsearch.common.settings.Settings;
25-
import org.elasticsearch.core.TimeValue;
2626
import org.elasticsearch.common.xcontent.XContentBuilder;
2727
import org.elasticsearch.common.xcontent.XContentFactory;
28+
import org.elasticsearch.common.xcontent.XContentType;
29+
import org.elasticsearch.common.xcontent.support.XContentMapValues;
30+
import org.elasticsearch.core.TimeValue;
2831
import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder;
2932
import org.elasticsearch.plugins.Plugin;
33+
import org.elasticsearch.rest.RestStatus;
3034
import org.elasticsearch.test.ESIntegTestCase;
3135
import org.elasticsearch.test.InternalSettingsPlugin;
3236
import org.hamcrest.Matchers;
@@ -44,6 +48,8 @@
4448
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING;
4549
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
4650
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
51+
import static org.hamcrest.Matchers.contains;
52+
import static org.hamcrest.Matchers.containsInAnyOrder;
4753
import static org.hamcrest.Matchers.containsString;
4854
import static org.hamcrest.Matchers.equalTo;
4955
import static org.hamcrest.Matchers.instanceOf;
@@ -327,4 +333,78 @@ public void testBulkRequestWithNotFoundDynamicTemplate() throws Exception {
327333
assertThat(bulkItemResponses.getItems()[1].getFailureMessage(),
328334
containsString("Can't find dynamic template for dynamic template name [bar_foo] of field [address.location]"));
329335
}
336+
337+
public void testDynamicRuntimeNoConflicts() {
338+
assertAcked(client().admin().indices().prepareCreate("test").setMapping("{\"_doc\":{\"dynamic\":\"runtime\"}}").get());
339+
340+
List<IndexRequest> docs = new ArrayList<>();
341+
docs.add(new IndexRequest("test").source("one.two.three", new int[]{1, 2, 3}));
342+
docs.add(new IndexRequest("test").source("one.two", 1.2));
343+
docs.add(new IndexRequest("test").source("one", "one"));
344+
docs.add(new IndexRequest("test").source("{\"one\":{\"two\": { \"three\": \"three\"}}}", XContentType.JSON));
345+
Collections.shuffle(docs, random());
346+
BulkRequest bulkRequest = new BulkRequest();
347+
for (IndexRequest doc : docs) {
348+
bulkRequest.add(doc);
349+
}
350+
BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet();
351+
assertFalse(bulkItemResponses.buildFailureMessage(), bulkItemResponses.hasFailures());
352+
353+
GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
354+
Map<String, Object> sourceAsMap = getMappingsResponse.getMappings().get("test").sourceAsMap();
355+
assertFalse(sourceAsMap.containsKey("properties"));
356+
@SuppressWarnings("unchecked")
357+
Map<String, Object> runtime = (Map<String, Object>)sourceAsMap.get("runtime");
358+
//depending on the order of the documents field types may differ, but there are no mapping conflicts
359+
assertThat(runtime.keySet(), containsInAnyOrder("one", "one.two", "one.two.three"));
360+
}
361+
362+
public void testDynamicRuntimeObjectFields() {
363+
assertAcked(client().admin().indices().prepareCreate("test").setMapping("{\"_doc\":{\"properties\":{" +
364+
"\"obj\":{\"properties\":{\"runtime\":{\"type\":\"object\",\"dynamic\":\"runtime\"}}}}}}").get());
365+
366+
List<IndexRequest> docs = new ArrayList<>();
367+
docs.add(new IndexRequest("test").source("obj.one", 1));
368+
docs.add(new IndexRequest("test").source("anything", 1));
369+
docs.add(new IndexRequest("test").source("obj.runtime.one.two", "test"));
370+
docs.add(new IndexRequest("test").source("obj.runtime.one", "one"));
371+
docs.add(new IndexRequest("test").source("{\"obj\":{\"runtime\":{\"one\":{\"two\": \"test\"}}}}", XContentType.JSON));
372+
Collections.shuffle(docs, random());
373+
BulkRequest bulkRequest = new BulkRequest();
374+
for (IndexRequest doc : docs) {
375+
bulkRequest.add(doc);
376+
}
377+
BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet();
378+
assertFalse(bulkItemResponses.buildFailureMessage(), bulkItemResponses.hasFailures());
379+
380+
MapperParsingException exception = expectThrows(MapperParsingException.class,
381+
() -> client().prepareIndex("test").setSource("obj.runtime", "value").get());
382+
assertEquals("object mapping for [obj.runtime] tried to parse field [obj.runtime] as object, but found a concrete value",
383+
exception.getMessage());
384+
385+
assertEquals("{\"test\":{\"mappings\":" +
386+
"{\"runtime\":{\"obj.runtime.one\":{\"type\":\"keyword\"},\"obj.runtime.one.two\":{\"type\":\"keyword\"}}," +
387+
"\"properties\":{\"anything\":{\"type\":\"long\"}," +
388+
"\"obj\":{\"properties\":{\"one\":{\"type\":\"long\"}," +
389+
"\"runtime\":{\"type\":\"object\",\"dynamic\":\"runtime\"}}}}}}}",
390+
Strings.toString(client().admin().indices().prepareGetMappings("test").get()));
391+
392+
assertAcked(client().admin().indices().preparePutMapping("test").setSource("{\"_doc\":{\"properties\":{\"obj\":{\"properties\":" +
393+
"{\"runtime\":{\"properties\":{\"dynamic\":{\"type\":\"object\", \"dynamic\":true}}}}}}}}", XContentType.JSON));
394+
395+
assertEquals(RestStatus.CREATED, client().prepareIndex("test").setSource("obj.runtime.dynamic.leaf", 1).get().status());
396+
GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
397+
Map<String, Object> sourceAsMap = getMappingsResponse.getMappings().get("test").sourceAsMap();
398+
assertThat(
399+
XContentMapValues.extractRawValues("properties.obj.properties.runtime.properties.dynamic.properties.leaf.type", sourceAsMap),
400+
contains("long"));
401+
}
402+
403+
private static Map<String, Object> getMappedField(Map<String, Object> sourceAsMap, String name) {
404+
@SuppressWarnings("unchecked")
405+
Map<String, Object> properties = (Map<String, Object>)sourceAsMap.get("properties");
406+
@SuppressWarnings("unchecked")
407+
Map<String, Object> mappedField = (Map<String, Object>)properties.get(name);
408+
return mappedField;
409+
}
330410
}

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,16 @@
1313
import org.apache.lucene.index.LeafReaderContext;
1414
import org.apache.lucene.search.Query;
1515
import org.elasticsearch.Version;
16+
import org.elasticsearch.common.Explicit;
1617
import org.elasticsearch.common.Strings;
17-
import org.elasticsearch.core.Tuple;
1818
import org.elasticsearch.common.time.DateFormatter;
1919
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
2020
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
2121
import org.elasticsearch.common.xcontent.XContentBuilder;
2222
import org.elasticsearch.common.xcontent.XContentHelper;
2323
import org.elasticsearch.common.xcontent.XContentParser;
2424
import org.elasticsearch.common.xcontent.XContentType;
25+
import org.elasticsearch.core.Tuple;
2526
import org.elasticsearch.index.IndexSettings;
2627
import org.elasticsearch.index.analysis.IndexAnalyzers;
2728
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -568,12 +569,19 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
568569
ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
569570
if (dynamic == ObjectMapper.Dynamic.STRICT) {
570571
throw new StrictDynamicMappingException(mapper.fullPath(), currentFieldName);
571-
} else if ( dynamic == ObjectMapper.Dynamic.FALSE) {
572+
} else if (dynamic == ObjectMapper.Dynamic.FALSE) {
572573
// not dynamic, read everything up to end object
573574
context.parser().skipChildren();
574575
} else {
575-
Mapper dynamicObjectMapper = dynamic.getDynamicFieldsBuilder().createDynamicObjectMapper(context, currentFieldName);
576-
context.addDynamicMapper(dynamicObjectMapper);
576+
Mapper dynamicObjectMapper;
577+
if (dynamic == ObjectMapper.Dynamic.RUNTIME) {
578+
//with dynamic:runtime all leaf fields will be runtime fields unless explicitly mapped,
579+
//hence we don't dynamically create empty objects under properties, but rather carry around an artificial object mapper
580+
dynamicObjectMapper = new NoOpObjectMapper(currentFieldName, context.path().pathAsText(currentFieldName));
581+
} else {
582+
dynamicObjectMapper = dynamic.getDynamicFieldsBuilder().createDynamicObjectMapper(context, currentFieldName);
583+
context.addDynamicMapper(dynamicObjectMapper);
584+
}
577585
context.path().add(currentFieldName);
578586
parseObjectOrField(context, dynamicObjectMapper);
579587
context.path().remove();
@@ -759,7 +767,8 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(ParseContext
759767
int pathsAdded = 0;
760768
ObjectMapper parent = mapper;
761769
for (int i = 0; i < paths.length-1; i++) {
762-
String currentPath = context.path().pathAsText(paths[i]);
770+
String name = paths[i];
771+
String currentPath = context.path().pathAsText(name);
763772
Mapper existingFieldMapper = context.mappingLookup().getMapper(currentPath);
764773
if (existingFieldMapper != null) {
765774
throw new MapperParsingException(
@@ -771,13 +780,14 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(ParseContext
771780
// One mapping is missing, check if we are allowed to create a dynamic one.
772781
ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context);
773782
if (dynamic == ObjectMapper.Dynamic.STRICT) {
774-
throw new StrictDynamicMappingException(parent.fullPath(), paths[i]);
783+
throw new StrictDynamicMappingException(parent.fullPath(), name);
775784
} else if (dynamic == ObjectMapper.Dynamic.FALSE) {
776785
// Should not dynamically create any more mappers so return the last mapper
777786
return new Tuple<>(pathsAdded, parent);
787+
} else if (dynamic == ObjectMapper.Dynamic.RUNTIME) {
788+
mapper = new NoOpObjectMapper(name, currentPath);
778789
} else {
779-
//objects are created under properties even with dynamic: runtime, as the runtime section only holds leaf fields
780-
final Mapper fieldMapper = dynamic.getDynamicFieldsBuilder().createDynamicObjectMapper(context, paths[i]);
790+
final Mapper fieldMapper = dynamic.getDynamicFieldsBuilder().createDynamicObjectMapper(context, name);
781791
if (fieldMapper instanceof ObjectMapper == false) {
782792
assert context.sourceToParse().dynamicTemplates().containsKey(currentPath) :
783793
"dynamic templates [" + context.sourceToParse().dynamicTemplates() + "]";
@@ -957,4 +967,10 @@ protected String contentType() {
957967
throw new UnsupportedOperationException();
958968
}
959969
}
970+
971+
private static class NoOpObjectMapper extends ObjectMapper {
972+
NoOpObjectMapper(String name, String fullPath) {
973+
super(name, fullPath, new Explicit<>(true, false), Nested.NO, Dynamic.RUNTIME, Collections.emptyMap(), Version.CURRENT);
974+
}
975+
}
960976
}

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@
1010

1111
import org.elasticsearch.ElasticsearchParseException;
1212
import org.elasticsearch.common.CheckedBiConsumer;
13-
import org.elasticsearch.core.CheckedRunnable;
1413
import org.elasticsearch.common.settings.Settings;
1514
import org.elasticsearch.common.time.DateFormatter;
1615
import org.elasticsearch.common.xcontent.XContentParser;
16+
import org.elasticsearch.core.CheckedRunnable;
1717
import org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
1818
import org.elasticsearch.script.ScriptCompiler;
1919

@@ -23,8 +23,9 @@
2323

2424
/**
2525
* Encapsulates the logic for dynamically creating fields as part of document parsing.
26-
* Objects are always created the same, but leaf fields can be mapped under properties, as concrete fields that get indexed,
26+
* Fields can be mapped under properties, as concrete fields that get indexed,
2727
* or as runtime fields that are evaluated at search-time and have no indexing overhead.
28+
* Objects get dynamically mapped only under dynamic:true.
2829
*/
2930
final class DynamicFieldsBuilder {
3031
private static final Concrete CONCRETE = new Concrete(DocumentParser::parseObjectOrField);
@@ -121,18 +122,15 @@ void createDynamicFieldFromValue(final ParseContext context,
121122

122123
/**
123124
* Returns a dynamically created object mapper, eventually based on a matching dynamic template.
124-
* Note that objects are always mapped under properties.
125125
*/
126126
Mapper createDynamicObjectMapper(ParseContext context, String name) {
127-
//dynamic:runtime maps objects under properties, exactly like dynamic:true
128127
Mapper mapper = createObjectMapperFromTemplate(context, name);
129128
return mapper != null ? mapper :
130129
new ObjectMapper.Builder(name, context.indexSettings().getIndexVersionCreated()).enabled(true).build(context.path());
131130
}
132131

133132
/**
134133
* Returns a dynamically created object mapper, based exclusively on a matching dynamic template, null otherwise.
135-
* Note that objects are always mapped under properties.
136134
*/
137135
Mapper createObjectMapperFromTemplate(ParseContext context, String name) {
138136
Mapper.Builder templateBuilder = findTemplateBuilderForObject(context, name);
@@ -311,7 +309,7 @@ void newDynamicBinaryField(ParseContext context, String name) throws IOException
311309

312310
/**
313311
* Dynamically creates runtime fields, in the runtime section.
314-
* Used for leaf fields, when their parent object is mapped as dynamic:runtime.
312+
* Used for sub-fields of objects that are mapped as dynamic:runtime.
315313
* @see Dynamic
316314
*/
317315
private static final class Runtime implements Strategy {

‎server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -556,8 +556,7 @@ public void testPropagateDynamicRuntimeWithDynamicMapper() throws Exception {
556556
}));
557557
assertNull(doc.rootDoc().getField("foo.bar.baz"));
558558
assertEquals("{\"_doc\":{\"dynamic\":\"false\"," +
559-
"\"runtime\":{\"foo.bar.baz\":{\"type\":\"keyword\"},\"foo.baz\":{\"type\":\"keyword\"}}," +
560-
"\"properties\":{\"foo\":{\"dynamic\":\"runtime\",\"properties\":{\"bar\":{\"type\":\"object\"}}}}}}",
559+
"\"runtime\":{\"foo.bar.baz\":{\"type\":\"keyword\"},\"foo.baz\":{\"type\":\"keyword\"}}}}",
561560
Strings.toString(doc.dynamicMappingsUpdate()));
562561
}
563562

‎server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@
77
*/
88
package org.elasticsearch.index.mapper;
99

10-
import org.elasticsearch.core.CheckedConsumer;
1110
import org.elasticsearch.common.Strings;
1211
import org.elasticsearch.common.bytes.BytesReference;
1312
import org.elasticsearch.common.xcontent.XContentBuilder;
1413
import org.elasticsearch.common.xcontent.XContentFactory;
14+
import org.elasticsearch.core.CheckedConsumer;
1515

1616
import java.io.IOException;
1717
import java.time.Instant;
@@ -302,8 +302,7 @@ public void testDynamicRuntimeFieldWithinObject() throws Exception {
302302
}));
303303

304304
assertEquals("{\"_doc\":{\"dynamic\":\"runtime\"," +
305-
"\"runtime\":{\"foo.bar.baz\":{\"type\":\"long\"}}," +
306-
"\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\":\"object\"}}}}}}",
305+
"\"runtime\":{\"foo.bar.baz\":{\"type\":\"long\"}}}}",
307306
Strings.toString(doc.dynamicMappingsUpdate()));
308307
}
309308

@@ -326,8 +325,7 @@ public void testDynamicRuntimeMappingDynamicObject() throws Exception {
326325
assertEquals("{\"_doc\":{\"dynamic\":\"runtime\"," +
327326
"\"runtime\":{\"object.foo.bar.baz\":{\"type\":\"long\"}}," +
328327
"\"properties\":{\"dynamic_object\":{\"dynamic\":\"true\"," +
329-
"\"properties\":{\"foo\":{" + "\"properties\":{\"bar\":{" + "\"properties\":{\"baz\":" + "{\"type\":\"long\"}}}}}}}," +
330-
"\"object\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\":\"object\"}}}}}}}}",
328+
"\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"long\"}}}}}}}}}}",
331329
Strings.toString(doc.dynamicMappingsUpdate()));
332330
}
333331

@@ -350,8 +348,7 @@ public void testDynamicMappingDynamicRuntimeObject() throws Exception {
350348
assertEquals("{\"_doc\":{\"dynamic\":\"true\",\"" +
351349
"runtime\":{\"runtime_object.foo.bar.baz\":{\"type\":\"keyword\"}}," +
352350
"\"properties\":{\"object\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"properties\":{" +
353-
"\"baz\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}}}}," +
354-
"\"runtime_object\":{\"dynamic\":\"runtime\",\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\":\"object\"}}}}}}}}",
351+
"\"baz\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}}}}}}}",
355352
Strings.toString(doc.dynamicMappingsUpdate()));
356353
}
357354

‎server/src/test/java/org/elasticsearch/index/mapper/DynamicRuntimeTests.java

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,6 @@
99
package org.elasticsearch.index.mapper;
1010

1111
import org.elasticsearch.common.Strings;
12-
import org.elasticsearch.index.mapper.DocumentMapper;
13-
import org.elasticsearch.index.mapper.MapperServiceTestCase;
14-
import org.elasticsearch.index.mapper.ObjectMapper;
15-
import org.elasticsearch.index.mapper.ParsedDocument;
1612

1713
import java.io.IOException;
1814

@@ -72,7 +68,7 @@ public void testWithObjects() throws IOException {
7268
"{\"_doc\":{\"dynamic\":\"false\","
7369
+ "\"runtime\":{\"dynamic_runtime.child.field4\":{\"type\":\"keyword\"},"
7470
+ "\"dynamic_runtime.field3\":{\"type\":\"keyword\"}},"
75-
+ "\"properties\":{\"dynamic_runtime\":{\"dynamic\":\"runtime\",\"properties\":{\"child\":{\"type\":\"object\"}}},"
71+
+ "\"properties\":{"
7672
+ "\"dynamic_true\":{\"dynamic\":\"true\",\"properties\":{\"child\":{\"properties\":{"
7773
+ "\"field2\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}},"
7874
+ "\"field1\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}}}",
@@ -109,4 +105,21 @@ public void testWithDynamicTemplate() throws IOException {
109105
Strings.toString(parsedDoc.dynamicMappingsUpdate())
110106
);
111107
}
108+
109+
public void testDotsInFieldNames() throws IOException {
110+
DocumentMapper documentMapper = createDocumentMapper(topMapping(b -> b.field("dynamic", ObjectMapper.Dynamic.RUNTIME)));
111+
ParsedDocument doc = documentMapper.parse(source(b -> {
112+
b.field("one.two.three.four", "1234");
113+
b.field("one.two.three", 123);
114+
b.array("one.two", 1.2, 1.2, 1.2);
115+
b.field("one", "one");
116+
}));
117+
assertEquals("{\"_doc\":{\"dynamic\":\"runtime\",\"runtime\":{" +
118+
"\"one\":{\"type\":\"keyword\"}," +
119+
"\"one.two\":{\"type\":\"double\"}," +
120+
"\"one.two.three\":{\"type\":\"long\"}," +
121+
"\"one.two.three.four\":{\"type\":\"keyword\"}}}}",
122+
Strings.toString(doc.dynamicMappingsUpdate())
123+
);
124+
}
112125
}

0 commit comments

Comments
 (0)