Skip to content

Commit bff9113

Browse files
javannajavanna
authored
Read subobjects mapping parameter in advance (#86894)
As part of #86166 we added support for a new mapping parameter called `subobjects` that can be set to the `object` field type. Such parameter will affect not only how incoming documents will be dynamically mapped in the future, but also how field names are treated as part of the mappings themselves. Mappings get parsed into a map, where keys ordering is not guaranteed. In case a mappings call contains `subobjects` set to `false` and also sub-fields for that same object, we need to make sure that we read the parameter in advance in order to know how to treat the sub-field and decide whether we need to expand dots in field names or leave them as they are.
1 parent ca542fe commit bff9113

File tree

17 files changed

+245
-54
lines changed

17 files changed

+245
-54
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.elasticsearch.index.mapper.Mapping;
4141
import org.elasticsearch.index.mapper.MappingLookup;
4242
import org.elasticsearch.index.mapper.MetadataFieldMapper;
43+
import org.elasticsearch.index.mapper.ObjectMapper;
4344
import org.elasticsearch.index.mapper.RootObjectMapper;
4445
import org.elasticsearch.indices.EmptySystemIndices;
4546
import org.elasticsearch.indices.IndicesService;
@@ -218,7 +219,7 @@ public void setup() throws Exception {
218219
false,
219220
Version.CURRENT
220221
).build(MapperBuilderContext.ROOT);
221-
RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc");
222+
RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc", ObjectMapper.Defaults.SUBOBJECTS);
222223
root.add(
223224
new DateFieldMapper.Builder(
224225
"@timestamp",

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
"Metrics indexing":
2+
"Metrics object indexing":
33
- skip:
44
features: allowed_warnings_regex
55
version: " - 8.2.99"
@@ -17,6 +17,9 @@
1717
mapping:
1818
type: object
1919
subobjects: false
20+
properties:
21+
host.name:
22+
type: keyword
2023

2124
- do:
2225
allowed_warnings_regex:
@@ -26,15 +29,62 @@
2629
id: 1
2730
refresh: true
2831
body:
29-
{ metrics.time: 10, metrics.time.max: 100, metrics.time.min: 1 }
32+
{ metrics.host.name: localhost, metrics.host.id: 1, metrics.time: 10, metrics.time.max: 100, metrics.time.min: 1 }
3033

3134
- do:
3235
field_caps:
3336
index: test-1
34-
fields: metrics.time*
37+
fields: metrics*
38+
- match: {fields.metrics\.host\.id.long.searchable: true}
39+
- match: {fields.metrics\.host\.id.long.aggregatable: true}
40+
- match: {fields.metrics\.host\.name.keyword.searchable: true}
41+
- match: {fields.metrics\.host\.name.keyword.aggregatable: true}
3542
- match: {fields.metrics\.time.long.searchable: true}
3643
- match: {fields.metrics\.time.long.aggregatable: true}
3744
- match: {fields.metrics\.time\.max.long.searchable: true}
3845
- match: {fields.metrics\.time\.max.long.aggregatable: true}
3946
- match: {fields.metrics\.time\.min.long.searchable: true}
4047
- match: {fields.metrics\.time\.min.long.aggregatable: true}
48+
49+
---
50+
"Root without subobjects":
51+
- skip:
52+
features: allowed_warnings_regex
53+
version: " - 8.2.99"
54+
reason: added in 8.3.0
55+
56+
- do:
57+
indices.put_template:
58+
name: test
59+
body:
60+
index_patterns: test-*
61+
mappings:
62+
subobjects: false
63+
properties:
64+
host.name:
65+
type: keyword
66+
67+
- do:
68+
allowed_warnings_regex:
69+
- "index \\[test-1\\] matches multiple legacy templates \\[global, test\\], composable templates will only match a single template"
70+
index:
71+
index: test-1
72+
id: 1
73+
refresh: true
74+
body:
75+
{ host.name: localhost, host.id: 1, time: 10, time.max: 100, time.min: 1 }
76+
77+
- do:
78+
field_caps:
79+
index: test-1
80+
fields: [host*, time*]
81+
- match: {fields.host\.name.keyword.searchable: true}
82+
- match: {fields.host\.name.keyword.aggregatable: true}
83+
- match: {fields.host\.id.long.searchable: true}
84+
- match: {fields.host\.id.long.aggregatable: true}
85+
- match: {fields.time.long.searchable: true}
86+
- match: {fields.time.long.aggregatable: true}
87+
- match: {fields.time\.max.long.searchable: true}
88+
- match: {fields.time\.max.long.aggregatable: true}
89+
- match: {fields.time\.min.long.searchable: true}
90+
- match: {fields.time\.min.long.aggregatable: true}

‎server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.action.bulk.BulkResponse;
1414
import org.elasticsearch.action.index.IndexRequest;
1515
import org.elasticsearch.action.index.IndexRequestBuilder;
16+
import org.elasticsearch.action.index.IndexResponse;
1617
import org.elasticsearch.action.search.SearchResponse;
1718
import org.elasticsearch.action.support.WriteRequest;
1819
import org.elasticsearch.cluster.ClusterState;
@@ -494,4 +495,77 @@ public void testDynamicRuntimeObjectFields() {
494495
e.getMessage()
495496
);
496497
}
498+
499+
public void testSubobjectsFalseAtRoot() {
500+
assertAcked(client().admin().indices().prepareCreate("test").setMapping("""
501+
{
502+
"_doc": {
503+
"subobjects" : false,
504+
"properties": {
505+
"host.name": {
506+
"type": "keyword"
507+
}
508+
}
509+
}
510+
}""").get());
511+
512+
IndexRequest request = new IndexRequest("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
513+
.source("host.name", "localhost", "host.id", 111, "time", 100, "time.max", 1000);
514+
IndexResponse indexResponse = client().index(request).actionGet();
515+
assertEquals(RestStatus.CREATED, indexResponse.status());
516+
517+
Map<String, Object> mappings = client().admin().indices().prepareGetMappings("test").get().mappings().get("test").getSourceAsMap();
518+
@SuppressWarnings("unchecked")
519+
Map<String, Object> properties = (Map<String, Object>) mappings.get("properties");
520+
assertEquals(4, properties.size());
521+
assertNotNull(properties.get("host.name"));
522+
assertNotNull(properties.get("host.id"));
523+
assertNotNull(properties.get("time"));
524+
assertNotNull(properties.get("time.max"));
525+
}
526+
527+
@SuppressWarnings("unchecked")
528+
public void testSubobjectsFalse() {
529+
assertAcked(client().admin().indices().prepareCreate("test").setMapping("""
530+
{
531+
"_doc": {
532+
"properties": {
533+
"foo.metrics" : {
534+
"subobjects" : false,
535+
"properties" : {
536+
"host.name": {
537+
"type": "keyword"
538+
}
539+
}
540+
}
541+
}
542+
}
543+
}""").get());
544+
545+
IndexRequest request = new IndexRequest("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
546+
.source(
547+
"foo.metrics.host.name",
548+
"localhost",
549+
"foo.metrics.host.id",
550+
111,
551+
"foo.metrics.time",
552+
100,
553+
"foo.metrics.time.max",
554+
1000
555+
);
556+
IndexResponse indexResponse = client().index(request).actionGet();
557+
assertEquals(RestStatus.CREATED, indexResponse.status());
558+
559+
Map<String, Object> mappings = client().admin().indices().prepareGetMappings("test").get().mappings().get("test").getSourceAsMap();
560+
Map<String, Object> properties = (Map<String, Object>) mappings.get("properties");
561+
Map<String, Object> foo = (Map<String, Object>) properties.get("foo");
562+
properties = (Map<String, Object>) foo.get("properties");
563+
Map<String, Object> metrics = (Map<String, Object>) properties.get("metrics");
564+
properties = (Map<String, Object>) metrics.get("properties");
565+
assertEquals(4, properties.size());
566+
assertNotNull(properties.get("host.name"));
567+
assertNotNull(properties.get("host.id"));
568+
assertNotNull(properties.get("time"));
569+
assertNotNull(properties.get("time.max"));
570+
}
497571
}

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ public class DocumentMapper {
2525
* @return the newly created document mapper
2626
*/
2727
public static DocumentMapper createEmpty(MapperService mapperService) {
28-
RootObjectMapper root = new RootObjectMapper.Builder(MapperService.SINGLE_MAPPING_NAME).build(MapperBuilderContext.ROOT);
28+
RootObjectMapper root = new RootObjectMapper.Builder(MapperService.SINGLE_MAPPING_NAME, ObjectMapper.Defaults.SUBOBJECTS).build(
29+
MapperBuilderContext.ROOT
30+
);
2931
MetadataFieldMapper[] metadata = mapperService.getMetadataMappers().values().toArray(new MetadataFieldMapper[0]);
3032
Mapping mapping = new Mapping(root, metadata, null);
3133
return new DocumentMapper(mapperService.documentParser(), mapping, mapping.toCompressedXContent());

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,8 @@ static Mapper createDynamicObjectMapper(DocumentParserContext context, String na
161161
Mapper mapper = createObjectMapperFromTemplate(context, name);
162162
return mapper != null
163163
? mapper
164-
: new ObjectMapper.Builder(name).enabled(ObjectMapper.Defaults.ENABLED).build(context.createMapperBuilderContext());
164+
: new ObjectMapper.Builder(name, ObjectMapper.Defaults.SUBOBJECTS).enabled(ObjectMapper.Defaults.ENABLED)
165+
.build(context.createMapperBuilderContext());
165166
}
166167

167168
/**

‎server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
public final class Mapping implements ToXContentFragment {
3333

3434
public static final Mapping EMPTY = new Mapping(
35-
new RootObjectMapper.Builder("_doc").build(MapperBuilderContext.ROOT),
35+
new RootObjectMapper.Builder("_doc", ObjectMapper.Defaults.SUBOBJECTS).build(MapperBuilderContext.ROOT),
3636
new MetadataFieldMapper[0],
3737
null
3838
);

‎server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public static class Builder extends ObjectMapper.Builder {
3333
private final Version indexCreatedVersion;
3434

3535
public Builder(String name, Version indexCreatedVersion) {
36-
super(name);
36+
super(name, Explicit.IMPLICIT_TRUE);
3737
this.indexCreatedVersion = indexCreatedVersion;
3838
}
3939

@@ -57,6 +57,9 @@ public static class TypeParser extends ObjectMapper.TypeParser {
5757
@Override
5858
public Mapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext)
5959
throws MapperParsingException {
60+
if (parseSubobjects(node).explicit()) {
61+
throw new MapperParsingException("Nested type [" + name + "] does not support [subobjects] parameter");
62+
}
6063
NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(name, parserContext.indexVersionCreated());
6164
parseNested(name, node, builder);
6265
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
@@ -67,9 +70,6 @@ public Mapper.Builder parse(String name, Map<String, Object> node, MappingParser
6770
iterator.remove();
6871
}
6972
}
70-
if (builder.subobjects.explicit()) {
71-
throw new MapperParsingException("Nested type [" + name + "] does not support [subobjects] parameter");
72-
}
7373
return builder;
7474
}
7575

‎server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public class ObjectMapper extends Mapper implements Cloneable {
3737

3838
public static class Defaults {
3939
public static final boolean ENABLED = true;
40+
public static final Explicit<Boolean> SUBOBJECTS = Explicit.IMPLICIT_TRUE;
4041
}
4142

4243
public enum Dynamic {
@@ -61,26 +62,21 @@ DynamicFieldsBuilder getDynamicFieldsBuilder() {
6162
}
6263

6364
public static class Builder extends Mapper.Builder {
64-
65+
protected final Explicit<Boolean> subobjects;
6566
protected Explicit<Boolean> enabled = Explicit.IMPLICIT_TRUE;
66-
protected Explicit<Boolean> subobjects = Explicit.IMPLICIT_TRUE;
6767
protected Dynamic dynamic;
6868
protected final List<Mapper.Builder> mappersBuilders = new ArrayList<>();
6969

70-
public Builder(String name) {
70+
public Builder(String name, Explicit<Boolean> subobjects) {
7171
super(name);
72+
this.subobjects = subobjects;
7273
}
7374

7475
public Builder enabled(boolean enabled) {
7576
this.enabled = Explicit.explicitBoolean(enabled);
7677
return this;
7778
}
7879

79-
public Builder subobjects(boolean subobjects) {
80-
this.subobjects = Explicit.explicitBoolean(subobjects);
81-
return this;
82-
}
83-
8480
public Builder dynamic(Dynamic dynamic) {
8581
this.dynamic = dynamic;
8682
return this;
@@ -186,7 +182,8 @@ public boolean supportsVersion(Version indexCreatedVersion) {
186182
@Override
187183
public Mapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext)
188184
throws MapperParsingException {
189-
ObjectMapper.Builder builder = new Builder(name);
185+
Explicit<Boolean> subobjects = parseSubobjects(node);
186+
ObjectMapper.Builder builder = new Builder(name, subobjects);
190187
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
191188
Map.Entry<String, Object> entry = iterator.next();
192189
String fieldName = entry.getKey();
@@ -219,9 +216,6 @@ protected static boolean parseObjectOrDocumentTypeProperties(
219216
} else if (fieldName.equals("enabled")) {
220217
builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled"));
221218
return true;
222-
} else if (fieldName.equals("subobjects")) {
223-
builder.subobjects(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".subobjects"));
224-
return true;
225219
} else if (fieldName.equals("properties")) {
226220
if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
227221
// nothing to do here, empty (to support "properties: []" case)
@@ -242,6 +236,14 @@ protected static boolean parseObjectOrDocumentTypeProperties(
242236
return false;
243237
}
244238

239+
protected static Explicit<Boolean> parseSubobjects(Map<String, Object> node) {
240+
Object subobjectsNode = node.remove("subobjects");
241+
if (subobjectsNode != null) {
242+
return Explicit.explicitBoolean(XContentMapValues.nodeBooleanValue(subobjectsNode, "subobjects.subobjects"));
243+
}
244+
return Explicit.IMPLICIT_TRUE;
245+
}
246+
245247
protected static void parseProperties(
246248
ObjectMapper.Builder objBuilder,
247249
Map<String, Object> propsNode,
@@ -298,7 +300,7 @@ protected static void parseProperties(
298300
String realFieldName = fieldNameParts[fieldNameParts.length - 1];
299301
fieldBuilder = typeParser.parse(realFieldName, propNode, parserContext);
300302
for (int i = fieldNameParts.length - 2; i >= 0; --i) {
301-
ObjectMapper.Builder intermediate = new ObjectMapper.Builder(fieldNameParts[i]);
303+
ObjectMapper.Builder intermediate = new ObjectMapper.Builder(fieldNameParts[i], Defaults.SUBOBJECTS);
302304
intermediate.add(fieldBuilder);
303305
fieldBuilder = intermediate;
304306
}
@@ -367,9 +369,8 @@ protected ObjectMapper clone() {
367369
* @return a Builder that will produce an empty ObjectMapper with the same configuration as this one
368370
*/
369371
public ObjectMapper.Builder newBuilder(Version indexVersionCreated) {
370-
ObjectMapper.Builder builder = new ObjectMapper.Builder(simpleName());
372+
ObjectMapper.Builder builder = new ObjectMapper.Builder(simpleName(), subobjects);
371373
builder.enabled = this.enabled;
372-
builder.subobjects = this.subobjects;
373374
builder.dynamic = this.dynamic;
374375
return builder;
375376
}
@@ -515,7 +516,7 @@ void toXContent(XContentBuilder builder, Params params, ToXContent custom) throw
515516
if (isEnabled() != Defaults.ENABLED) {
516517
builder.field("enabled", enabled.value());
517518
}
518-
if (subobjects() == false) {
519+
if (subobjects != Defaults.SUBOBJECTS) {
519520
builder.field("subobjects", subobjects.value());
520521
}
521522
if (custom != null) {

‎server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,8 @@ public static class Builder extends ObjectMapper.Builder {
7070
protected Explicit<Boolean> dateDetection = Defaults.DATE_DETECTION;
7171
protected Explicit<Boolean> numericDetection = Defaults.NUMERIC_DETECTION;
7272

73-
public Builder(String name) {
74-
super(name);
73+
public Builder(String name, Explicit<Boolean> subobjects) {
74+
super(name, subobjects);
7575
}
7676

7777
public Builder dynamicDateTimeFormatter(Collection<DateFormatter> dateTimeFormatters) {
@@ -152,7 +152,8 @@ static final class TypeParser extends ObjectMapper.TypeParser {
152152
@Override
153153
public RootObjectMapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext)
154154
throws MapperParsingException {
155-
RootObjectMapper.Builder builder = new Builder(name);
155+
Explicit<Boolean> subobjects = parseSubobjects(node);
156+
RootObjectMapper.Builder builder = new Builder(name, subobjects);
156157
Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
157158
while (iterator.hasNext()) {
158159
Map.Entry<String, Object> entry = iterator.next();
@@ -278,9 +279,8 @@ protected ObjectMapper clone() {
278279

279280
@Override
280281
public RootObjectMapper.Builder newBuilder(Version indexVersionCreated) {
281-
RootObjectMapper.Builder builder = new RootObjectMapper.Builder(name());
282+
RootObjectMapper.Builder builder = new RootObjectMapper.Builder(name(), subobjects);
282283
builder.enabled = enabled;
283-
builder.subobjects = subobjects;
284284
builder.dynamic = dynamic;
285285
return builder;
286286
}

‎server/src/test/java/org/elasticsearch/cluster/action/index/MappingUpdatedActionTests.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.elasticsearch.index.mapper.MapperBuilderContext;
2727
import org.elasticsearch.index.mapper.Mapping;
2828
import org.elasticsearch.index.mapper.MetadataFieldMapper;
29+
import org.elasticsearch.index.mapper.ObjectMapper;
2930
import org.elasticsearch.index.mapper.RootObjectMapper;
3031
import org.elasticsearch.test.ESTestCase;
3132

@@ -146,7 +147,9 @@ public void testSendUpdateMappingUsingAutoPutMappingAction() {
146147
);
147148
mua.setClient(client);
148149

149-
RootObjectMapper rootObjectMapper = new RootObjectMapper.Builder("name").build(MapperBuilderContext.ROOT);
150+
RootObjectMapper rootObjectMapper = new RootObjectMapper.Builder("name", ObjectMapper.Defaults.SUBOBJECTS).build(
151+
MapperBuilderContext.ROOT
152+
);
150153
Mapping update = new Mapping(rootObjectMapper, new MetadataFieldMapper[0], Map.of());
151154

152155
mua.sendUpdateMapping(new Index("name", "uuid"), update, ActionListener.noop());

0 commit comments

Comments
 (0)