Skip to content

Commit 57e4d10

Browse files
mayya-sharipovamayya-sharipova
authored
Limit the number of nested documents (#27405)
Add an index level setting `index.mapping.nested_objects.limit` to control the number of nested json objects that can be in a single document across all fields. Defaults to 10000. Throw an error if the number of created nested documents exceed this limit during the parsing of a document. Closes #26962
1 parent 4cffe8f commit 57e4d10

File tree

8 files changed

+222
-2
lines changed

8 files changed

+222
-2
lines changed

‎core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
141141
Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING,
142142
MapperService.INDEX_MAPPER_DYNAMIC_SETTING,
143143
MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
144+
MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING,
144145
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
145146
MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING,
146147
BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,

‎core/src/main/java/org/elasticsearch/index/mapper/MapperService.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,9 @@ public enum MergeReason {
9292
public static final String DEFAULT_MAPPING = "_default_";
9393
public static final Setting<Long> INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING =
9494
Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope);
95+
// maximum allowed number of nested json objects across all fields in a single document
96+
public static final Setting<Long> INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING =
97+
Setting.longSetting("index.mapping.nested_objects.limit", 10000L, 0, Property.Dynamic, Property.IndexScope);
9598
public static final Setting<Long> INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING =
9699
Setting.longSetting("index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope);
97100
public static final Setting<Long> INDEX_MAPPING_DEPTH_LIMIT_SETTING =

‎core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,10 @@ public static class InternalParseContext extends ParseContext {
305305

306306
private SeqNoFieldMapper.SequenceIDFields seqID;
307307

308+
private final long maxAllowedNumNestedDocs;
309+
310+
private long numNestedDocs;
311+
308312

309313
private final List<Mapper> dynamicMappers;
310314

@@ -321,6 +325,8 @@ public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperPars
321325
this.version = null;
322326
this.sourceToParse = source;
323327
this.dynamicMappers = new ArrayList<>();
328+
this.maxAllowedNumNestedDocs = MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.get(indexSettings);
329+
this.numNestedDocs = 0L;
324330
}
325331

326332
@Override
@@ -366,6 +372,13 @@ public Document doc() {
366372

367373
@Override
368374
protected void addDoc(Document doc) {
375+
numNestedDocs ++;
376+
if (numNestedDocs > maxAllowedNumNestedDocs) {
377+
throw new MapperParsingException(
378+
"The number of nested documents has exceeded the allowed limit of [" + maxAllowedNumNestedDocs + "]."
379+
+ " This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey()
380+
+ "] index level setting.");
381+
}
369382
this.documents.add(doc);
370383
}
371384

‎core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java

Lines changed: 141 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@
1919

2020
package org.elasticsearch.index.mapper;
2121

22-
import java.util.HashMap;
2322
import java.util.HashSet;
2423
import org.apache.lucene.index.IndexableField;
2524
import org.elasticsearch.Version;
2625
import org.elasticsearch.common.compress.CompressedXContent;
2726
import org.elasticsearch.common.settings.Settings;
27+
import org.elasticsearch.common.xcontent.XContentBuilder;
2828
import org.elasticsearch.common.xcontent.XContentFactory;
2929
import org.elasticsearch.common.xcontent.XContentType;
3030
import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -524,4 +524,144 @@ public void testParentObjectMapperAreNested() throws Exception {
524524
assertFalse(objectMapper.parentObjectMapperAreNested(mapperService));
525525
}
526526

527+
public void testLimitNestedDocsDefaultSettings() throws Exception{
528+
Settings settings = Settings.builder().build();
529+
MapperService mapperService = createIndex("test1", settings).mapperService();
530+
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
531+
.startObject("nested1").field("type", "nested").endObject()
532+
.endObject().endObject().endObject().string();
533+
DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping));
534+
long defaultMaxNoNestedDocs = MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.get(settings);
535+
536+
// parsing a doc with No. nested objects > defaultMaxNoNestedDocs fails
537+
XContentBuilder docBuilder = XContentFactory.jsonBuilder();
538+
docBuilder.startObject();
539+
{
540+
docBuilder.startArray("nested1");
541+
{
542+
for(int i = 0; i <= defaultMaxNoNestedDocs; i++) {
543+
docBuilder.startObject().field("f", i).endObject();
544+
}
545+
}
546+
docBuilder.endArray();
547+
}
548+
docBuilder.endObject();
549+
SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON);
550+
MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source1));
551+
assertEquals(
552+
"The number of nested documents has exceeded the allowed limit of [" + defaultMaxNoNestedDocs
553+
+ "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey()
554+
+ "] index level setting.",
555+
e.getMessage()
556+
);
557+
}
558+
559+
public void testLimitNestedDocs() throws Exception{
560+
// setting limit to allow only two nested objects in the whole doc
561+
long maxNoNestedDocs = 2L;
562+
MapperService mapperService = createIndex("test1", Settings.builder()
563+
.put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), maxNoNestedDocs).build()).mapperService();
564+
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
565+
.startObject("nested1").field("type", "nested").endObject()
566+
.endObject().endObject().endObject().string();
567+
DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping));
568+
569+
// parsing a doc with 2 nested objects succeeds
570+
XContentBuilder docBuilder = XContentFactory.jsonBuilder();
571+
docBuilder.startObject();
572+
{
573+
docBuilder.startArray("nested1");
574+
{
575+
docBuilder.startObject().field("field1", "11").field("field2", "21").endObject();
576+
docBuilder.startObject().field("field1", "12").field("field2", "22").endObject();
577+
}
578+
docBuilder.endArray();
579+
}
580+
docBuilder.endObject();
581+
SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON);
582+
ParsedDocument doc = docMapper.parse(source1);
583+
assertThat(doc.docs().size(), equalTo(3));
584+
585+
// parsing a doc with 3 nested objects fails
586+
XContentBuilder docBuilder2 = XContentFactory.jsonBuilder();
587+
docBuilder2.startObject();
588+
{
589+
docBuilder2.startArray("nested1");
590+
{
591+
docBuilder2.startObject().field("field1", "11").field("field2", "21").endObject();
592+
docBuilder2.startObject().field("field1", "12").field("field2", "22").endObject();
593+
docBuilder2.startObject().field("field1", "13").field("field2", "23").endObject();
594+
}
595+
docBuilder2.endArray();
596+
}
597+
docBuilder2.endObject();
598+
SourceToParse source2 = SourceToParse.source("test1", "type", "2", docBuilder2.bytes(), XContentType.JSON);
599+
MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2));
600+
assertEquals(
601+
"The number of nested documents has exceeded the allowed limit of [" + maxNoNestedDocs
602+
+ "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey()
603+
+ "] index level setting.",
604+
e.getMessage()
605+
);
606+
}
607+
608+
public void testLimitNestedDocsMultipleNestedFields() throws Exception{
609+
// setting limit to allow only two nested objects in the whole doc
610+
long maxNoNestedDocs = 2L;
611+
MapperService mapperService = createIndex("test1", Settings.builder()
612+
.put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), maxNoNestedDocs).build()).mapperService();
613+
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
614+
.startObject("nested1").field("type", "nested").endObject()
615+
.startObject("nested2").field("type", "nested").endObject()
616+
.endObject().endObject().endObject().string();
617+
DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping));
618+
619+
// parsing a doc with 2 nested objects succeeds
620+
XContentBuilder docBuilder = XContentFactory.jsonBuilder();
621+
docBuilder.startObject();
622+
{
623+
docBuilder.startArray("nested1");
624+
{
625+
docBuilder.startObject().field("field1", "11").field("field2", "21").endObject();
626+
}
627+
docBuilder.endArray();
628+
docBuilder.startArray("nested2");
629+
{
630+
docBuilder.startObject().field("field1", "21").field("field2", "22").endObject();
631+
}
632+
docBuilder.endArray();
633+
}
634+
docBuilder.endObject();
635+
SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON);
636+
ParsedDocument doc = docMapper.parse(source1);
637+
assertThat(doc.docs().size(), equalTo(3));
638+
639+
// parsing a doc with 3 nested objects fails
640+
XContentBuilder docBuilder2 = XContentFactory.jsonBuilder();
641+
docBuilder2.startObject();
642+
{
643+
docBuilder2.startArray("nested1");
644+
{
645+
docBuilder2.startObject().field("field1", "11").field("field2", "21").endObject();
646+
}
647+
docBuilder2.endArray();
648+
docBuilder2.startArray("nested2");
649+
{
650+
docBuilder2.startObject().field("field1", "12").field("field2", "22").endObject();
651+
docBuilder2.startObject().field("field1", "13").field("field2", "23").endObject();
652+
}
653+
docBuilder2.endArray();
654+
655+
}
656+
docBuilder2.endObject();
657+
SourceToParse source2 = SourceToParse.source("test1", "type", "2", docBuilder2.bytes(), XContentType.JSON);
658+
MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2));
659+
assertEquals(
660+
"The number of nested documents has exceeded the allowed limit of [" + maxNoNestedDocs
661+
+ "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey()
662+
+ "] index level setting.",
663+
e.getMessage()
664+
);
665+
}
666+
527667
}

‎docs/reference/mapping.asciidoc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,12 @@ causing a mapping explosion:
9090
Indexing 1 document with 100 nested fields actually indexes 101 documents
9191
as each nested document is indexed as a separate hidden document.
9292

93+
`index.mapping.nested_objects.limit`::
94+
The maximum number of `nested` json objects within a single document across
95+
all nested fields, defaults to 10000. Indexing one document with an array of
96+
100 objects within a nested field, will actually create 101 documents, as
97+
each nested object will be indexed as a separate hidden document.
98+
9399

94100
[float]
95101
== Dynamic mapping

‎docs/reference/mapping/types/nested.asciidoc

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,3 +201,13 @@ Indexing a document with 100 nested fields actually indexes 101 documents as eac
201201
document is indexed as a separate document. To safeguard against ill-defined mappings
202202
the number of nested fields that can be defined per index has been limited to 50. See
203203
<<mapping-limit-settings>>.
204+
205+
206+
==== Limiting the number of `nested` json objects
207+
Indexing a document with an array of 100 objects within a nested field, will actually
208+
create 101 documents, as each nested object will be indexed as a separate document.
209+
To prevent out of memory errors when a single document contains too many nested json
210+
objects, the number of nested json objects that a single document may contain across all fields
211+
has been limited to 10000. See <<mapping-limit-settings>>.
212+
213+

‎docs/reference/migration/migrate_7_0/mappings.asciidoc

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,10 @@ The `_all` field deprecated in 6 have now been removed.
77

88
==== `index_options` for numeric fields has been removed
99

10-
The `index_options` field for numeric fields has been deprecated in 6 and has now been removed.
10+
The `index_options` field for numeric fields has been deprecated in 6 and has now been removed.
11+
12+
==== Limiting the number of `nested` json objects
13+
14+
To safeguard against out of memory errors, the number of nested json objects within a single
15+
document across all fields has been limited to 10000. This default limit can be changed with
16+
the index setting `index.mapping.nested_objects.limit`.

‎rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
---
2+
setup:
3+
- do:
4+
indices.create:
5+
index: test_1
6+
body:
7+
settings:
8+
index.mapping.nested_objects.limit: 2
9+
mappings:
10+
test_type:
11+
properties:
12+
nested1:
13+
type: nested
14+
15+
---
16+
"Indexing a doc with No. nested objects less or equal to index.mapping.nested_objects.limit should succeed":
17+
- skip:
18+
version: " - 6.99.99"
19+
reason: index.mapping.nested_objects setting has been added in 7.0.0
20+
- do:
21+
create:
22+
index: test_1
23+
type: test_type
24+
id: 1
25+
body:
26+
"nested1" : [ { "foo": "bar" }, { "foo": "bar2" } ]
27+
- match: { _version: 1}
28+
29+
---
30+
"Indexing a doc with No. nested objects more than index.mapping.nested_objects.limit should fail":
31+
- skip:
32+
version: " - 6.99.99"
33+
reason: index.mapping.nested_objects setting has been added in 7.0.0
34+
- do:
35+
catch: /The number of nested documents has exceeded the allowed limit of \[2\]. This limit can be set by changing the \[index.mapping.nested_objects.limit\] index level setting\./
36+
create:
37+
index: test_1
38+
type: test_type
39+
id: 1
40+
body:
41+
"nested1" : [ { "foo": "bar" }, { "foo": "bar2" }, { "foo": "bar3" } ]

0 commit comments

Comments
 (0)