From d59543a8808c4f8315f06195d73dea3583ba1fff Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 15 Nov 2017 12:44:41 -0500 Subject: [PATCH 1/4] Limit the number of nested documents Add an index level setting `index.mapping.nested_objects.limit` to control the number of nested json objects that can be in a single document across all fields. Defaults to 10000. Throw an error if the number of created nested documents exceed this limit during the parsing of a document. Closes #26962 --- .../common/settings/IndexScopedSettings.java | 1 + .../index/mapper/MapperService.java | 3 + .../index/mapper/ParseContext.java | 13 +++ .../index/mapper/NestedObjectMapperTests.java | 92 ++++++++++++++++++- docs/reference/mapping.asciidoc | 6 ++ docs/reference/mapping/types/nested.asciidoc | 10 ++ .../migration/migrate_7_0/mappings.asciidoc | 8 +- 7 files changed, 131 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index d40488eaa34f8..b3b0fdb8cf850 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -141,6 +141,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING, MapperService.INDEX_MAPPER_DYNAMIC_SETTING, MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 93d7f4b13380a..b21f47d8feb9b 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -92,6 +92,9 @@ public enum MergeReason { public static final String DEFAULT_MAPPING = "_default_"; public static final Setting INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING = Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope); + // maximum allowed number of nested json objects across all fields in a single document + public static final Setting INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING = + Setting.longSetting("index.mapping.nested_objects.limit", 10000L, 0, Property.Dynamic, Property.IndexScope); public static final Setting INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING = Setting.longSetting("index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope); public static final Setting INDEX_MAPPING_DEPTH_LIMIT_SETTING = diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java index c6ed6b315a015..a700acfb52cd3 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java @@ -305,6 +305,10 @@ public static class InternalParseContext extends ParseContext { private SeqNoFieldMapper.SequenceIDFields seqID; + private final Long maxAllowedNumNestedDocs; + + private Long numNestedDocs; + private final List dynamicMappers; @@ -321,6 +325,8 @@ public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperPars this.version = null; this.sourceToParse = source; this.dynamicMappers = new ArrayList<>(); + this.maxAllowedNumNestedDocs = indexSettings.getAsLong(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), 10000L); + this.numNestedDocs = 0L; } @Override @@ -366,6 +372,13 @@ public Document doc() { @Override protected void addDoc(Document doc) { + numNestedDocs ++; + if (numNestedDocs > maxAllowedNumNestedDocs) { + throw new MapperParsingException( + "The number of nested documents has exceeded the allowed limit of [" + maxAllowedNumNestedDocs + "]." + + " This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + + "] index level setting."); + } this.documents.add(doc); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 39d4de2359e78..038cb1a93ff34 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import java.util.HashMap; import java.util.HashSet; import org.apache.lucene.index.IndexableField; import org.elasticsearch.Version; @@ -524,4 +523,95 @@ public void testParentObjectMapperAreNested() throws Exception { assertFalse(objectMapper.parentObjectMapperAreNested(mapperService)); } + public void testLimitNestedDocs() throws Exception{ + // setting limit to allow only two nested objects in the whole doc + Long nested_docs_limit = 2L; + MapperService mapperService = createIndex("test1", Settings.builder() + .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), nested_docs_limit).build()).mapperService(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("nested1").field("type", "nested").endObject() + .endObject().endObject().endObject().string(); + DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + + // parsing a doc with 2 nested objects succeeds + SourceToParse source1 = SourceToParse.source("test1", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .startArray("nested1") + .startObject().field("field1", "11").field("field2", "21").endObject() + .startObject().field("field1", "12").field("field2", "22").endObject() + .endArray() + .endObject() + .bytes(), + XContentType.JSON); + ParsedDocument doc = docMapper.parse(source1); + assertThat(doc.docs().size(), equalTo(3)); + + // parsing a doc with 3 nested objects fails + SourceToParse source2 = SourceToParse.source("test1", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .startArray("nested1") + .startObject().field("field1", "f11").field("field2", "f21").endObject() + .startObject().field("field1", "f12").field("field2", "f22").endObject() + .startObject().field("field1", "f13").field("field2", "f23").endObject() + .endArray() + .endObject() + .bytes(), + XContentType.JSON); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2)); + assertEquals( + "The number of nested documents has exceeded the allowed limit of [" + nested_docs_limit + + "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + + "] index level setting.", + e.getMessage() + ); + } + + public void testLimitNestedDocsMultipleNestedFields() throws Exception{ + // setting limit to allow only two nested objects in the whole doc + Long nested_docs_limit = 2L; + MapperService mapperService = createIndex("test1", Settings.builder() + .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), nested_docs_limit).build()).mapperService(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("nested1").field("type", "nested").endObject() + .startObject("nested2").field("type", "nested").endObject() + .endObject().endObject().endObject().string(); + DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + + // parsing a doc with 2 nested objects succeeds + SourceToParse source1 = SourceToParse.source("test1", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .startArray("nested1") + .startObject().field("field1", "11").field("field2", "21").endObject() + .endArray() + .startArray("nested2") + .startObject().field("field1", "21").field("field2", "22").endObject() + .endArray() + .endObject() + .bytes(), + XContentType.JSON); + ParsedDocument doc = docMapper.parse(source1); + assertThat(doc.docs().size(), equalTo(3)); + + // parsing a doc with 3 nested objects fails + SourceToParse source2 = SourceToParse.source("test1", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .startArray("nested1") + .startObject().field("field1", "f11").field("field2", "f21").endObject() + .endArray() + .startArray("nested2") + .startObject().field("field1", "f12").field("field2", "f22").endObject() + .startObject().field("field1", "f13").field("field2", "f23").endObject() + .endArray() + .endObject() + .bytes(), + XContentType.JSON); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2)); + assertEquals( + "The number of nested documents has exceeded the allowed limit of [" + nested_docs_limit + + "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + + "] index level setting.", + e.getMessage() + ); + } + } diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc index bc5c1def32f68..6f8f1b38d6f22 100644 --- a/docs/reference/mapping.asciidoc +++ b/docs/reference/mapping.asciidoc @@ -90,6 +90,12 @@ causing a mapping explosion: Indexing 1 document with 100 nested fields actually indexes 101 documents as each nested document is indexed as a separate hidden document. +`index.mapping.nested_objects.limit`:: + The maximum number of `nested` json objects within a single document across + all nested fields, defaults to 10000. Indexing one document with an array of + 100 objects within a nested field, will actually create 101 documents, as + each nested object will be indexed as a separate hidden document. + [float] == Dynamic mapping diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc index b5b0d3394eafe..2cbae7383232f 100644 --- a/docs/reference/mapping/types/nested.asciidoc +++ b/docs/reference/mapping/types/nested.asciidoc @@ -201,3 +201,13 @@ Indexing a document with 100 nested fields actually indexes 101 documents as eac document is indexed as a separate document. To safeguard against ill-defined mappings the number of nested fields that can be defined per index has been limited to 50. See <>. + + +==== Limiting the number of `nested` json objects +Indexing a document with an array of 100 objects within a nested field, will actually +create 101 documents, as each nested object will be indexed as a separate document. +To prevent out of memory errors when a single document contains too many nested json +objects, the number of nested json objects that a single document may contain has been +limited to 10000. See <>. + + diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc index 4f8893829f9c7..7ef5348d053a1 100644 --- a/docs/reference/migration/migrate_7_0/mappings.asciidoc +++ b/docs/reference/migration/migrate_7_0/mappings.asciidoc @@ -7,4 +7,10 @@ The `_all` field deprecated in 6 have now been removed. ==== `index_options` for numeric fields has been removed -The `index_options` field for numeric fields has been deprecated in 6 and has now been removed. \ No newline at end of file +The `index_options` field for numeric fields has been deprecated in 6 and has now been removed. + +==== Limiting the number of `nested` json objects + +To safeguard against out of memory errors, the number of nested json objects within a single +document has been limited to 10000. This default limit can be changed with the index setting +`index.mapping.nested_objects.limit`. \ No newline at end of file From a433f7d0f23ab01b7b9204d7e1919b81f5b14258 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 20 Nov 2017 17:15:25 -0500 Subject: [PATCH 2/4] Limit the number of nested documents Add an index level setting `index.mapping.nested_objects.limit` to control the number of nested json objects that can be in a single document across all fields. Defaults to 10000. Throw an error if the number of created nested documents exceed this limit during the parsing of a document. Closes #26962 --- .../index/mapper/ParseContext.java | 6 +- .../index/mapper/NestedObjectMapperTests.java | 131 ++++++++++++------ docs/reference/mapping/types/nested.asciidoc | 4 +- .../migration/migrate_7_0/mappings.asciidoc | 4 +- .../rest-api-spec/test/create/70_nested.yml | 41 ++++++ 5 files changed, 135 insertions(+), 51 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java index a700acfb52cd3..4a7af6cba47ea 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java @@ -305,9 +305,9 @@ public static class InternalParseContext extends ParseContext { private SeqNoFieldMapper.SequenceIDFields seqID; - private final Long maxAllowedNumNestedDocs; + private final long maxAllowedNumNestedDocs; - private Long numNestedDocs; + private long numNestedDocs; private final List dynamicMappers; @@ -325,7 +325,7 @@ public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperPars this.version = null; this.sourceToParse = source; this.dynamicMappers = new ArrayList<>(); - this.maxAllowedNumNestedDocs = indexSettings.getAsLong(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), 10000L); + this.maxAllowedNumNestedDocs = MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.get(indexSettings); this.numNestedDocs = 0L; } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 038cb1a93ff34..baa155e008c8c 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -523,9 +524,34 @@ public void testParentObjectMapperAreNested() throws Exception { assertFalse(objectMapper.parentObjectMapperAreNested(mapperService)); } + public void testLimitNestedDocsDefaultSettings() throws Exception{ + MapperService mapperService = createIndex("test1", Settings.builder().build()).mapperService(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("nested1").field("type", "nested").endObject() + .endObject().endObject().endObject().string(); + DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + + // parsing a doc with 3 nested objects succeeds + XContentBuilder docBuilder = XContentFactory.jsonBuilder(); + docBuilder.startObject(); + { + docBuilder.startArray("nested1"); + { + docBuilder.startObject().field("field1", "11").field("field2", "21").endObject(); + docBuilder.startObject().field("field1", "12").field("field2", "22").endObject(); + docBuilder.startObject().field("field1", "13").field("field2", "23").endObject(); + } + docBuilder.endArray(); + } + docBuilder.endObject(); + SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON); + ParsedDocument doc = docMapper.parse(source1); + assertThat(doc.docs().size(), equalTo(4)); + } + public void testLimitNestedDocs() throws Exception{ // setting limit to allow only two nested objects in the whole doc - Long nested_docs_limit = 2L; + long nested_docs_limit = 2L; MapperService mapperService = createIndex("test1", Settings.builder() .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), nested_docs_limit).build()).mapperService(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") @@ -534,29 +560,35 @@ public void testLimitNestedDocs() throws Exception{ DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); // parsing a doc with 2 nested objects succeeds - SourceToParse source1 = SourceToParse.source("test1", "type", "1", XContentFactory.jsonBuilder() - .startObject() - .startArray("nested1") - .startObject().field("field1", "11").field("field2", "21").endObject() - .startObject().field("field1", "12").field("field2", "22").endObject() - .endArray() - .endObject() - .bytes(), - XContentType.JSON); + XContentBuilder docBuilder = XContentFactory.jsonBuilder(); + docBuilder.startObject(); + { + docBuilder.startArray("nested1"); + { + docBuilder.startObject().field("field1", "11").field("field2", "21").endObject(); + docBuilder.startObject().field("field1", "12").field("field2", "22").endObject(); + } + docBuilder.endArray(); + } + docBuilder.endObject(); + SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON); ParsedDocument doc = docMapper.parse(source1); assertThat(doc.docs().size(), equalTo(3)); // parsing a doc with 3 nested objects fails - SourceToParse source2 = SourceToParse.source("test1", "type", "1", XContentFactory.jsonBuilder() - .startObject() - .startArray("nested1") - .startObject().field("field1", "f11").field("field2", "f21").endObject() - .startObject().field("field1", "f12").field("field2", "f22").endObject() - .startObject().field("field1", "f13").field("field2", "f23").endObject() - .endArray() - .endObject() - .bytes(), - XContentType.JSON); + XContentBuilder docBuilder2 = XContentFactory.jsonBuilder(); + docBuilder2.startObject(); + { + docBuilder2.startArray("nested1"); + { + docBuilder2.startObject().field("field1", "11").field("field2", "21").endObject(); + docBuilder2.startObject().field("field1", "12").field("field2", "22").endObject(); + docBuilder2.startObject().field("field1", "13").field("field2", "23").endObject(); + } + docBuilder2.endArray(); + } + docBuilder2.endObject(); + SourceToParse source2 = SourceToParse.source("test1", "type", "2", docBuilder2.bytes(), XContentType.JSON); MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2)); assertEquals( "The number of nested documents has exceeded the allowed limit of [" + nested_docs_limit @@ -568,7 +600,7 @@ public void testLimitNestedDocs() throws Exception{ public void testLimitNestedDocsMultipleNestedFields() throws Exception{ // setting limit to allow only two nested objects in the whole doc - Long nested_docs_limit = 2L; + long nested_docs_limit = 2L; MapperService mapperService = createIndex("test1", Settings.builder() .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), nested_docs_limit).build()).mapperService(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") @@ -578,33 +610,44 @@ public void testLimitNestedDocsMultipleNestedFields() throws Exception{ DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); // parsing a doc with 2 nested objects succeeds - SourceToParse source1 = SourceToParse.source("test1", "type", "1", XContentFactory.jsonBuilder() - .startObject() - .startArray("nested1") - .startObject().field("field1", "11").field("field2", "21").endObject() - .endArray() - .startArray("nested2") - .startObject().field("field1", "21").field("field2", "22").endObject() - .endArray() - .endObject() - .bytes(), - XContentType.JSON); + XContentBuilder docBuilder = XContentFactory.jsonBuilder(); + docBuilder.startObject(); + { + docBuilder.startArray("nested1"); + { + docBuilder.startObject().field("field1", "11").field("field2", "21").endObject(); + } + docBuilder.endArray(); + docBuilder.startArray("nested2"); + { + docBuilder.startObject().field("field1", "21").field("field2", "22").endObject(); + } + docBuilder.endArray(); + } + docBuilder.endObject(); + SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON); ParsedDocument doc = docMapper.parse(source1); assertThat(doc.docs().size(), equalTo(3)); // parsing a doc with 3 nested objects fails - SourceToParse source2 = SourceToParse.source("test1", "type", "1", XContentFactory.jsonBuilder() - .startObject() - .startArray("nested1") - .startObject().field("field1", "f11").field("field2", "f21").endObject() - .endArray() - .startArray("nested2") - .startObject().field("field1", "f12").field("field2", "f22").endObject() - .startObject().field("field1", "f13").field("field2", "f23").endObject() - .endArray() - .endObject() - .bytes(), - XContentType.JSON); + XContentBuilder docBuilder2 = XContentFactory.jsonBuilder(); + docBuilder2.startObject(); + { + docBuilder2.startArray("nested1"); + { + docBuilder2.startObject().field("field1", "11").field("field2", "21").endObject(); + } + docBuilder2.endArray(); + docBuilder2.startArray("nested2"); + { + docBuilder2.startObject().field("field1", "12").field("field2", "22").endObject(); + docBuilder2.startObject().field("field1", "13").field("field2", "23").endObject(); + } + docBuilder2.endArray(); + + } + docBuilder2.endObject(); + SourceToParse source2 = SourceToParse.source("test1", "type", "2", docBuilder2.bytes(), XContentType.JSON); MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2)); assertEquals( "The number of nested documents has exceeded the allowed limit of [" + nested_docs_limit diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc index 2cbae7383232f..b5d96513cf745 100644 --- a/docs/reference/mapping/types/nested.asciidoc +++ b/docs/reference/mapping/types/nested.asciidoc @@ -207,7 +207,7 @@ the number of nested fields that can be defined per index has been limited to 50 Indexing a document with an array of 100 objects within a nested field, will actually create 101 documents, as each nested object will be indexed as a separate document. To prevent out of memory errors when a single document contains too many nested json -objects, the number of nested json objects that a single document may contain has been -limited to 10000. See <>. +objects, the number of nested json objects that a single document may contain across all fields +has been limited to 10000. See <>. diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc index 7ef5348d053a1..215282c49d7bd 100644 --- a/docs/reference/migration/migrate_7_0/mappings.asciidoc +++ b/docs/reference/migration/migrate_7_0/mappings.asciidoc @@ -12,5 +12,5 @@ The `index_options` field for numeric fields has been deprecated in 6 and has n ==== Limiting the number of `nested` json objects To safeguard against out of memory errors, the number of nested json objects within a single -document has been limited to 10000. This default limit can be changed with the index setting -`index.mapping.nested_objects.limit`. \ No newline at end of file +document across all fields has been limited to 10000. This default limit can be changed with +the index setting `index.mapping.nested_objects.limit`. \ No newline at end of file diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml new file mode 100644 index 0000000000000..1627dca4d15fd --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml @@ -0,0 +1,41 @@ +--- +setup: + - do: + indices.create: + index: test_1 + body: + settings: + index.mapping.nested_objects.limit: 2 + mappings: + test_type: + properties: + nested1: + type: nested + +--- +"Indexing a doc with No. nested objects less or equal to index.mapping.nested_objects.limit should succeed": + - skip: + version: " - 6.99.99" + reason: index.mapping.nested_objects setting was introduced from this version + - do: + create: + index: test_1 + type: test_type + id: 1 + body: + "nested1" : [ { "foo": "bar" }, { "foo": "bar2" } ] + - match: { _version: 1} + +--- +"Indexing a doc with No. nested objects more than index.mapping.nested_objects.limit should fail": + - skip: + version: " - 6.99.99" + reason: index.mapping.nested_objects setting was introduced from this version + - do: + catch: /The number of nested documents has exceeded the allowed limit of \[2\]. This limit can be set by changing the \[index.mapping.nested_objects.limit\] index level setting\./ + create: + index: test_1 + type: test_type + id: 1 + body: + "nested1" : [ { "foo": "bar" }, { "foo": "bar2" }, { "foo": "bar3" } ] From e45f7bb456aff4d0861e3915e88821fca03940c7 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 21 Nov 2017 14:37:02 -0500 Subject: [PATCH 3/4] Limit the number of nested documents Add an index level setting `index.mapping.nested_objects.limit` to control the number of nested json objects that can be in a single document across all fields. Defaults to 10000. Throw an error if the number of created nested documents exceed this limit during the parsing of a document. Closes #26962 --- .../index/mapper/NestedObjectMapperTests.java | 33 +++++++++++-------- .../rest-api-spec/test/create/70_nested.yml | 4 +-- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index baa155e008c8c..eb79b243f57de 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -525,35 +525,42 @@ public void testParentObjectMapperAreNested() throws Exception { } public void testLimitNestedDocsDefaultSettings() throws Exception{ - MapperService mapperService = createIndex("test1", Settings.builder().build()).mapperService(); + Settings settings = Settings.builder().build(); + MapperService mapperService = createIndex("test1", settings).mapperService(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").endObject() .endObject().endObject().endObject().string(); DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + long defaultMaxNoNestedDocs = MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.get(settings); - // parsing a doc with 3 nested objects succeeds + // parsing a doc with No. nested objects > defaultMaxNoNestedDocs fails XContentBuilder docBuilder = XContentFactory.jsonBuilder(); docBuilder.startObject(); { docBuilder.startArray("nested1"); { - docBuilder.startObject().field("field1", "11").field("field2", "21").endObject(); - docBuilder.startObject().field("field1", "12").field("field2", "22").endObject(); - docBuilder.startObject().field("field1", "13").field("field2", "23").endObject(); + for(int i=0; i<=defaultMaxNoNestedDocs; i++) { + docBuilder.startObject().field("f", i).endObject(); + } } docBuilder.endArray(); } docBuilder.endObject(); SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON); - ParsedDocument doc = docMapper.parse(source1); - assertThat(doc.docs().size(), equalTo(4)); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source1)); + assertEquals( + "The number of nested documents has exceeded the allowed limit of [" + defaultMaxNoNestedDocs + + "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + + "] index level setting.", + e.getMessage() + ); } public void testLimitNestedDocs() throws Exception{ // setting limit to allow only two nested objects in the whole doc - long nested_docs_limit = 2L; + long maxNoNestedDocs = 2L; MapperService mapperService = createIndex("test1", Settings.builder() - .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), nested_docs_limit).build()).mapperService(); + .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), maxNoNestedDocs).build()).mapperService(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").endObject() .endObject().endObject().endObject().string(); @@ -591,7 +598,7 @@ public void testLimitNestedDocs() throws Exception{ SourceToParse source2 = SourceToParse.source("test1", "type", "2", docBuilder2.bytes(), XContentType.JSON); MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2)); assertEquals( - "The number of nested documents has exceeded the allowed limit of [" + nested_docs_limit + "The number of nested documents has exceeded the allowed limit of [" + maxNoNestedDocs + "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + "] index level setting.", e.getMessage() @@ -600,9 +607,9 @@ public void testLimitNestedDocs() throws Exception{ public void testLimitNestedDocsMultipleNestedFields() throws Exception{ // setting limit to allow only two nested objects in the whole doc - long nested_docs_limit = 2L; + long maxNoNestedDocs = 2L; MapperService mapperService = createIndex("test1", Settings.builder() - .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), nested_docs_limit).build()).mapperService(); + .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), maxNoNestedDocs).build()).mapperService(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").endObject() .startObject("nested2").field("type", "nested").endObject() @@ -650,7 +657,7 @@ public void testLimitNestedDocsMultipleNestedFields() throws Exception{ SourceToParse source2 = SourceToParse.source("test1", "type", "2", docBuilder2.bytes(), XContentType.JSON); MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2)); assertEquals( - "The number of nested documents has exceeded the allowed limit of [" + nested_docs_limit + "The number of nested documents has exceeded the allowed limit of [" + maxNoNestedDocs + "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + "] index level setting.", e.getMessage() diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml index 1627dca4d15fd..2c912a2165a83 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml @@ -16,7 +16,7 @@ setup: "Indexing a doc with No. nested objects less or equal to index.mapping.nested_objects.limit should succeed": - skip: version: " - 6.99.99" - reason: index.mapping.nested_objects setting was introduced from this version + reason: index.mapping.nested_objects setting has been added in 7.0.0 - do: create: index: test_1 @@ -30,7 +30,7 @@ setup: "Indexing a doc with No. nested objects more than index.mapping.nested_objects.limit should fail": - skip: version: " - 6.99.99" - reason: index.mapping.nested_objects setting was introduced from this version + reason: index.mapping.nested_objects setting has been added in 7.0.0 - do: catch: /The number of nested documents has exceeded the allowed limit of \[2\]. This limit can be set by changing the \[index.mapping.nested_objects.limit\] index level setting\./ create: From 5d4273805441b87547390d9d53ae50e8ecd7645c Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 22 Nov 2017 08:18:15 -0500 Subject: [PATCH 4/4] Limit the number of nested documents Add an index level setting `index.mapping.nested_objects.limit` to control the number of nested json objects that can be in a single document across all fields. Defaults to 10000. Throw an error if the number of created nested documents exceed this limit during the parsing of a document. Closes #26962 --- .../org/elasticsearch/index/mapper/NestedObjectMapperTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index eb79b243f57de..e17d58d4a14b2 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -539,7 +539,7 @@ public void testLimitNestedDocsDefaultSettings() throws Exception{ { docBuilder.startArray("nested1"); { - for(int i=0; i<=defaultMaxNoNestedDocs; i++) { + for(int i = 0; i <= defaultMaxNoNestedDocs; i++) { docBuilder.startObject().field("f", i).endObject(); } }