Skip to content

Commit 91a1591

Browse files
csoulioscsoulios
authored
Add constraints to dimension fields (#74939)
This PR adds the following constraints to dimension fields: It must be an indexed field and must has doc values It cannot be multi-valued The number of dimension fields in the index mapping must not be more than 16. This should be configurable through an index property (index.mapping.dimension_fields.limit) keyword fields cannot be more than 1024 bytes long keyword fields must not use a normalizer Based on the code added in PR #74450 Relates to #74660
1 parent 95469c3 commit 91a1591

File tree

13 files changed

+254
-12
lines changed

13 files changed

+254
-12
lines changed

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
144144
MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING,
145145
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
146146
MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING,
147+
MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING,
147148
MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING,
148149
BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
149150
IndexModule.INDEX_STORE_TYPE_SETTING,

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.util.function.Function;
3232

3333
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING;
34+
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING;
3435
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING;
3536
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING;
3637
import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING;
@@ -379,6 +380,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
379380
private volatile long mappingTotalFieldsLimit;
380381
private volatile long mappingDepthLimit;
381382
private volatile long mappingFieldNameLengthLimit;
383+
private volatile long mappingDimensionFieldsLimit;
382384

383385
/**
384386
* The maximum number of refresh listeners allows on this shard.
@@ -503,6 +505,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
503505
mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
504506
mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING);
505507
mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING);
508+
mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING);
506509

507510
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
508511
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING,
@@ -558,6 +561,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
558561
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit);
559562
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit);
560563
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit);
564+
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit);
561565
}
562566

563567
private void setSearchIdleAfter(TimeValue searchIdleAfter) { this.searchIdleAfter = searchIdleAfter; }
@@ -1021,4 +1025,12 @@ public long getMappingFieldNameLengthLimit() {
10211025
private void setMappingFieldNameLengthLimit(long value) {
10221026
this.mappingFieldNameLengthLimit = value;
10231027
}
1028+
1029+
public long getMappingDimensionFieldsLimit() {
1030+
return mappingDimensionFieldsLimit;
1031+
}
1032+
1033+
private void setMappingDimensionFieldsLimit(long value) {
1034+
this.mappingDimensionFieldsLimit = value;
1035+
}
10241036
}

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.elasticsearch.index.mapper;
1010

11+
import org.apache.lucene.document.Field;
1112
import org.apache.lucene.document.InetAddressPoint;
1213
import org.apache.lucene.document.SortedSetDocValuesField;
1314
import org.apache.lucene.document.StoredField;
@@ -72,8 +73,7 @@ public static class Builder extends FieldMapper.Builder {
7273
private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
7374

7475
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
75-
private final Parameter<Boolean> dimension
76-
= Parameter.boolParam("dimension", false, m -> toType(m).dimension, false);
76+
private final Parameter<Boolean> dimension;
7777

7878
private final boolean ignoreMalformedByDefault;
7979
private final Version indexCreatedVersion;
@@ -88,6 +88,14 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform
8888
= Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
8989
this.script.precludesParameters(nullValue, ignoreMalformed);
9090
addScriptValidation(script, indexed, hasDocValues);
91+
this.dimension = Parameter.boolParam("dimension", false, m -> toType(m).dimension, false)
92+
.setValidator(v -> {
93+
if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) {
94+
throw new IllegalArgumentException(
95+
"Field [dimension] requires that [" + indexed.name + "] and [" + hasDocValues.name + "] are true"
96+
);
97+
}
98+
});
9199
}
92100

93101
Builder nullValue(String nullValue) {
@@ -467,7 +475,17 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
467475

468476
private void indexValue(DocumentParserContext context, InetAddress address) {
469477
if (indexed) {
470-
context.doc().add(new InetAddressPoint(fieldType().name(), address));
478+
Field field = new InetAddressPoint(fieldType().name(), address);
479+
if (dimension) {
480+
// Add dimension field with key so that we ensure it is single-valued.
481+
// Dimension fields are always indexed.
482+
if (context.doc().getByKey(fieldType().name()) != null) {
483+
throw new IllegalArgumentException("Dimension field [" + fieldType().name() + "] cannot be a multi-valued field.");
484+
}
485+
context.doc().addWithKey(fieldType().name(), field);
486+
} else {
487+
context.doc().add(field);
488+
}
471489
}
472490
if (hasDocValues) {
473491
context.doc().add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address))));

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -124,12 +124,13 @@ public Builder(String name, IndexAnalyzers indexAnalyzers, ScriptCompiler script
124124
this.script.precludesParameters(nullValue);
125125
addScriptValidation(script, indexed, hasDocValues);
126126

127-
this.dimension = Parameter.boolParam("dimension", false, m -> toType(m).dimension, false)
128-
.setValidator(v -> {
129-
if (v && ignoreAbove.getValue() < ignoreAbove.getDefaultValue()) {
130-
throw new IllegalArgumentException("Field [ignore_above] cannot be set in conjunction with field [dimension]");
131-
}
132-
});
127+
this.dimension = Parameter.boolParam("dimension", false, m -> toType(m).dimension, false).setValidator(v -> {
128+
if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) {
129+
throw new IllegalArgumentException(
130+
"Field [dimension] requires that [" + indexed.name + "] and [" + hasDocValues.name + "] are true"
131+
);
132+
}
133+
}).precludesParameters(normalizer, ignoreAbove);
133134
}
134135

135136
public Builder(String name) {
@@ -431,6 +432,9 @@ public boolean isDimension() {
431432
}
432433
}
433434

435+
/** The maximum keyword length allowed for a dimension field */
436+
private static final int DIMENSION_MAX_BYTES = 1024;
437+
434438
private final boolean indexed;
435439
private final boolean hasDocValues;
436440
private final String nullValue;
@@ -509,9 +513,24 @@ private void indexValue(DocumentParserContext context, String value) {
509513

510514
// convert to utf8 only once before feeding postings/dv/stored fields
511515
final BytesRef binaryValue = new BytesRef(value);
516+
if (dimension && binaryValue.length > DIMENSION_MAX_BYTES) {
517+
throw new IllegalArgumentException(
518+
"Dimension field [" + fieldType().name() + "] cannot be more than [" + DIMENSION_MAX_BYTES + "] bytes long."
519+
);
520+
}
512521
if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
513522
Field field = new KeywordField(fieldType().name(), binaryValue, fieldType);
514-
context.doc().add(field);
523+
if (dimension) {
524+
// Check that a dimension field is single-valued and not an array
525+
if (context.doc().getByKey(fieldType().name()) != null) {
526+
throw new IllegalArgumentException("Dimension field [" + fieldType().name() + "] cannot be a multi-valued field.");
527+
}
528+
// Add dimension field with key so that we ensure it is single-valued.
529+
// Dimension fields are always indexed.
530+
context.doc().addWithKey(fieldType().name(), field);
531+
} else {
532+
context.doc().add(field);
533+
}
515534

516535
if (fieldType().hasDocValues() == false && fieldType.omitNorms()) {
517536
context.addToFieldNames(fieldType().name());

‎server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,13 @@ public boolean isAggregatable() {
165165
}
166166
}
167167

168+
/**
169+
* @return true if field has been marked as a dimension field
170+
*/
171+
public boolean isDimension() {
172+
return false;
173+
}
174+
168175
/** Generates a query that will only match documents that contain the given value.
169176
* The default implementation returns a {@link TermQuery} over the value bytes
170177
* @throws IllegalArgumentException if {@code value} cannot be converted to the expected data type or if the field is not searchable

‎server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ public enum MergeReason {
8989
Setting.longSetting("index.mapping.depth.limit", 20L, 1, Property.Dynamic, Property.IndexScope);
9090
public static final Setting<Long> INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING =
9191
Setting.longSetting("index.mapping.field_name_length.limit", Long.MAX_VALUE, 1L, Property.Dynamic, Property.IndexScope);
92+
public static final Setting<Long> INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING =
93+
Setting.longSetting("index.mapping.dimension_fields.limit", 16, 0, Property.Dynamic, Property.IndexScope);
94+
9295

9396
private final IndexAnalyzers indexAnalyzers;
9497
private final MappingParser mappingParser;

‎server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ void checkLimits(IndexSettings settings) {
204204
checkObjectDepthLimit(settings.getMappingDepthLimit());
205205
checkFieldNameLengthLimit(settings.getMappingFieldNameLengthLimit());
206206
checkNestedLimit(settings.getMappingNestedFieldsLimit());
207+
checkDimensionFieldLimit(settings.getMappingDimensionFieldsLimit());
207208
}
208209

209210
private void checkFieldLimit(long limit) {
@@ -217,6 +218,16 @@ void checkFieldLimit(long limit, int additionalFieldsToAdd) {
217218
}
218219
}
219220

221+
private void checkDimensionFieldLimit(long limit) {
222+
long dimensionFieldCount = fieldMappers.values()
223+
.stream()
224+
.filter(m -> m instanceof FieldMapper && ((FieldMapper) m).fieldType().isDimension())
225+
.count();
226+
if (dimensionFieldCount > limit) {
227+
throw new IllegalArgumentException("Limit of total dimension fields [" + limit + "] has been exceeded");
228+
}
229+
}
230+
220231
private void checkObjectDepthLimit(long limit) {
221232
for (String objectPath : objectMappers.keySet()) {
222233
int numDots = 0;

‎server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,11 @@ public Builder(String name, NumberType type, ScriptCompiler compiler, boolean ig
119119
if (v && EnumSet.of(NumberType.INTEGER, NumberType.LONG, NumberType.BYTE, NumberType.SHORT).contains(type) == false) {
120120
throw new IllegalArgumentException("Parameter [dimension] cannot be set to numeric type [" + type.name + "]");
121121
}
122+
if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) {
123+
throw new IllegalArgumentException(
124+
"Field [dimension] requires that [" + indexed.name + "] and [" + hasDocValues.name + "] are true"
125+
);
126+
}
122127
});
123128

124129
this.script.precludesParameters(ignoreMalformed, coerce, nullValue);
@@ -1174,8 +1179,20 @@ private static Number value(XContentParser parser, NumberType numberType, Number
11741179
}
11751180

11761181
private void indexValue(DocumentParserContext context, Number numericValue) {
1177-
context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue,
1178-
indexed, hasDocValues, stored));
1182+
List<Field> fields = fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, stored);
1183+
if (dimension) {
1184+
// Check that a dimension field is single-valued and not an array
1185+
if (context.doc().getByKey(fieldType().name()) != null) {
1186+
throw new IllegalArgumentException("Dimension field [" + fieldType().name() + "] cannot be a multi-valued field.");
1187+
}
1188+
if (fields.size() > 0) {
1189+
// Add the first field by key so that we can validate if it has been added
1190+
context.doc().addWithKey(fieldType().name(), fields.get(0));
1191+
context.doc().addAll(fields.subList(1, fields.size()));
1192+
}
1193+
} else {
1194+
context.doc().addAll(fields);
1195+
}
11791196

11801197
if (hasDocValues == false && (stored || indexed)) {
11811198
context.addToFieldNames(fieldType().name());

‎server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,4 +287,17 @@ public void testEmptyDocumentMapper() {
287287
assertEquals(10, documentMapper.mappers().getMapping().getMetadataMappersMap().size());
288288
assertEquals(10, documentMapper.mappers().getMatchingFieldNames("*").size());
289289
}
290+
291+
public void testTooManyDimensionFields() {
292+
// By default no more than 16 dimensions per document are supported
293+
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> createDocumentMapper(mapping(b -> {
294+
for (int i = 0; i < 17; i++) {
295+
b.startObject("field" + i)
296+
.field("type", randomFrom("ip", "keyword", "long", "integer", "byte", "short"))
297+
.field("dimension", true)
298+
.endObject();
299+
}
300+
})));
301+
assertThat(e.getMessage(), containsString("Limit of total dimension fields [16] has been exceeded"));
302+
}
290303
}

‎server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,45 @@ public void testDimension() throws IOException {
215215
assertDimension(false, IpFieldMapper.IpFieldType::isDimension);
216216
}
217217

218+
public void testDimensionIndexedAndDocvalues() {
219+
{
220+
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
221+
minimalMapping(b);
222+
b.field("dimension", true).field("index", false).field("doc_values", false);
223+
})));
224+
assertThat(e.getCause().getMessage(),
225+
containsString("Field [dimension] requires that [index] and [doc_values] are true"));
226+
}
227+
{
228+
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
229+
minimalMapping(b);
230+
b.field("dimension", true).field("index", true).field("doc_values", false);
231+
})));
232+
assertThat(e.getCause().getMessage(),
233+
containsString("Field [dimension] requires that [index] and [doc_values] are true"));
234+
}
235+
{
236+
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
237+
minimalMapping(b);
238+
b.field("dimension", true).field("index", false).field("doc_values", true);
239+
})));
240+
assertThat(e.getCause().getMessage(),
241+
containsString("Field [dimension] requires that [index] and [doc_values] are true"));
242+
}
243+
}
244+
245+
public void testDimensionMultiValuedField() throws IOException {
246+
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
247+
minimalMapping(b);
248+
b.field("dimension", true);
249+
}));
250+
251+
Exception e = expectThrows(MapperParsingException.class,
252+
() -> mapper.parse(source(b -> b.array("field", "192.168.1.1", "192.168.1.1"))));
253+
assertThat(e.getCause().getMessage(),
254+
containsString("Dimension field [field] cannot be a multi-valued field"));
255+
}
256+
218257
@Override
219258
protected String generateRandomInputValue(MappedFieldType ft) {
220259
return NetworkAddress.format(randomIp(randomBoolean()));

‎server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,66 @@ public void testDimensionAndIgnoreAbove() {
320320
containsString("Field [ignore_above] cannot be set in conjunction with field [dimension]"));
321321
}
322322

323+
public void testDimensionAndNormalizer() {
324+
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
325+
minimalMapping(b);
326+
b.field("dimension", true).field("normalizer", "my_normalizer");
327+
})));
328+
assertThat(e.getCause().getMessage(),
329+
containsString("Field [normalizer] cannot be set in conjunction with field [dimension]"));
330+
}
331+
332+
public void testDimensionIndexedAndDocvalues() {
333+
{
334+
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
335+
minimalMapping(b);
336+
b.field("dimension", true).field("index", false).field("doc_values", false);
337+
})));
338+
assertThat(e.getCause().getMessage(),
339+
containsString("Field [dimension] requires that [index] and [doc_values] are true"));
340+
}
341+
{
342+
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
343+
minimalMapping(b);
344+
b.field("dimension", true).field("index", true).field("doc_values", false);
345+
})));
346+
assertThat(e.getCause().getMessage(),
347+
containsString("Field [dimension] requires that [index] and [doc_values] are true"));
348+
}
349+
{
350+
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
351+
minimalMapping(b);
352+
b.field("dimension", true).field("index", false).field("doc_values", true);
353+
})));
354+
assertThat(e.getCause().getMessage(),
355+
containsString("Field [dimension] requires that [index] and [doc_values] are true"));
356+
}
357+
}
358+
359+
public void testDimensionMultiValuedField() throws IOException {
360+
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
361+
minimalMapping(b);
362+
b.field("dimension", true);
363+
}));
364+
365+
Exception e = expectThrows(MapperParsingException.class,
366+
() -> mapper.parse(source(b -> b.array("field", "1234", "45678"))));
367+
assertThat(e.getCause().getMessage(),
368+
containsString("Dimension field [field] cannot be a multi-valued field"));
369+
}
370+
371+
public void testDimensionExtraLongKeyword() throws IOException {
372+
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
373+
minimalMapping(b);
374+
b.field("dimension", true);
375+
}));
376+
377+
Exception e = expectThrows(MapperParsingException.class,
378+
() -> mapper.parse(source(b -> b.field("field", randomAlphaOfLengthBetween(1024, 2048)))));
379+
assertThat(e.getCause().getMessage(),
380+
containsString("Dimension field [field] cannot be more than [1024] bytes long."));
381+
}
382+
323383
public void testConfigureSimilarity() throws IOException {
324384
MapperService mapperService = createMapperService(
325385
fieldMapping(b -> b.field("type", "keyword").field("similarity", "boolean"))

0 commit comments

Comments
 (0)