diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 62b1114c74a97..a38f770026596 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -52,6 +52,8 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.index.similarity.SimilarityService; import java.io.IOException; import java.util.ArrayList; @@ -64,6 +66,7 @@ import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue; import static org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType.hasGaps; +import static org.elasticsearch.index.mapper.TypeParsers.checkNull; import static org.elasticsearch.index.mapper.TypeParsers.parseTextField; /** @@ -113,30 +116,39 @@ public Mapper.Builder parse(String name, builder.indexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer()); builder.searchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer()); builder.searchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer()); - parseTextField(builder, name, node, parserContext); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { final Map.Entry entry = iterator.next(); final String fieldName = entry.getKey(); final Object fieldNode = entry.getValue(); - + checkNull(fieldName, fieldNode); if (fieldName.equals("max_shingle_size")) { builder.maxShingleSize(nodeIntegerValue(fieldNode)); iterator.remove(); + } else if (fieldName.equals("similarity")) { + SimilarityProvider similarityProvider = TypeParsers.resolveSimilarity(parserContext, fieldName, fieldNode.toString()); + builder.similarity(similarityProvider); + iterator.remove(); } // TODO should we allow to configure the prefix field } + parseTextField(builder, name, node, parserContext); return builder; } } public static class Builder extends FieldMapper.Builder { private int maxShingleSize = Defaults.MAX_SHINGLE_SIZE; + private SimilarityProvider similarity; public Builder(String name) { super(name, Defaults.FIELD_TYPE); this.builder = this; } + public void similarity(SimilarityProvider similarity) { + this.similarity = similarity; + } + public Builder maxShingleSize(int maxShingleSize) { if (maxShingleSize < MAX_SHINGLE_SIZE_LOWER_BOUND || maxShingleSize > MAX_SHINGLE_SIZE_UPPER_BOUND) { throw new MapperParsingException("[max_shingle_size] must be at least [" + MAX_SHINGLE_SIZE_LOWER_BOUND + "] and at most " + @@ -157,11 +169,10 @@ public Builder docValues(boolean docValues) { @Override public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { - SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, meta); + SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity, meta); ft.setIndexAnalyzer(indexAnalyzer); ft.setSearchAnalyzer(searchAnalyzer); ft.setSearchQuoteAnalyzer(searchQuoteAnalyzer); - ft.setSimilarity(similarity); // set up the prefix field FieldType prefixft = new FieldType(fieldType); @@ -235,8 +246,8 @@ static class SearchAsYouTypeFieldType extends StringFieldType { PrefixFieldType prefixField; ShingleFieldType[] shingleFields = new ShingleFieldType[0]; - SearchAsYouTypeFieldType(String name, FieldType fieldType, Map meta) { - super(name, fieldType.indexOptions() != IndexOptions.NONE, false, new TextSearchInfo(fieldType), meta); + SearchAsYouTypeFieldType(String name, FieldType fieldType, SimilarityProvider similarity, Map meta) { + super(name, fieldType.indexOptions() != IndexOptions.NONE, false, new TextSearchInfo(fieldType, similarity), meta); } SearchAsYouTypeFieldType(SearchAsYouTypeFieldType other) { @@ -382,7 +393,7 @@ static final class PrefixFieldType extends StringFieldType { final String parentField; PrefixFieldType(String parentField, FieldType fieldType, int minChars, int maxChars) { - super(parentField + PREFIX_FIELD_SUFFIX, true, false, new TextSearchInfo(fieldType), Collections.emptyMap()); + super(parentField + PREFIX_FIELD_SUFFIX, true, false, new TextSearchInfo(fieldType, null), Collections.emptyMap()); this.minChars = minChars; this.maxChars = maxChars; this.parentField = parentField; @@ -535,7 +546,7 @@ static class ShingleFieldType extends StringFieldType { PrefixFieldType prefixFieldType; ShingleFieldType(String name, int shingleSize, FieldType fieldType) { - super(name, true, false, new TextSearchInfo(fieldType), Collections.emptyMap()); + super(name, true, false, new TextSearchInfo(fieldType, null), Collections.emptyMap()); this.shingleSize = shingleSize; } @@ -689,7 +700,8 @@ protected void mergeOptions(FieldMapper other, List conflicts) { this.shingleFields[i] = (ShingleFieldMapper) this.shingleFields[i].merge(m.shingleFields[i]); } } - if (Objects.equals(this.fieldType().similarity(), other.fieldType().similarity()) == false) { + if (Objects.equals(this.fieldType().getTextSearchInfo().getSimilarity(), + other.fieldType().getTextSearchInfo().getSimilarity()) == false) { conflicts.add("mapper [" + name() + "] has different [similarity] settings"); } } @@ -719,6 +731,11 @@ public ShingleFieldMapper[] shingleFields() { protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); doXContentAnalyzers(builder, includeDefaults); + if (fieldType().getTextSearchInfo().getSimilarity() != null) { + builder.field("similarity", fieldType().getTextSearchInfo().getSimilarity().name()); + } else if (includeDefaults) { + builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY); + } builder.field("max_shingle_size", maxShingleSize); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 50f50ada355f3..459d301d59445 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -32,6 +32,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.similarities.BM25Similarity; +import org.apache.lucene.search.similarities.BooleanSimilarity; import org.apache.lucene.search.spans.FieldMaskingSpanQuery; import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanTermQuery; @@ -56,6 +58,7 @@ import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.plugins.Plugin; import org.hamcrest.Matcher; import org.hamcrest.Matchers; @@ -88,6 +91,10 @@ public void addModifiers() { a.maxShingleSize(3); b.maxShingleSize(2); }); + addModifier("similarity", false, (a, b) -> { + a.similarity(new SimilarityProvider("BM25", new BM25Similarity())); + b.similarity(new SimilarityProvider("boolean", new BooleanSimilarity())); + }); } @Override diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java index 0d043590bc3c1..506279ab0695b 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java @@ -51,7 +51,7 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase meta) { - final SearchAsYouTypeFieldType fieldType = new SearchAsYouTypeFieldType(name, Defaults.FIELD_TYPE, meta); + final SearchAsYouTypeFieldType fieldType = new SearchAsYouTypeFieldType(name, Defaults.FIELD_TYPE, null, meta); fieldType.setPrefixField(new PrefixFieldType(NAME, Defaults.FIELD_TYPE, Defaults.MIN_GRAM, Defaults.MAX_GRAM)); fieldType.setShingleFields(new ShingleFieldType[] { new ShingleFieldType(fieldType.name(), 2, Defaults.FIELD_TYPE) }); return fieldType; @@ -62,7 +62,7 @@ public void testTermQuery() { assertThat(fieldType.termQuery("foo", null), equalTo(new TermQuery(new Term(NAME, "foo")))); - SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, Collections.emptyMap()); + SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, null, Collections.emptyMap()); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("foo", null)); assertThat(e.getMessage(), equalTo("Cannot search on field [" + NAME + "] since it is not indexed.")); } @@ -73,7 +73,7 @@ public void testTermsQuery() { assertThat(fieldType.termsQuery(asList("foo", "bar"), null), equalTo(new TermInSetQuery(NAME, asList(new BytesRef("foo"), new BytesRef("bar"))))); - SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, Collections.emptyMap()); + SearchAsYouTypeFieldType unsearchable = new SearchAsYouTypeFieldType(NAME, UNSEARCHABLE, null, Collections.emptyMap()); final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termsQuery(asList("foo", "bar"), null)); assertThat(e.getMessage(), equalTo("Cannot search on field [" + NAME + "] since it is not indexed.")); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index da8eef96bba34..55f2d745d0694 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -195,7 +195,7 @@ public static final class CompletionFieldType extends TermBasedFieldType { private ContextMappings contextMappings = null; public CompletionFieldType(String name, FieldType luceneFieldType, Map meta) { - super(name, true, false, new TextSearchInfo(luceneFieldType), meta); + super(name, true, false, new TextSearchInfo(luceneFieldType, null), meta); } public CompletionFieldType(String name) { @@ -402,7 +402,6 @@ public CompletionFieldMapper build(BuilderContext context) { ft.setIndexAnalyzer(indexAnalyzer); ft.setSearchAnalyzer(searchAnalyzer); ft.setSearchQuoteAnalyzer(searchQuoteAnalyzer); - ft.setSimilarity(similarity); return new CompletionFieldMapper(name, this.fieldType, ft, multiFieldsBuilder.build(this, context), copyTo, maxInputLength); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 58d7e3d34a79d..c527e677e5bb5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -32,8 +32,6 @@ import org.elasticsearch.common.xcontent.support.AbstractXContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType; -import org.elasticsearch.index.similarity.SimilarityProvider; -import org.elasticsearch.index.similarity.SimilarityService; import java.io.IOException; import java.util.ArrayList; @@ -70,7 +68,6 @@ public abstract static class Builder> extends Mapper.Builde protected NamedAnalyzer indexAnalyzer; protected NamedAnalyzer searchAnalyzer; protected NamedAnalyzer searchQuoteAnalyzer; - protected SimilarityProvider similarity; protected Builder(String name, FieldType fieldType) { super(name); @@ -159,11 +156,6 @@ public T searchQuoteAnalyzer(NamedAnalyzer searchQuoteAnalyzer) { return builder; } - public T similarity(SimilarityProvider similarity) { - this.similarity = similarity; - return builder; - } - public T setEagerGlobalOrdinals(boolean eagerGlobalOrdinals) { this.eagerGlobalOrdinals = eagerGlobalOrdinals; return builder; @@ -374,10 +366,6 @@ private void mergeSharedOptions(FieldMapper mergeWith, List conflicts) { } else if (mappedFieldType.indexAnalyzer().name().equals(otherm.indexAnalyzer().name()) == false) { conflicts.add("mapper [" + name() + "] has different [analyzer]"); } - - if (Objects.equals(mappedFieldType.similarity(), otherm.similarity()) == false) { - conflicts.add("mapper [" + name() + "] has different [similarity]"); - } } /** @@ -423,12 +411,6 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, builder.field("store", fieldType.stored()); } - if (fieldType().similarity() != null) { - builder.field("similarity", fieldType().similarity().name()); - } else if (includeDefaults) { - builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY); - } - multiFields.toXContent(builder, params); copyTo.toXContent(builder, params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 5e6c3d9c2b350..94fe5b3424528 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import java.io.IOException; @@ -52,6 +53,7 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.index.mapper.TypeParsers.checkNull; import static org.elasticsearch.index.mapper.TypeParsers.parseField; /** @@ -97,6 +99,7 @@ public static class Builder extends FieldMapper.Builder { private String normalizerName = "default"; private boolean eagerGlobalOrdinals = Defaults.EAGER_GLOBAL_ORDINALS; private boolean splitQueriesOnWhitespace = Defaults.SPLIT_QUERIES_ON_WHITESPACE; + private SimilarityProvider similarity; public Builder(String name) { super(name, Defaults.FIELD_TYPE); @@ -109,6 +112,10 @@ public Builder omitNorms(boolean omitNorms) { return builder; } + public void similarity(SimilarityProvider similarity) { + this.similarity = similarity; + } + public Builder ignoreAbove(int ignoreAbove) { if (ignoreAbove < 0) { throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove); @@ -181,11 +188,11 @@ public static class TypeParser implements Mapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder(name); - parseField(builder, name, node, parserContext); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String propName = entry.getKey(); Object propNode = entry.getValue(); + checkNull(propName, propNode); if (propName.equals("null_value")) { if (propNode == null) { throw new MapperParsingException("Property [null_value] cannot be null."); @@ -209,8 +216,13 @@ public Mapper.Builder parse(String name, Map node, ParserCont } else if (propName.equals("split_queries_on_whitespace")) { builder.splitQueriesOnWhitespace(XContentMapValues.nodeBooleanValue(propNode, "split_queries_on_whitespace")); iterator.remove(); + } else if (propName.equals("similarity")) { + SimilarityProvider similarityProvider = TypeParsers.resolveSimilarity(parserContext, name, propNode.toString()); + builder.similarity(similarityProvider); + iterator.remove(); } } + parseField(builder, name, node, parserContext); return builder; } } @@ -223,12 +235,11 @@ public KeywordFieldType(String name, boolean hasDocValues, FieldType fieldType, boolean eagerGlobalOrdinals, NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer, SimilarityProvider similarity, Map meta, float boost) { super(name, fieldType.indexOptions() != IndexOptions.NONE, - hasDocValues, new TextSearchInfo(fieldType), meta); + hasDocValues, new TextSearchInfo(fieldType, similarity), meta); this.hasNorms = fieldType.omitNorms() == false; setEagerGlobalOrdinals(eagerGlobalOrdinals); setIndexAnalyzer(normalizer); setSearchAnalyzer(searchAnalyzer); - setSimilarity(similarity); setBoost(boost); } @@ -422,6 +433,10 @@ protected void mergeOptions(FieldMapper other, List conflicts) { if (Objects.equals(fieldType().indexAnalyzer(), k.fieldType().indexAnalyzer()) == false) { conflicts.add("mapper [" + name() + "] has different [normalizer]"); } + if (Objects.equals(mappedFieldType.getTextSearchInfo().getSimilarity(), + k.fieldType().getTextSearchInfo().getSimilarity()) == false) { + conflicts.add("mapper [" + name() + "] has different [similarity]"); + } this.ignoreAbove = k.ignoreAbove; this.splitQueriesOnWhitespace = k.splitQueriesOnWhitespace; this.fieldType().setSearchAnalyzer(k.fieldType().searchAnalyzer()); @@ -456,6 +471,12 @@ else if (includeDefaults) { builder.field("normalizer", "default"); } + if (fieldType().getTextSearchInfo().getSimilarity() != null) { + builder.field("similarity", fieldType().getTextSearchInfo().getSimilarity().name()); + } else if (includeDefaults) { + builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY); + } + if (includeDefaults || splitQueriesOnWhitespace) { builder.field("split_queries_on_whitespace", splitQueriesOnWhitespace); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index a648f709a59aa..ef8f2416e5769 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -46,7 +46,6 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; -import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.search.DocValueFormat; import java.io.IOException; @@ -68,7 +67,6 @@ public abstract class MappedFieldType { private NamedAnalyzer indexAnalyzer; private NamedAnalyzer searchAnalyzer; private NamedAnalyzer searchQuoteAnalyzer; - private SimilarityProvider similarity; private boolean eagerGlobalOrdinals; private Map meta; @@ -80,7 +78,6 @@ protected MappedFieldType(MappedFieldType ref) { this.indexAnalyzer = ref.indexAnalyzer(); this.searchAnalyzer = ref.searchAnalyzer(); this.searchQuoteAnalyzer = ref.searchQuoteAnalyzer; - this.similarity = ref.similarity(); this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals; this.meta = ref.meta; this.textSearchInfo = ref.textSearchInfo; @@ -125,14 +122,13 @@ public boolean equals(Object o) { Objects.equals(searchAnalyzer, fieldType.searchAnalyzer) && Objects.equals(searchQuoteAnalyzer(), fieldType.searchQuoteAnalyzer()) && Objects.equals(eagerGlobalOrdinals, fieldType.eagerGlobalOrdinals) && - Objects.equals(similarity, fieldType.similarity) && Objects.equals(meta, fieldType.meta); } @Override public int hashCode() { return Objects.hash(name, boost, docValues, indexAnalyzer, searchAnalyzer, searchQuoteAnalyzer, - eagerGlobalOrdinals, similarity == null ? null : similarity.name(), meta); + eagerGlobalOrdinals, meta); } // TODO: we need to override freeze() and add safety checks that all settings are actually set @@ -180,14 +176,6 @@ public void setSearchQuoteAnalyzer(NamedAnalyzer analyzer) { this.searchQuoteAnalyzer = analyzer; } - public SimilarityProvider similarity() { - return similarity; - } - - public void setSimilarity(SimilarityProvider similarity) { - this.similarity = similarity; - } - /** Given a value that comes from the stored fields API, convert it to the * expected type. For instance a date field would store dates as longs and * format it back to a string in this method. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 95a4df8694233..c17f6e8b62119 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -68,6 +68,8 @@ import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.query.IntervalBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import java.io.IOException; @@ -80,6 +82,7 @@ import java.util.Objects; import java.util.function.IntPredicate; +import static org.elasticsearch.index.mapper.TypeParsers.checkNull; import static org.elasticsearch.index.mapper.TypeParsers.parseTextField; /** A {@link FieldMapper} for full-text fields. */ @@ -126,6 +129,7 @@ public static class Builder extends FieldMapper.Builder { private double fielddataMinFreq = Defaults.FIELDDATA_MIN_FREQUENCY; private double fielddataMaxFreq = Defaults.FIELDDATA_MAX_FREQUENCY; private int fielddataMinSegSize = Defaults.FIELDDATA_MIN_SEGMENT_SIZE; + private SimilarityProvider similarity; public Builder(String name) { super(name, Defaults.FIELD_TYPE); @@ -150,6 +154,10 @@ public Builder indexPhrases(boolean indexPhrases) { return builder; } + public void similarity(SimilarityProvider similarity) { + this.similarity = similarity; + } + @Override public Builder docValues(boolean docValues) { if (docValues) { @@ -187,12 +195,11 @@ public Builder indexPrefixes(int minChars, int maxChars) { } private TextFieldType buildFieldType(BuilderContext context) { - TextFieldType ft = new TextFieldType(buildFullName(context), fieldType, meta); + TextFieldType ft = new TextFieldType(buildFullName(context), fieldType, similarity, meta); ft.setIndexAnalyzer(indexAnalyzer); ft.setSearchAnalyzer(searchAnalyzer); ft.setSearchQuoteAnalyzer(searchQuoteAnalyzer); ft.setEagerGlobalOrdinals(eagerGlobalOrdinals); - ft.setSimilarity(similarity); if (fielddata) { ft.setFielddata(true); ft.setFielddataMinFrequency(fielddataMinFreq); @@ -275,11 +282,11 @@ public Mapper.Builder parse(String fieldName, Map node, ParserCo builder.indexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer()); builder.searchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer()); builder.searchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer()); - parseTextField(builder, fieldName, node, parserContext); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String propName = entry.getKey(); Object propNode = entry.getValue(); + checkNull(propName, propNode); if (propName.equals("position_increment_gap")) { int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1); builder.positionIncrementGap(newPositionIncrementGap); @@ -310,8 +317,13 @@ public Mapper.Builder parse(String fieldName, Map node, ParserCo } else if (propName.equals("index_phrases")) { builder.indexPhrases(XContentMapValues.nodeBooleanValue(propNode, "index_phrases")); iterator.remove(); + } else if (propName.equals("similarity")) { + SimilarityProvider similarityProvider = TypeParsers.resolveSimilarity(parserContext, fieldName, propNode.toString()); + builder.similarity(similarityProvider); + iterator.remove(); } } + parseTextField(builder, fieldName, node, parserContext); return builder; } } @@ -554,9 +566,9 @@ public static class TextFieldType extends StringFieldType { private boolean indexPhrases = false; private final FieldType indexedFieldType; - public TextFieldType(String name, FieldType indexedFieldType, Map meta) { + public TextFieldType(String name, FieldType indexedFieldType, SimilarityProvider similarity, Map meta) { super(name, indexedFieldType.indexOptions() != IndexOptions.NONE, false, - new TextSearchInfo(indexedFieldType), meta); + new TextSearchInfo(indexedFieldType, similarity), meta); this.indexedFieldType = indexedFieldType; fielddata = false; fielddataMinFrequency = Defaults.FIELDDATA_MIN_FREQUENCY; @@ -565,13 +577,13 @@ public TextFieldType(String name, FieldType indexedFieldType, Map meta) { - super(name, indexed, false, new TextSearchInfo(Defaults.FIELD_TYPE), meta); + super(name, indexed, false, new TextSearchInfo(Defaults.FIELD_TYPE, null), meta); this.indexedFieldType = Defaults.FIELD_TYPE; fielddata = false; } public TextFieldType(String name) { - this(name, Defaults.FIELD_TYPE, Collections.emptyMap()); + this(name, Defaults.FIELD_TYPE, null, Collections.emptyMap()); } protected TextFieldType(TextFieldType ref) { @@ -888,7 +900,8 @@ protected String contentType() { @Override protected void mergeOptions(FieldMapper other, List conflicts) { TextFieldMapper mw = (TextFieldMapper) other; - if (Objects.equals(mw.fieldType().similarity(), this.fieldType().similarity()) == false) { + if (Objects.equals(mw.fieldType().getTextSearchInfo().getSimilarity(), + this.fieldType().getTextSearchInfo().getSimilarity()) == false) { conflicts.add("mapper [" + name() + "] has different [similarity] settings"); } if (mw.fieldType().indexPhrases != this.fieldType().indexPhrases) { @@ -931,7 +944,11 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, builder.field("norms", fieldType.omitNorms() == false); } doXContentAnalyzers(builder, includeDefaults); - + if (fieldType().getTextSearchInfo().getSimilarity() != null) { + builder.field("similarity", fieldType().getTextSearchInfo().getSimilarity().name()); + } else if (includeDefaults) { + builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY); + } if (includeDefaults || fieldType().eagerGlobalOrdinals()) { builder.field("eager_global_ordinals", fieldType().eagerGlobalOrdinals()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java b/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java index 618226f7cdf12..f0d8fb14c12e1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java @@ -21,6 +21,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; +import org.elasticsearch.index.similarity.SimilarityProvider; /** * Encapsulates information about how to perform text searches over a field @@ -39,20 +40,30 @@ public class TextSearchInfo { * * Note that the results of {@link #isStored()} for this may not be accurate */ - public static final TextSearchInfo SIMPLE_MATCH_ONLY = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE); + public static final TextSearchInfo SIMPLE_MATCH_ONLY = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE, null); /** * Specifies that this field does not support text searching of any kind */ - public static final TextSearchInfo NONE = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE); + public static final TextSearchInfo NONE = new TextSearchInfo(SIMPLE_MATCH_ONLY_FIELD_TYPE, null); private final FieldType luceneFieldType; + private final SimilarityProvider similarity; /** - * Create a TextSearchInfo by wrapping a lucene FieldType + * Create a new TextSearchInfo + * + * @param luceneFieldType the lucene {@link FieldType} of the field to be searched + * @param similarity defines which Similarity to use when searching. If set to {@code null} + * then the default Similarity will be used. */ - public TextSearchInfo(FieldType luceneFieldType) { + public TextSearchInfo(FieldType luceneFieldType, SimilarityProvider similarity) { this.luceneFieldType = luceneFieldType; + this.similarity = similarity; + } + + public SimilarityProvider getSimilarity() { + return similarity; } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index b2f67ee03d5ac..a3e469df0b913 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -169,6 +169,16 @@ public static void parseTextField(FieldMapper.Builder builder, String name, M } } + public static void checkNull(String propName, Object propNode) { + if (false == propName.equals("null_value") && propNode == null) { + /* + * No properties *except* null_value are allowed to have null. So we catch it here and tell the user something useful rather + * than send them a null pointer exception later. + */ + throw new MapperParsingException("[" + propName + "] must not have a [null] value"); + } + } + /** * Parse the {@code meta} key of the mapping. */ @@ -225,13 +235,7 @@ public static void parseField(FieldMapper.Builder builder, String name, Map entry = iterator.next(); final String propName = entry.getKey(); final Object propNode = entry.getValue(); - if (false == propName.equals("null_value") && propNode == null) { - /* - * No properties *except* null_value are allowed to have null. So we catch it here and tell the user something useful rather - * than send them a null pointer exception later. - */ - throw new MapperParsingException("[" + propName + "] must not have a [null] value"); - } + checkNull(propName, propNode); if (propName.equals("store")) { builder.store(XContentMapValues.nodeBooleanValue(propNode, name + ".store")); iterator.remove(); @@ -248,8 +252,8 @@ public static void parseField(FieldMapper.Builder builder, String name, Map { + a.similarity(new SimilarityProvider("BM25", new BM25Similarity())); + b.similarity(new SimilarityProvider("boolean", new BooleanSimilarity())); + }); } public void testDefaults() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index b33383f290477..2514f93131398 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -478,6 +478,17 @@ public void testLongIndexingOutOfRange() throws Exception { assertTrue(response.status() == RestStatus.CREATED); } + public void testDeprecatedSimilarityParameter() throws Exception { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("field") + .field("type", "long") + .field("similarity", "bm25") + .endObject().endObject().endObject().endObject()); + + parser.parse("type", new CompressedXContent(mapping)); + assertWarnings("The [similarity] parameter has no effect on field [field] and will be removed in 8.0"); + } + private void parseRequest(NumberType type, BytesReference content) throws IOException { createDocumentMapper(type).parse(new SourceToParse("test", "1", content, XContentType.JSON)); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index d312c8dcf4ee9..8a4eebf3ffe76 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -42,6 +42,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.similarities.BM25Similarity; +import org.apache.lucene.search.similarities.BooleanSimilarity; import org.apache.lucene.search.spans.FieldMaskingSpanQuery; import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanOrQuery; @@ -71,6 +73,7 @@ import org.elasticsearch.index.search.MatchQuery; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -123,6 +126,10 @@ public void addModifiers() { a.indexOptions(IndexOptions.DOCS_AND_FREQS); b.indexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); }); + addModifier("similarity", false, (a, b) -> { + a.similarity(new SimilarityProvider("BM25", new BM25Similarity())); + b.similarity(new SimilarityProvider("boolean", new BooleanSimilarity())); + }); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java index c1ded88e2312c..8694bb283b7e3 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java @@ -79,9 +79,10 @@ public void testResolveSimilaritiesFromMapping_bm25() throws IOException { .put("index.similarity.my_similarity.discount_overlaps", false) .build(); MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); - assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(LegacyBM25Similarity.class)); + assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(LegacyBM25Similarity.class)); - LegacyBM25Similarity similarity = (LegacyBM25Similarity) mapperService.fieldType("field1").similarity().get(); + LegacyBM25Similarity similarity + = (LegacyBM25Similarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(); assertThat(similarity.getK1(), equalTo(2.0f)); assertThat(similarity.getB(), equalTo(0.5f)); assertThat(similarity.getDiscountOverlaps(), equalTo(false)); @@ -95,7 +96,7 @@ public void testResolveSimilaritiesFromMapping_boolean() throws IOException { .endObject().endObject(); MapperService mapperService = createIndex("foo", Settings.EMPTY, mapping).mapperService(); - assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(BooleanSimilarity.class)); + assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(BooleanSimilarity.class)); } public void testResolveSimilaritiesFromMapping_DFR() throws IOException { @@ -113,9 +114,9 @@ public void testResolveSimilaritiesFromMapping_DFR() throws IOException { .put("index.similarity.my_similarity.normalization.h2.c", 3f) .build(); MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); - assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(DFRSimilarity.class)); + assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(DFRSimilarity.class)); - DFRSimilarity similarity = (DFRSimilarity) mapperService.fieldType("field1").similarity().get(); + DFRSimilarity similarity = (DFRSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(); assertThat(similarity.getBasicModel(), instanceOf(BasicModelG.class)); assertThat(similarity.getAfterEffect(), instanceOf(AfterEffectL.class)); assertThat(similarity.getNormalization(), instanceOf(NormalizationH2.class)); @@ -137,9 +138,9 @@ public void testResolveSimilaritiesFromMapping_IB() throws IOException { .put("index.similarity.my_similarity.normalization.h2.c", 3f) .build(); MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); - assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(IBSimilarity.class)); + assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(IBSimilarity.class)); - IBSimilarity similarity = (IBSimilarity) mapperService.fieldType("field1").similarity().get(); + IBSimilarity similarity = (IBSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(); assertThat(similarity.getDistribution(), instanceOf(DistributionSPL.class)); assertThat(similarity.getLambda(), instanceOf(LambdaTTF.class)); assertThat(similarity.getNormalization(), instanceOf(NormalizationH2.class)); @@ -160,8 +161,8 @@ public void testResolveSimilaritiesFromMapping_DFI() throws IOException { MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); MappedFieldType fieldType = mapperService.fieldType("field1"); - assertThat(fieldType.similarity().get(), instanceOf(DFISimilarity.class)); - DFISimilarity similarity = (DFISimilarity) fieldType.similarity().get(); + assertThat(fieldType.getTextSearchInfo().getSimilarity().get(), instanceOf(DFISimilarity.class)); + DFISimilarity similarity = (DFISimilarity) fieldType.getTextSearchInfo().getSimilarity().get(); assertThat(similarity.getIndependence(), instanceOf(IndependenceChiSquared.class)); } @@ -178,9 +179,10 @@ public void testResolveSimilaritiesFromMapping_LMDirichlet() throws IOException .build(); MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); - assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(LMDirichletSimilarity.class)); + assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(LMDirichletSimilarity.class)); - LMDirichletSimilarity similarity = (LMDirichletSimilarity) mapperService.fieldType("field1").similarity().get(); + LMDirichletSimilarity similarity + = (LMDirichletSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(); assertThat(similarity.getMu(), equalTo(3000f)); } @@ -196,9 +198,11 @@ public void testResolveSimilaritiesFromMapping_LMJelinekMercer() throws IOExcept .put("index.similarity.my_similarity.lambda", 0.7f) .build(); MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); - assertThat(mapperService.fieldType("field1").similarity().get(), instanceOf(LMJelinekMercerSimilarity.class)); + assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), + instanceOf(LMJelinekMercerSimilarity.class)); - LMJelinekMercerSimilarity similarity = (LMJelinekMercerSimilarity) mapperService.fieldType("field1").similarity().get(); + LMJelinekMercerSimilarity similarity + = (LMJelinekMercerSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(); assertThat(similarity.getLambda(), equalTo(0.7f)); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java index 39d5c8fd51f49..c8d974bbaa414 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java @@ -22,8 +22,6 @@ import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.search.similarities.BM25Similarity; -import org.apache.lucene.search.similarities.BooleanSimilarity; import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; @@ -34,7 +32,6 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; @@ -106,10 +103,6 @@ protected Set unsupportedProperties() { a.searchQuoteAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer())); a.searchQuoteAnalyzer(new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer())); }), - new Modifier("similarity", false, (a, b) -> { - a.similarity(new SimilarityProvider("BM25", new BM25Similarity())); - b.similarity(new SimilarityProvider("boolean", new BooleanSimilarity())); - }), new Modifier("store", false, (a, b) -> { a.store(true); b.store(false); diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index b5aaf1239c645..62605c85bc5dc 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -63,7 +63,6 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; @@ -145,11 +144,6 @@ public Builder store(boolean store) { return this; } - @Override - public Builder similarity(SimilarityProvider similarity) { - throw new MapperParsingException("The field [" + name + "] cannot have custom similarities"); - } - @Override public Builder index(boolean index) { if (index == false) { @@ -217,7 +211,7 @@ public static final class WildcardFieldType extends MappedFieldType { static Analyzer lowercaseNormalizer = new LowercaseNormalizer(); public WildcardFieldType(String name, FieldType fieldType, Map meta) { - super(name, true, true, new TextSearchInfo(fieldType), meta); + super(name, true, true, new TextSearchInfo(fieldType, null), meta); setIndexAnalyzer(WILDCARD_ANALYZER); setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); }