From db1a07982bd6cddcc0fbc0c0d6583d0d3a374c76 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 29 Nov 2019 11:00:42 +0100 Subject: [PATCH 01/21] Introduce a `singleton_keyword` field. This field is a specialization of the `keyword` field for the case when all documents have the same value. It typically performs more efficiently than keywords at query time by figuring out whether all or none of the documents match at rewrite time, like `term` queries on `_index`. The name is up for discussion. I liked including `keyword` in it, so that we still have room for a `singleton_numeric` in the future. However I'm unsure whether to call it `singleton`, `constant` or something else, any opinions? For this field there is a choice between 1. accepting values in `_source` when they are equal to the value configured in mappings, but rejecting mapping updates 2. rejecting values in `_source` but then allowing updates to the value that is configured in the mapping This commit implements option 1, so that it is possible to reindex from/to an index that has the field mapped as a keyword with no changes to the source. --- docs/reference/mapping/types.asciidoc | 6 +- .../mapping/types/singleton-keyword.asciidoc | 73 ++++++ ...va => SingletonKeywordIndexFieldData.java} | 14 +- .../index/mapper/IndexFieldMapper.java | 18 +- .../index/mapper/SingletonFieldType.java | 54 ++++ .../index/mapper/TypeFieldMapper.java | 4 +- .../index/query/PrefixQueryBuilder.java | 43 +-- .../index/query/TermQueryBuilder.java | 37 +-- .../index/query/TermsQueryBuilder.java | 55 ++-- .../index/query/WildcardQueryBuilder.java | 25 +- .../index/query/PrefixQueryBuilderTests.java | 20 +- .../index/query/TermQueryBuilderTests.java | 35 ++- .../index/query/TermsQueryBuilderTests.java | 29 ++- .../query/WildcardQueryBuilderTests.java | 2 +- .../elasticsearch/xpack/core/XPackPlugin.java | 10 +- .../mapper/SingletonKeywordFieldMapper.java | 246 ++++++++++++++++++ .../SingletonKeywordFieldMapperTests.java | 61 +++++ .../SingletonKeywordFieldTypeTests.java | 66 +++++ .../test/singleton_keyword/10_basic.yml | 169 ++++++++++++ 19 files changed, 841 insertions(+), 126 deletions(-) create mode 100644 docs/reference/mapping/types/singleton-keyword.asciidoc rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{ConstantIndexFieldData.java => SingletonKeywordIndexFieldData.java} (89%) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/SingletonFieldType.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapper.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapperTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldTypeTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/singleton_keyword/10_basic.yml diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 1472d1bf641eb..d20623a5fabf3 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -57,6 +57,8 @@ string:: <> and <> <>:: `histogram` for pre-aggregated numerical values for percentiles aggregations. +<>:: Specialization of `keyword` for the case when all documents have the same value. + [float] [[types-array-handling]] === Arrays @@ -126,4 +128,6 @@ include::types/text.asciidoc[] include::types/token-count.asciidoc[] -include::types/shape.asciidoc[] \ No newline at end of file +include::types/shape.asciidoc[] + +include::types/singleton-keyword.asciidoc[] diff --git a/docs/reference/mapping/types/singleton-keyword.asciidoc b/docs/reference/mapping/types/singleton-keyword.asciidoc new file mode 100644 index 0000000000000..7bfbfd980c1fe --- /dev/null +++ b/docs/reference/mapping/types/singleton-keyword.asciidoc @@ -0,0 +1,73 @@ +[role="xpack"] +[testenv="basic"] + +[[singleton-keyword]] +=== Singleton keyword datatype +++++ +Singleton keyword +++++ + +Singleton keyword is a specialization of the <> field for +the case that all documents in the index have the same value. + +[source,console] +-------------------------------- +PUT logs-debug +{ + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "text" + }, + "level": { + "type": "singleton_keyword", + "value": "debug" + } + } + } +} +-------------------------------- + +`singleton_keyword` supports the same queries and aggregations as `keyword` +fields do, but takes advantage of the fact that all documents have the same +value per index to be more efficient. + +It is both allowed to submit documents that don't have a value for the field or +that have a value equal to the value configured in mappings. The two below +indexing requests are equivalent: + +[source,console] +-------------------------------- +POST logs-debug/_doc +{ + "date": "2019-12-12", + "message": "Starting up Elasticsearch", + "level": "debug" +} + +POST logs-debug/_doc +{ + "date": "2019-12-12", + "message": "Starting up Elasticsearch" +} +-------------------------------- +//TEST[continued] + +However providing a value that is different from the one configured in the +mapping is disallowed. + +[[singleton-keyword-params]] +==== Parameters for singleton keyword fields + +The following mapping parameters are accepted: + +[horizontal] + +`value`:: + + The value to associate with all documents in the index. This parameter is + required and can't be updated on an existing index. + diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SingletonKeywordIndexFieldData.java similarity index 89% rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/SingletonKeywordIndexFieldData.java index 29e74bf818afa..73feef26ce7c2 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SingletonKeywordIndexFieldData.java @@ -46,7 +46,7 @@ import java.util.Collections; import java.util.function.Function; -public class ConstantIndexFieldData extends AbstractIndexOrdinalsFieldData { +public class SingletonKeywordIndexFieldData extends AbstractIndexOrdinalsFieldData { public static class Builder implements IndexFieldData.Builder { @@ -59,16 +59,16 @@ public Builder(Function valueFunction) { @Override public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { - return new ConstantIndexFieldData(indexSettings, fieldType.name(), valueFunction.apply(mapperService)); + return new SingletonKeywordIndexFieldData(indexSettings, fieldType.name(), valueFunction.apply(mapperService)); } } - private static class ConstantAtomicFieldData extends AbstractAtomicOrdinalsFieldData { + private static class SingletonKeywordAtomicFieldData extends AbstractAtomicOrdinalsFieldData { private final String value; - ConstantAtomicFieldData(String value) { + SingletonKeywordAtomicFieldData(String value) { super(DEFAULT_SCRIPT_FUNCTION); this.value = value; } @@ -126,14 +126,14 @@ public void close() { } - private final ConstantAtomicFieldData atomicFieldData; + private final SingletonKeywordAtomicFieldData atomicFieldData; - private ConstantIndexFieldData(IndexSettings indexSettings, String name, String value) { + private SingletonKeywordIndexFieldData(IndexSettings indexSettings, String name, String value) { super(indexSettings, name, null, null, TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE); - atomicFieldData = new ConstantAtomicFieldData(value); + atomicFieldData = new SingletonKeywordAtomicFieldData(value); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index c3693f4ded9f3..8f531fb7bb4e5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -21,7 +21,6 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -31,7 +30,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; +import org.elasticsearch.index.fielddata.plain.SingletonKeywordIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; @@ -89,7 +88,7 @@ public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext c } } - static final class IndexFieldType extends MappedFieldType { + static final class IndexFieldType extends SingletonFieldType { IndexFieldType() {} @@ -107,17 +106,6 @@ public String typeName() { return CONTENT_TYPE; } - @Override - public boolean isSearchable() { - // The _index field is always searchable. - return true; - } - - @Override - public Query existsQuery(QueryShardContext context) { - return new MatchAllDocsQuery(); - } - /** * This termQuery impl looks at the context to determine the index that * is being queried and then returns a MATCH_ALL_QUERY or MATCH_NO_QUERY @@ -187,7 +175,7 @@ public Query wildcardQuery(String value, @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { - return new ConstantIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName); + return new SingletonKeywordIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SingletonFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/SingletonFieldType.java new file mode 100644 index 0000000000000..bf12f3f938a85 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/SingletonFieldType.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.index.query.QueryShardContext; + +/** + * A {@link MappedFieldType} that has the same value for all documents. + */ +public abstract class SingletonFieldType extends MappedFieldType { + + public SingletonFieldType() { + super(); + } + + public SingletonFieldType(SingletonFieldType other) { + super(other); + } + + @Override + public final boolean isSearchable() { + return true; + } + + @Override + public final boolean isAggregatable() { + return true; + } + + @Override + public final Query existsQuery(QueryShardContext context) { + return new MatchAllDocsQuery(); + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 63ca066f9ae2d..cfed59d8e91d5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -40,7 +40,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; +import org.elasticsearch.index.fielddata.plain.SingletonKeywordIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; @@ -110,7 +110,7 @@ public String typeName() { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { Function typeFunction = mapperService -> mapperService.documentMapper().type(); - return new ConstantIndexFieldData.Builder(typeFunction); + return new SingletonKeywordIndexFieldData.Builder(typeFunction); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java index db596e2ecfc7b..62da66dde2711 100644 --- a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java @@ -19,20 +19,20 @@ package org.elasticsearch.index.query; -import org.apache.lucene.index.Term; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; -import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.SingletonFieldType; import org.elasticsearch.index.query.support.QueryParsers; import java.io.IOException; @@ -171,14 +171,24 @@ public String getWriteableName() { @Override protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { - if ("_index".equals(fieldName)) { - // Special-case optimisation for canMatch phase: - // We can skip querying this shard if the index name doesn't match the value of this query on the "_index" field. - QueryShardContext shardContext = queryRewriteContext.convertToShardContext(); - if (shardContext != null && shardContext.indexMatches(value + "*") == false) { + QueryShardContext context = queryRewriteContext.convertToShardContext(); + if (context != null) { + MappedFieldType fieldType = context.fieldMapper(this.fieldName); + if (fieldType == null) { return new MatchNoneQueryBuilder(); - } + } else if (fieldType instanceof SingletonFieldType) { + // This logic is correct for all field types, but by only applying it to singleton + // fields we also have the guarantee that it doesn't perform I/O, which is important + // since rewrites might happen on a network thread. + Query query = fieldType.prefixQuery(value, null, context); // the rewrite method doesn't matter + if (query instanceof MatchAllDocsQuery) { + return new MatchAllQueryBuilder().queryName(queryName).boost(boost); + } else if (query instanceof MatchNoDocsQuery) { + return new MatchNoneQueryBuilder(); + } + } } + return super.doRewrite(queryRewriteContext); } @@ -186,20 +196,11 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws protected Query doToQuery(QueryShardContext context) throws IOException { MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null, LoggingDeprecationHandler.INSTANCE); - Query query = null; MappedFieldType fieldType = context.fieldMapper(fieldName); - if (fieldType != null) { - query = fieldType.prefixQuery(value, method, context); + if (fieldType == null) { + throw new IllegalStateException("Rewrite first"); } - if (query == null) { - PrefixQuery prefixQuery = new PrefixQuery(new Term(fieldName, BytesRefs.toBytesRef(value))); - if (method != null) { - prefixQuery.setRewriteMethod(method); - } - query = prefixQuery; - } - - return query; + return fieldType.prefixQuery(value, method, context); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java index 262bfb2c6b5b3..e52f07b43ab44 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java @@ -19,15 +19,15 @@ package org.elasticsearch.index.query; -import org.apache.lucene.index.Term; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.SingletonFieldType; import java.io.IOException; @@ -132,28 +132,33 @@ public static TermQueryBuilder fromXContent(XContentParser parser) throws IOExce @Override protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { - if ("_index".equals(fieldName)) { - // Special-case optimisation for canMatch phase: - // We can skip querying this shard if the index name doesn't match the value of this query on the "_index" field. - QueryShardContext shardContext = queryRewriteContext.convertToShardContext(); - if (shardContext != null && shardContext.indexMatches(BytesRefs.toString(value)) == false) { + QueryShardContext context = queryRewriteContext.convertToShardContext(); + if (context != null) { + MappedFieldType fieldType = context.fieldMapper(this.fieldName); + if (fieldType == null) { return new MatchNoneQueryBuilder(); - } + } else if (fieldType instanceof SingletonFieldType) { + // This logic is correct for all field types, but by only applying it to singleton + // fields we also have the guarantee that it doesn't perform I/O, which is important + // since rewrites might happen on a network thread. + Query query = fieldType.termQuery(value, context); + if (query instanceof MatchAllDocsQuery) { + return new MatchAllQueryBuilder().queryName(queryName).boost(boost); + } else if (query instanceof MatchNoDocsQuery) { + return new MatchNoneQueryBuilder(); + } + } } return super.doRewrite(queryRewriteContext); } @Override protected Query doToQuery(QueryShardContext context) throws IOException { - Query query = null; MappedFieldType mapper = context.fieldMapper(this.fieldName); - if (mapper != null) { - query = mapper.termQuery(this.value, context); - } - if (query == null) { - query = new TermQuery(new Term(this.fieldName, BytesRefs.toBytesRef(this.value))); + if (mapper == null) { + throw new IllegalStateException("Rewrite first"); } - return query; + return mapper.termQuery(this.value, context); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java index 2cececd041b18..362aca701ad92 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java @@ -19,8 +19,9 @@ package org.elasticsearch.index.query; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.SetOnce; @@ -33,13 +34,12 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lucene.BytesRefs; -import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.SingletonFieldType; import org.elasticsearch.indices.TermsLookup; import java.io.IOException; @@ -417,12 +417,9 @@ public String getWriteableName() { @Override protected Query doToQuery(QueryShardContext context) throws IOException { - if (termsLookup != null || supplier != null) { + if (termsLookup != null || supplier != null || values == null || values.isEmpty()) { throw new UnsupportedOperationException("query must be rewritten first"); } - if (values == null || values.isEmpty()) { - return Queries.newMatchNoDocsQuery("No terms supplied for \"" + getName() + "\" query."); - } int maxTermsCount = context.getIndexSettings().getMaxTermsCount(); if (values.size() > maxTermsCount){ throw new IllegalArgumentException( @@ -431,16 +428,10 @@ protected Query doToQuery(QueryShardContext context) throws IOException { IndexSettings.MAX_TERMS_COUNT_SETTING.getKey() + "] index level setting."); } MappedFieldType fieldType = context.fieldMapper(fieldName); - - if (fieldType != null) { - return fieldType.termsQuery(values, context); - } else { - BytesRef[] filterValues = new BytesRef[values.size()]; - for (int i = 0; i < filterValues.length; i++) { - filterValues[i] = BytesRefs.toBytesRef(values.get(i)); - } - return new TermInSetQuery(fieldName, filterValues); + if (fieldType == null) { + throw new IllegalStateException("Rewrite first"); } + return fieldType.termsQuery(values, context); } private void fetch(TermsLookup termsLookup, Client client, ActionListener> actionListener) { @@ -482,21 +473,29 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) { }))); return new TermsQueryBuilder(this.fieldName, supplier::get); } - if ("_index".equals(this.fieldName) && values != null) { - // Special-case optimisation for canMatch phase: - // We can skip querying this shard if the index name doesn't match any of the search terms. - QueryShardContext shardContext = queryRewriteContext.convertToShardContext(); - if (shardContext != null) { - for (Object localValue : values) { - if (shardContext.indexMatches(BytesRefs.toString(localValue))) { - // We can match - at least one index name matches - return this; - } - } - // all index names are invalid - no possibility of a match on this shard. + + if (values == null || values.isEmpty()) { + return new MatchNoneQueryBuilder(); + } + + QueryShardContext context = queryRewriteContext.convertToShardContext(); + if (context != null) { + MappedFieldType fieldType = context.fieldMapper(this.fieldName); + if (fieldType == null) { return new MatchNoneQueryBuilder(); + } else if (fieldType instanceof SingletonFieldType) { + // This logic is correct for all field types, but by only applying it to singleton + // fields we also have the guarantee that it doesn't perform I/O, which is important + // since rewrites might happen on a network thread. + Query query = fieldType.termsQuery(values, context); + if (query instanceof MatchAllDocsQuery) { + return new MatchAllQueryBuilder().queryName(queryName).boost(boost); + } else if (query instanceof MatchNoDocsQuery) { + return new MatchNoneQueryBuilder(); + } } } + return this; } } diff --git a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java index 115fa8d476dfd..fde7abd89b16e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; @@ -27,11 +28,11 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.SingletonFieldType; import org.elasticsearch.index.query.support.QueryParsers; import java.io.IOException; @@ -182,14 +183,24 @@ public static WildcardQueryBuilder fromXContent(XContentParser parser) throws IO @Override protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { - if ("_index".equals(fieldName)) { - // Special-case optimisation for canMatch phase: - // We can skip querying this shard if the index name doesn't match the value of this query on the "_index" field. - QueryShardContext shardContext = queryRewriteContext.convertToShardContext(); - if (shardContext != null && shardContext.indexMatches(BytesRefs.toString(value)) == false) { + QueryShardContext context = queryRewriteContext.convertToShardContext(); + if (context != null) { + MappedFieldType fieldType = context.fieldMapper(this.fieldName); + if (fieldType == null) { return new MatchNoneQueryBuilder(); - } + } else if (fieldType instanceof SingletonFieldType) { + // This logic is correct for all field types, but by only applying it to singleton + // fields we also have the guarantee that it doesn't perform I/O, which is important + // since rewrites might happen on a network thread. + Query query = fieldType.wildcardQuery(value, null, context); // the rewrite method doesn't matter + if (query instanceof MatchAllDocsQuery) { + return new MatchAllQueryBuilder().queryName(queryName).boost(boost); + } else if (query instanceof MatchNoDocsQuery) { + return new MatchNoneQueryBuilder(); + } + } } + return super.doRewrite(queryRewriteContext); } diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java index dba92d712c107..a70c0af4493c5 100644 --- a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java @@ -20,11 +20,13 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; +import org.hamcrest.Matchers; import java.io.IOException; import java.util.HashMap; @@ -68,12 +70,14 @@ private static PrefixQueryBuilder randomPrefixQuery() { @Override protected void doAssertLuceneQuery(PrefixQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { - assertThat(query, instanceOf(PrefixQuery.class)); - PrefixQuery prefixQuery = (PrefixQuery) query; + assertThat(query, Matchers.anyOf(instanceOf(PrefixQuery.class), instanceOf(MatchNoDocsQuery.class))); + if (context.fieldMapper(queryBuilder.fieldName()) != null) { // The field is mapped + PrefixQuery prefixQuery = (PrefixQuery) query; - String expectedFieldName = expectedFieldName(queryBuilder.fieldName()); - assertThat(prefixQuery.getPrefix().field(), equalTo(expectedFieldName)); - assertThat(prefixQuery.getPrefix().text(), equalTo(queryBuilder.value())); + String expectedFieldName = expectedFieldName(queryBuilder.fieldName()); + assertThat(prefixQuery.getPrefix().field(), equalTo(expectedFieldName)); + assertThat(prefixQuery.getPrefix().text(), equalTo(queryBuilder.value())); + } } public void testIllegalArguments() { @@ -88,10 +92,10 @@ public void testIllegalArguments() { public void testBlendedRewriteMethod() throws IOException { String rewrite = "top_terms_blended_freqs_10"; - Query parsedQuery = parseQuery(prefixQuery("field", "val").rewrite(rewrite)).toQuery(createShardContext()); + Query parsedQuery = parseQuery(prefixQuery(STRING_FIELD_NAME, "val").rewrite(rewrite)).toQuery(createShardContext()); assertThat(parsedQuery, instanceOf(PrefixQuery.class)); PrefixQuery prefixQuery = (PrefixQuery) parsedQuery; - assertThat(prefixQuery.getPrefix(), equalTo(new Term("field", "val"))); + assertThat(prefixQuery.getPrefix(), equalTo(new Term(STRING_FIELD_NAME, "val"))); assertThat(prefixQuery.getRewriteMethod(), instanceOf(MultiTermQuery.TopTermsBlendedFreqScoringRewrite.class)); } @@ -153,7 +157,7 @@ public void testRewriteIndexQueryToNotMatchNone() throws Exception { PrefixQueryBuilder query = prefixQuery("_index", getIndex().getName()); QueryShardContext queryShardContext = createShardContext(); QueryBuilder rewritten = query.rewrite(queryShardContext); - assertThat(rewritten, instanceOf(PrefixQueryBuilder.class)); + assertThat(rewritten, instanceOf(MatchAllQueryBuilder.class)); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java index 0bf6ddbc57438..b3d782b4c2f0f 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java @@ -21,11 +21,11 @@ import com.fasterxml.jackson.core.io.JsonStringEncoder; import org.apache.lucene.index.Term; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.mapper.MappedFieldType; import java.io.IOException; @@ -91,7 +91,7 @@ protected TermQueryBuilder createQueryBuilder(String fieldName, Object value) { @Override protected void doAssertLuceneQuery(TermQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { - assertThat(query, either(instanceOf(TermQuery.class)).or(instanceOf(PointRangeQuery.class))); + assertThat(query, either(instanceOf(TermQuery.class)).or(instanceOf(PointRangeQuery.class)).or(instanceOf(MatchNoDocsQuery.class))); MappedFieldType mapper = context.fieldMapper(queryBuilder.fieldName()); if (query instanceof TermQuery) { TermQuery termQuery = (TermQuery) query; @@ -99,14 +99,12 @@ protected void doAssertLuceneQuery(TermQueryBuilder queryBuilder, Query query, Q String expectedFieldName = expectedFieldName(queryBuilder.fieldName()); assertThat(termQuery.getTerm().field(), equalTo(expectedFieldName)); - if (mapper != null) { - Term term = ((TermQuery) mapper.termQuery(queryBuilder.value(), null)).getTerm(); - assertThat(termQuery.getTerm(), equalTo(term)); - } else { - assertThat(termQuery.getTerm().bytes(), equalTo(BytesRefs.toBytesRef(queryBuilder.value()))); - } - } else { + Term term = ((TermQuery) mapper.termQuery(queryBuilder.value(), null)).getTerm(); + assertThat(termQuery.getTerm(), equalTo(term)); + } else if (mapper != null) { assertEquals(query, mapper.termQuery(queryBuilder.value(), null)); + } else { + assertThat(query, instanceOf(MatchNoDocsQuery.class)); } } @@ -185,6 +183,21 @@ public void testRewriteIndexQueryToNotMatchNone() throws IOException { TermQueryBuilder query = QueryBuilders.termQuery("_index", getIndex().getName()); QueryShardContext queryShardContext = createShardContext(); QueryBuilder rewritten = query.rewrite(queryShardContext); - assertThat(rewritten, instanceOf(TermQueryBuilder.class)); - } + assertThat(rewritten, instanceOf(MatchAllQueryBuilder.class)); + } + + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + TermQueryBuilder queryBuilder = createTestQueryBuilder(); + if (context.fieldMapper(queryBuilder.fieldName()) == null) { + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } else { + // no exception + queryBuilder.toQuery(context); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java index b4a42e65e5f8a..13ab9a0b62463 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java @@ -108,8 +108,6 @@ private TermsLookup randomTermsLookup() { protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { if (queryBuilder.termsLookup() == null && (queryBuilder.values() == null || queryBuilder.values().isEmpty())) { assertThat(query, instanceOf(MatchNoDocsQuery.class)); - MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query; - assertThat(matchNoDocsQuery.toString(), containsString("No terms supplied for \"terms\" query.")); } else if (queryBuilder.termsLookup() != null && randomTerms.size() == 0){ assertThat(query, instanceOf(MatchNoDocsQuery.class)); MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query; @@ -117,7 +115,8 @@ protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, } else { assertThat(query, either(instanceOf(TermInSetQuery.class)) .or(instanceOf(PointInSetQuery.class)) - .or(instanceOf(ConstantScoreQuery.class))); + .or(instanceOf(ConstantScoreQuery.class)) + .or(instanceOf(MatchNoDocsQuery.class))); if (query instanceof ConstantScoreQuery) { assertThat(((ConstantScoreQuery) query).getQuery(), instanceOf(BooleanQuery.class)); } @@ -137,8 +136,13 @@ protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, } String fieldName = expectedFieldName(queryBuilder.fieldName()); - TermInSetQuery expected = new TermInSetQuery(fieldName, - terms.stream().filter(Objects::nonNull).map(Object::toString).map(BytesRef::new).collect(Collectors.toList())); + Query expected; + if (context.fieldMapper(fieldName) != null) { + expected = new TermInSetQuery(fieldName, + terms.stream().filter(Objects::nonNull).map(Object::toString).map(BytesRef::new).collect(Collectors.toList())); + } else { + expected = new MatchNoDocsQuery(); + } assertEquals(expected, query); } } @@ -263,8 +267,16 @@ public void testMustRewrite() throws IOException { UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> termsQueryBuilder.toQuery(createShardContext())); assertEquals("query must be rewritten first", e.getMessage()); - assertEquals(rewriteAndFetch(termsQueryBuilder, createShardContext()), new TermsQueryBuilder(STRING_FIELD_NAME, - randomTerms.stream().filter(x -> x != null).collect(Collectors.toList()))); // terms lookup removes null values + + // terms lookup removes null values + List nonNullTerms = randomTerms.stream().filter(x -> x != null).collect(Collectors.toList()); + QueryBuilder expected; + if (nonNullTerms.isEmpty()) { + expected = new MatchNoneQueryBuilder(); + } else { + expected = new TermsQueryBuilder(STRING_FIELD_NAME, nonNullTerms); + } + assertEquals(expected, rewriteAndFetch(termsQueryBuilder, createShardContext())); } public void testGeo() throws Exception { @@ -325,7 +337,7 @@ public void testRewriteIndexQueryToNotMatchNone() throws IOException { TermsQueryBuilder query = new TermsQueryBuilder("_index", "does_not_exist", getIndex().getName()); QueryShardContext queryShardContext = createShardContext(); QueryBuilder rewritten = query.rewrite(queryShardContext); - assertThat(rewritten, instanceOf(TermsQueryBuilder.class)); + assertThat(rewritten, instanceOf(MatchAllQueryBuilder.class)); } @Override @@ -334,4 +346,5 @@ protected QueryBuilder parseQuery(XContentParser parser) throws IOException { assertThat(query, CoreMatchers.instanceOf(TermsQueryBuilder.class)); return query; } + } diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java index bf88ab9ee2da6..148875e3b1303 100644 --- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java @@ -152,6 +152,6 @@ public void testRewriteIndexQueryNotMatchNone() throws IOException { WildcardQueryBuilder query = new WildcardQueryBuilder("_index", firstHalfOfIndexName +"*"); QueryShardContext queryShardContext = createShardContext(); QueryBuilder rewritten = query.rewrite(queryShardContext); - assertThat(rewritten, instanceOf(WildcardQueryBuilder.class)); + assertThat(rewritten, instanceOf(MatchAllQueryBuilder.class)); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 1b20ceae9233e..ee9b4760bfa35 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -36,12 +36,14 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.LicensesMetaData; import org.elasticsearch.license.Licensing; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.ExtensiblePlugin; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.protocol.xpack.XPackInfoResponse; @@ -60,6 +62,7 @@ import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.action.XPackUsageResponse; +import org.elasticsearch.xpack.core.index.mapper.SingletonKeywordFieldMapper; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.rest.action.RestReloadAnalyzersAction; import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction; @@ -84,7 +87,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; -public class XPackPlugin extends XPackClientPlugin implements ExtensiblePlugin, RepositoryPlugin, EnginePlugin { +public class XPackPlugin extends XPackClientPlugin implements ExtensiblePlugin, RepositoryPlugin, EnginePlugin, MapperPlugin { private static Logger logger = LogManager.getLogger(XPackPlugin.class); private static DeprecationLogger deprecationLogger = new DeprecationLogger(logger); @@ -331,4 +334,9 @@ public List> getSettings() { settings.add(SourceOnlySnapshotRepository.SOURCE_ONLY); return settings; } + + @Override + public Map getMappers() { + return Collections.singletonMap(SingletonKeywordFieldMapper.CONTENT_TYPE, new SingletonKeywordFieldMapper.TypeParser()); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapper.java new file mode 100644 index 0000000000000..1ca77644bb58a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapper.java @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +package org.elasticsearch.xpack.core.index.mapper; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.plain.SingletonKeywordIndexFieldData; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.SingletonFieldType; +import org.elasticsearch.index.query.QueryShardContext; + +public class SingletonKeywordFieldMapper extends FieldMapper { + + public static final String CONTENT_TYPE = "singleton_keyword"; + + public static class Defaults { + public static final String NULL_VALUE = null; + public static final int IGNORE_ABOVE = Integer.MAX_VALUE; + } + + public static class Builder extends FieldMapper.Builder { + + private static MappedFieldType createFieldType(String value) { + SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ft.setValue(value); + ft.freeze(); + return ft; + } + + private Builder(String name, MappedFieldType fieldType) { + super(name, fieldType, fieldType); + builder = this; + } + + public Builder(String name, String value) { + this(name, createFieldType(value)); + } + + @Override + public SingletonKeywordFieldType fieldType() { + return (SingletonKeywordFieldType) super.fieldType(); + } + + @Override + public SingletonKeywordFieldMapper build(BuilderContext context) { + setupFieldType(context); + return new SingletonKeywordFieldMapper( + name, fieldType, defaultFieldType, + context.indexSettings()); + } + } + + public static class TypeParser implements Mapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + final Object value = node.remove("value"); + if (value == null) { + throw new MapperParsingException("Property [value] of field [" + name + "] is required and can't be null."); + } + if (value instanceof Number == false && value instanceof CharSequence == false) { + throw new MapperParsingException("Property [value] of field [" + name + + "] must be a number or a string, but got [" + value + "]"); + } + return new SingletonKeywordFieldMapper.Builder(name, value.toString()); + } + } + + public static final class SingletonKeywordFieldType extends SingletonFieldType { + + private String value; + + public SingletonKeywordFieldType() { + super(); + } + + protected SingletonKeywordFieldType(SingletonKeywordFieldType ref) { + super(ref); + this.value = ref.value; + } + + public SingletonKeywordFieldType clone() { + return new SingletonKeywordFieldType(this); + } + + @Override + public boolean equals(Object o) { + if (super.equals(o) == false) { + return false; + } + SingletonKeywordFieldType other = (SingletonKeywordFieldType) o; + return Objects.equals(value, other.value); + } + + @Override + public void checkCompatibility(MappedFieldType otherFT, List conflicts) { + super.checkCompatibility(otherFT, conflicts); + SingletonKeywordFieldType other = (SingletonKeywordFieldType) otherFT; + if (Objects.equals(value, other.value) == false) { + conflicts.add("mapper [" + name() + "] has different [value]"); + } + } + + @Override + public int hashCode() { + return 31 * super.hashCode() + Objects.hashCode(value); + } + + /** Return the value that this field wraps. */ + public String value() { + return value; + } + + /** Set the value. */ + public void setValue(String value) { + checkIfFrozen(); + this.value = Objects.requireNonNull(value); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + return new SingletonKeywordIndexFieldData.Builder(mapperService -> value); + } + + private static String valueToString(Object v) { + if (v instanceof BytesRef) { + return ((BytesRef) v).utf8ToString(); + } else { + return v.toString(); + } + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + if (Objects.equals(valueToString(value), this.value)) { + return new MatchAllDocsQuery(); + } else { + return new MatchNoDocsQuery(); + } + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + for (Object v : values) { + if (Objects.equals(valueToString(v), value)) { + return new MatchAllDocsQuery(); + } + } + return new MatchNoDocsQuery(); + } + + @Override + public Query prefixQuery(String value, + @Nullable MultiTermQuery.RewriteMethod method, + QueryShardContext context) { + if (this.value.startsWith(value)) { + return new MatchAllDocsQuery(); + } else { + return new MatchNoDocsQuery(); + } + } + + public Query wildcardQuery(String value, + @Nullable MultiTermQuery.RewriteMethod method, + QueryShardContext context) { + if (Regex.simpleMatch(value, this.value)) { + return new MatchAllDocsQuery(); + } else { + return new MatchNoDocsQuery(); + } + } + } + + protected SingletonKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + Settings indexSettings) { + super(simpleName, fieldType, defaultFieldType, indexSettings, MultiFields.empty(), CopyTo.empty()); + } + + @Override + protected SingletonKeywordFieldMapper clone() { + return (SingletonKeywordFieldMapper) super.clone(); + } + + @Override + public SingletonKeywordFieldType fieldType() { + return (SingletonKeywordFieldType) super.fieldType(); + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + String value; + if (context.externalValueSet()) { + value = context.externalValue().toString(); + } else { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + value = fieldType().nullValueAsString(); + } else { + value = parser.textOrNull(); + } + } + + if (Objects.equals(fieldType().value, value) == false) { + throw new IllegalArgumentException("[singleton_keyword] field [" + name() + + "] only accepts values that are equal to the wrapped value [" + fieldType().value() + "], but got [" + value + "]"); + } + } + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + builder.field("value", fieldType().value()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapperTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapperTests.java new file mode 100644 index 0000000000000..9dec3a14810a4 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapperTests.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.index.mapper; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.XPackPlugin; + +public class SingletonKeywordFieldMapperTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + List> plugins = new ArrayList<>(super.getPlugins()); + plugins.add(XPackPlugin.class); + return plugins; + } + + public void testDefaults() throws Exception { + IndexService indexService = createIndex("test"); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "singleton_keyword") + .field("value", "foo").endObject().endObject().endObject().endObject()); + DocumentMapper mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()); + ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", source, XContentType.JSON)); + assertNull(doc.rootDoc().getField("field")); + + source = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "foo").endObject()); + doc = mapper.parse(new SourceToParse("test", "1", source, XContentType.JSON)); + assertNull(doc.rootDoc().getField("field")); + + BytesReference illegalSource = BytesReference.bytes(XContentFactory.jsonBuilder() + .startObject().field("field", "bar").endObject()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> mapper.parse(new SourceToParse("test", "1", illegalSource, XContentType.JSON))); + assertEquals("[singleton_keyword] field [field] only accepts values that are equal to the wrapped value [foo], but got [bar]", + e.getCause().getMessage()); + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldTypeTests.java new file mode 100644 index 0000000000000..582ee2d6bbec6 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldTypeTests.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.index.mapper; + +import java.util.Arrays; +import java.util.Collections; + +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.xpack.core.index.mapper.SingletonKeywordFieldMapper.SingletonKeywordFieldType; +import org.junit.Before; + +public class SingletonKeywordFieldTypeTests extends FieldTypeTestCase { + + @Before + public void setupProperties() { + addModifier(new Modifier("value", false) { + @Override + public void modify(MappedFieldType type) { + ((SingletonKeywordFieldType) type).setValue("bar"); + } + }); + } + + @Override + protected MappedFieldType createDefaultFieldType() { + return new SingletonKeywordFieldType(); + } + + public void testTermQuery() { + SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ft.setValue("foo"); + assertEquals(new MatchAllDocsQuery(), ft.termQuery("foo", null)); + assertEquals(new MatchNoDocsQuery(), ft.termQuery("bar", null)); + } + + public void testTermsQuery() { + SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ft.setValue("foo"); + assertEquals(new MatchAllDocsQuery(), ft.termsQuery(Collections.singletonList("foo"), null)); + assertEquals(new MatchAllDocsQuery(), ft.termsQuery(Arrays.asList("bar", "foo", "quux"), null)); + assertEquals(new MatchNoDocsQuery(), ft.termsQuery(Collections.emptyList(), null)); + assertEquals(new MatchNoDocsQuery(), ft.termsQuery(Collections.singletonList("bar"), null)); + assertEquals(new MatchNoDocsQuery(), ft.termsQuery(Arrays.asList("bar", "quux"), null)); + } + + public void testWildcardQuery() { + SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ft.setValue("foo"); + assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("f*o", null, null)); + assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("b*r", null, null)); + } + + public void testPrefixQuery() { + SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ft.setValue("foo"); + assertEquals(new MatchAllDocsQuery(), ft.prefixQuery("fo", null, null)); + assertEquals(new MatchNoDocsQuery(), ft.prefixQuery("ba", null, null)); + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/singleton_keyword/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/singleton_keyword/10_basic.yml new file mode 100644 index 0000000000000..495130cf7b263 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/singleton_keyword/10_basic.yml @@ -0,0 +1,169 @@ +setup: + + - skip: + version: " - 7.99.99" + reason: "singleton_keyword was added in 7.6" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + foo: + type: singleton_keyword + value: bar + + - do: + indices.create: + index: test2 + body: + mappings: + properties: + foo: + type: singleton_keyword + value: baz + + - do: + index: + index: test1 + id: 1 + body: {} + + - do: + index: + index: test1 + id: 2 + body: { "foo": "bar" } + + - do: + index: + index: test2 + id: 1 + body: {} + + - do: + indices.refresh: {} + +--- +"Exist query": + + - do: + search: + index: test* + body: + size: 0 + query: + exists: + field: foo + + - match: { "hits.total.value": 3 } + + +--- +"Term query": + + - do: + search: + index: test* + pre_filter_shard_size: 1 + body: + size: 0 + query: + term: + foo: bar + + - match: { "hits.total.value": 2 } + - match: { _shards.skipped : 1} + + - do: + search: + index: test* + pre_filter_shard_size: 1 + body: + size: 0 + query: + term: + foo: baz + + - match: { "hits.total.value": 1 } + - match: { _shards.skipped : 1} + +--- +"Terms query": + + - do: + search: + index: test* + pre_filter_shard_size: 1 + body: + size: 0 + query: + terms: + foo: [bar, quux] + + - match: { "hits.total.value": 2 } + - match: { _shards.skipped : 1} + +--- +"Prefix query": + + - do: + search: + index: test* + body: + size: 0 + query: + prefix: + foo: ba + + - match: { "hits.total.value": 3 } + + - do: + search: + index: test* + pre_filter_shard_size: 1 + body: + size: 0 + query: + prefix: + foo: baz + + - match: { "hits.total.value": 1 } + - match: { _shards.skipped : 1} + +--- +"Wildcard query": + + - do: + search: + index: test* + pre_filter_shard_size: 1 + body: + size: 0 + query: + wildcard: + foo: "*r*" + + - match: { "hits.total.value": 2 } + - match: { _shards.skipped : 1} + +--- +"Terms agg": + + - do: + search: + index: test* + body: + size: 0 + aggs: + foo_terms: + terms: + field: foo + + - match: { aggregations.foo_terms.buckets.0.key: "bar" } + - match: { aggregations.foo_terms.buckets.0.doc_count: 2 } + - match: { aggregations.foo_terms.buckets.1.key: "baz" } + - match: { aggregations.foo_terms.buckets.1.doc_count: 1 } + - length: { aggregations.foo_terms.buckets: 2 } + From c253ce758841a9561b4a002bca95bfce2d3aa078 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 29 Nov 2019 16:19:19 +0100 Subject: [PATCH 02/21] iter --- .../percolator/PercolatorFieldMapper.java | 53 ++++++++++-------- .../percolator/QueryBuilderStoreTests.java | 10 ++++ .../index/query/AbstractQueryBuilder.java | 5 +- .../SignificantTermsAggregatorFactory.java | 3 +- .../index/query/BoolQueryBuilderTests.java | 55 ++++++++++++------- .../query/BoostingQueryBuilderTests.java | 4 +- .../query/ConstantScoreQueryBuilderTests.java | 5 +- .../index/query/PrefixQueryBuilderTests.java | 14 +++++ .../query/WildcardQueryBuilderTests.java | 17 +++++- .../index/query/WrapperQueryBuilderTests.java | 12 ++-- .../FunctionScoreQueryBuilderTests.java | 23 ++++---- .../bucket/terms/TermsAggregatorTests.java | 4 +- .../aggregations/AggregatorTestCase.java | 4 +- .../test/AbstractQueryTestCase.java | 13 +++-- .../DocumentSubsetBitsetCacheTests.java | 1 + ...ityIndexReaderWrapperIntegrationTests.java | 2 + 16 files changed, 149 insertions(+), 76 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 501d71465d679..f90ed1449be13 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -384,9 +384,34 @@ public FieldMapper updateFieldType(Map fullNameToFieldT return updated; } + private QueryShardContext getQueryShardContext() { + QueryShardContext originarQueryShardContext = this.queryShardContext.get(); + QueryShardContext queryShardContext = new QueryShardContext(originarQueryShardContext) { + @Override + public boolean convertNowRangeToMatchAll() { + return true; + } + }; + + // This means that fields in the query need to exist in the mapping prior to registering this query + // The reason that this is required, is that if a field doesn't exist then the query assumes defaults, which may be undesired. + // + // Even worse when fields mentioned in percolator queries do go added to map after the queries have been registered + // then the percolator queries don't work as expected any more. + // + // Query parsing can't introduce new fields in mappings (which happens when registering a percolator query), + // because field type can't be inferred from queries (like document do) so the best option here is to disallow + // the usage of unmapped fields in percolator queries to avoid unexpected behaviour + // + // if index.percolator.map_unmapped_fields_as_string is set to true, query can contain unmapped fields which will be mapped + // as an analyzed string. + queryShardContext.setAllowUnmappedFields(false); + queryShardContext.setMapUnmappedFieldAsString(isMapUnmappedFieldAsText()); + return queryShardContext; + } + @Override public void parse(ParseContext context) throws IOException { - QueryShardContext queryShardContext = this.queryShardContext.get(); if (context.doc().getField(queryBuilderField.name()) != null) { // If a percolator query has been defined in an array object then multiple percolator queries // could be provided. In order to prevent this we fail if we try to parse more than one query @@ -399,6 +424,8 @@ public void parse(ParseContext context) throws IOException { parser, parser.getTokenLocation() ); verifyQuery(queryBuilder); + + QueryShardContext queryShardContext = getQueryShardContext(); // Fetching of terms, shapes and indexed scripts happen during this rewrite: PlainActionFuture future = new PlainActionFuture<>(); Rewriteable.rewriteAndFetch(queryBuilder, queryShardContext, future); @@ -414,7 +441,7 @@ public boolean convertNowRangeToMatchAll() { return true; } }); - Query query = toQuery(queryShardContext, isMapUnmappedFieldAsText(), queryBuilderForProcessing); + Query query = queryBuilderForProcessing.toQuery(queryShardContext); processQuery(query, context); } @@ -472,28 +499,6 @@ void processQuery(Query query, ParseContext context) { doc.add(new NumericDocValuesField(minimumShouldMatchFieldMapper.name(), result.minimumShouldMatch)); } - static Query parseQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, XContentParser parser) throws IOException { - return toQuery(context, mapUnmappedFieldsAsString, parseQueryBuilder(parser, parser.getTokenLocation())); - } - - static Query toQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, QueryBuilder queryBuilder) throws IOException { - // This means that fields in the query need to exist in the mapping prior to registering this query - // The reason that this is required, is that if a field doesn't exist then the query assumes defaults, which may be undesired. - // - // Even worse when fields mentioned in percolator queries do go added to map after the queries have been registered - // then the percolator queries don't work as expected any more. - // - // Query parsing can't introduce new fields in mappings (which happens when registering a percolator query), - // because field type can't be inferred from queries (like document do) so the best option here is to disallow - // the usage of unmapped fields in percolator queries to avoid unexpected behaviour - // - // if index.percolator.map_unmapped_fields_as_string is set to true, query can contain unmapped fields which will be mapped - // as an analyzed string. - context.setAllowUnmappedFields(false); - context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString); - return queryBuilder.toQuery(context); - } - private static QueryBuilder parseQueryBuilder(XContentParser parser, XContentLocation location) { try { return parseInnerQueryBuilder(parser); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index 88c2a098deb21..5026d17137123 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -38,10 +38,15 @@ import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; +import org.elasticsearch.index.mapper.TextFieldTypeTests; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; @@ -93,6 +98,11 @@ public void testStoringQueryBuilders() throws IOException { when(queryShardContext.getXContentRegistry()).thenReturn(xContentRegistry()); when(queryShardContext.getForField(fieldMapper.fieldType())) .thenReturn(new BytesBinaryDVIndexFieldData(new Index("index", "uuid"), fieldMapper.name())); + when(queryShardContext.fieldMapper(Mockito.anyString())).thenAnswer(invocation -> { + TextFieldType fieldType = new TextFieldType(); + fieldType.setName((String) invocation.getArguments()[0]); + return fieldType; + }); PercolateQuery.QueryStore queryStore = PercolateQueryBuilder.createStore(fieldMapper.fieldType(), queryShardContext, false); try (IndexReader indexReader = DirectoryReader.open(directory)) { diff --git a/server/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java index def32f0c75059..7ac62f1d33531 100644 --- a/server/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.spans.SpanBoostQuery; import org.apache.lucene.search.spans.SpanQuery; @@ -101,7 +102,7 @@ public final Query toQuery(QueryShardContext context) throws IOException { if (boost != DEFAULT_BOOST) { if (query instanceof SpanQuery) { query = new SpanBoostQuery((SpanQuery) query, boost); - } else { + } else if (query instanceof MatchNoDocsQuery == false) { query = new BoostQuery(query, boost); } } @@ -230,7 +231,7 @@ static Collection toQueries(Collection queryBuilders, Query IOException { List queries = new ArrayList<>(queryBuilders.size()); for (QueryBuilder queryBuilder : queryBuilders) { - Query query = queryBuilder.toQuery(context); + Query query = queryBuilder.rewrite(context).toQuery(context); if (query != null) { queries.add(query); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java index 0687c81a64802..bcf94f01d4d72 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java @@ -37,6 +37,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; @@ -97,7 +98,7 @@ public SignificantTermsAggregatorFactory(String name, this.executionHint = executionHint; this.filter = filterBuilder == null ? null - : filterBuilder.toQuery(queryShardContext); + : Rewriteable.rewrite(filterBuilder, queryShardContext).toQuery(queryShardContext); IndexSearcher searcher = queryShardContext.searcher(); this.supersetNumDocs = filter == null // Important - need to use the doc count that includes deleted docs diff --git a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java index 1a1609fe7c50e..e00888a1aa2d6 100644 --- a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -85,7 +86,7 @@ protected void doAssertLuceneQuery(BoolQueryBuilder queryBuilder, Query query, Q if (clauses.isEmpty()) { assertThat(query, instanceOf(MatchAllDocsQuery.class)); - } else { + } else if (query instanceof MatchNoDocsQuery == false) { assertThat(query, instanceOf(BooleanQuery.class)); BooleanQuery booleanQuery = (BooleanQuery) query; if (queryBuilder.adjustPureNegative()) { @@ -113,7 +114,7 @@ private static List getBooleanClauses(List queryBui BooleanClause.Occur occur, QueryShardContext context) throws IOException { List clauses = new ArrayList<>(); for (QueryBuilder query : queryBuilders) { - Query innerQuery = query.toQuery(context); + Query innerQuery = query.rewrite(context).toQuery(context); if (innerQuery != null) { clauses.add(new BooleanClause(innerQuery, occur)); } @@ -195,15 +196,15 @@ public void testMinShouldMatchFilterWithoutShouldClauses() throws Exception { public void testMinShouldMatchBiggerThanNumberOfShouldClauses() throws Exception { BooleanQuery bq = (BooleanQuery) parseQuery( boolQuery() - .should(termQuery("foo", "bar")) - .should(termQuery("foo2", "bar2")) + .should(termQuery(STRING_FIELD_NAME, "bar")) + .should(termQuery(STRING_FIELD_NAME_2, "bar2")) .minimumShouldMatch("3")).toQuery(createShardContext()); assertEquals(3, bq.getMinimumNumberShouldMatch()); bq = (BooleanQuery) parseQuery( boolQuery() - .should(termQuery("foo", "bar")) - .should(termQuery("foo2", "bar2")) + .should(termQuery(STRING_FIELD_NAME, "bar")) + .should(termQuery(STRING_FIELD_NAME_2, "bar2")) .minimumShouldMatch(3)).toQuery(createShardContext()); assertEquals(3, bq.getMinimumNumberShouldMatch()); } @@ -211,8 +212,8 @@ public void testMinShouldMatchBiggerThanNumberOfShouldClauses() throws Exception public void testMinShouldMatchDisableCoord() throws Exception { BooleanQuery bq = (BooleanQuery) parseQuery( boolQuery() - .should(termQuery("foo", "bar")) - .should(termQuery("foo2", "bar2")) + .should(termQuery(STRING_FIELD_NAME, "bar")) + .should(termQuery(STRING_FIELD_NAME, "bar2")) .minimumShouldMatch("3")).toQuery(createShardContext()); assertEquals(3, bq.getMinimumNumberShouldMatch()); } @@ -292,22 +293,22 @@ public void testRewrite() throws IOException { boolean mustRewrite = false; if (randomBoolean()) { mustRewrite = true; - boolQueryBuilder.must(new WrapperQueryBuilder(new TermsQueryBuilder("foo", "must").toString())); + boolQueryBuilder.must(new WrapperQueryBuilder(new TermsQueryBuilder(STRING_FIELD_NAME, "must").toString())); } if (randomBoolean()) { mustRewrite = true; - boolQueryBuilder.should(new WrapperQueryBuilder(new TermsQueryBuilder("foo", "should").toString())); + boolQueryBuilder.should(new WrapperQueryBuilder(new TermsQueryBuilder(STRING_FIELD_NAME, "should").toString())); } if (randomBoolean()) { mustRewrite = true; - boolQueryBuilder.filter(new WrapperQueryBuilder(new TermsQueryBuilder("foo", "filter").toString())); + boolQueryBuilder.filter(new WrapperQueryBuilder(new TermsQueryBuilder(STRING_FIELD_NAME, "filter").toString())); } if (randomBoolean()) { mustRewrite = true; - boolQueryBuilder.mustNot(new WrapperQueryBuilder(new TermsQueryBuilder("foo", "must_not").toString())); + boolQueryBuilder.mustNot(new WrapperQueryBuilder(new TermsQueryBuilder(STRING_FIELD_NAME, "must_not").toString())); } if (mustRewrite == false && randomBoolean()) { - boolQueryBuilder.must(new TermsQueryBuilder("foo", "no_rewrite")); + boolQueryBuilder.must(new TermsQueryBuilder(STRING_FIELD_NAME, "no_rewrite")); } QueryBuilder rewritten = boolQueryBuilder.rewrite(createShardContext()); if (mustRewrite == false && boolQueryBuilder.must().isEmpty()) { @@ -318,16 +319,16 @@ public void testRewrite() throws IOException { if (mustRewrite) { assertNotSame(rewrite, boolQueryBuilder); if (boolQueryBuilder.must().isEmpty() == false) { - assertEquals(new TermsQueryBuilder("foo", "must"), rewrite.must().get(0)); + assertEquals(new TermsQueryBuilder(STRING_FIELD_NAME, "must"), rewrite.must().get(0)); } if (boolQueryBuilder.should().isEmpty() == false) { - assertEquals(new TermsQueryBuilder("foo", "should"), rewrite.should().get(0)); + assertEquals(new TermsQueryBuilder(STRING_FIELD_NAME, "should"), rewrite.should().get(0)); } if (boolQueryBuilder.mustNot().isEmpty() == false) { - assertEquals(new TermsQueryBuilder("foo", "must_not"), rewrite.mustNot().get(0)); + assertEquals(new TermsQueryBuilder(STRING_FIELD_NAME, "must_not"), rewrite.mustNot().get(0)); } if (boolQueryBuilder.filter().isEmpty() == false) { - assertEquals(new TermsQueryBuilder("foo", "filter"), rewrite.filter().get(0)); + assertEquals(new TermsQueryBuilder(STRING_FIELD_NAME, "filter"), rewrite.filter().get(0)); } } else { assertSame(rewrite, boolQueryBuilder); @@ -360,14 +361,14 @@ public void testRewriteWithMatchNone() throws IOException { assertEquals(new MatchNoneQueryBuilder(), rewritten); boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.must(new TermQueryBuilder("foo","bar")); + boolQueryBuilder.must(new TermQueryBuilder(STRING_FIELD_NAME,"bar")); boolQueryBuilder.filter(new WrapperQueryBuilder(new WrapperQueryBuilder(new MatchNoneQueryBuilder().toString()).toString())); rewritten = boolQueryBuilder.rewrite(createShardContext()); assertEquals(new MatchNoneQueryBuilder(), rewritten); boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.must(new TermQueryBuilder("foo","bar")); - boolQueryBuilder.filter(new BoolQueryBuilder().should(new TermQueryBuilder("foo","bar")) + boolQueryBuilder.must(new TermQueryBuilder(STRING_FIELD_NAME,"bar")); + boolQueryBuilder.filter(new BoolQueryBuilder().should(new TermQueryBuilder(STRING_FIELD_NAME,"bar")) .filter(new MatchNoneQueryBuilder())); rewritten = Rewriteable.rewrite(boolQueryBuilder, createShardContext()); assertEquals(new MatchNoneQueryBuilder(), rewritten); @@ -378,7 +379,7 @@ public void testRewriteWithMatchNone() throws IOException { assertEquals(new MatchNoneQueryBuilder(), rewritten); boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.should(new TermQueryBuilder("foo", "bar")); + boolQueryBuilder.should(new TermQueryBuilder(STRING_FIELD_NAME, "bar")); boolQueryBuilder.should(new WrapperQueryBuilder(new MatchNoneQueryBuilder().toString())); rewritten = Rewriteable.rewrite(boolQueryBuilder, createShardContext()); assertNotEquals(new MatchNoneQueryBuilder(), rewritten); @@ -387,4 +388,16 @@ public void testRewriteWithMatchNone() throws IOException { rewritten = Rewriteable.rewrite(boolQueryBuilder, createShardContext()); assertNotEquals(new MatchNoneQueryBuilder(), rewritten); } + + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + TermQueryBuilder termQuery = new TermQueryBuilder("unmapped_field", 42); + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + boolQuery.must(termQuery); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> boolQuery.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java index 534126ee5f35a..0d859392366ce 100644 --- a/server/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java @@ -40,8 +40,8 @@ protected BoostingQueryBuilder doCreateTestQueryBuilder() { @Override protected void doAssertLuceneQuery(BoostingQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { - Query positive = queryBuilder.positiveQuery().toQuery(context); - Query negative = queryBuilder.negativeQuery().toQuery(context); + Query positive = queryBuilder.positiveQuery().rewrite(context).toQuery(context); + Query negative = queryBuilder.negativeQuery().rewrite(context).toQuery(context); if (positive == null || negative == null) { assertThat(query, nullValue()); } else { diff --git a/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java index fd2ad04af5f95..3280aeecdda61 100644 --- a/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.AbstractQueryTestCase; @@ -41,9 +42,11 @@ protected ConstantScoreQueryBuilder doCreateTestQueryBuilder() { @Override protected void doAssertLuceneQuery(ConstantScoreQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { - Query innerQuery = queryBuilder.innerQuery().toQuery(context); + Query innerQuery = queryBuilder.innerQuery().rewrite(context).toQuery(context); if (innerQuery == null) { assertThat(query, nullValue()); + } else if (innerQuery instanceof MatchNoDocsQuery) { + assertThat(query, instanceOf(MatchNoDocsQuery.class)); } else { assertThat(query, instanceOf(ConstantScoreQuery.class)); ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query; diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java index a70c0af4493c5..b7f7ef132f1d0 100644 --- a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java @@ -160,4 +160,18 @@ public void testRewriteIndexQueryToNotMatchNone() throws Exception { assertThat(rewritten, instanceOf(MatchAllQueryBuilder.class)); } + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + PrefixQueryBuilder queryBuilder = createTestQueryBuilder(); + if (context.fieldMapper(queryBuilder.fieldName()) == null) { + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } else { + // no exception + queryBuilder.toQuery(context); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java index 148875e3b1303..d0bfbfefe9341 100644 --- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java @@ -153,5 +153,20 @@ public void testRewriteIndexQueryNotMatchNone() throws IOException { QueryShardContext queryShardContext = createShardContext(); QueryBuilder rewritten = query.rewrite(queryShardContext); assertThat(rewritten, instanceOf(MatchAllQueryBuilder.class)); - } + } + + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + WildcardQueryBuilder queryBuilder = createTestQueryBuilder(); + if (context.fieldMapper(queryBuilder.fieldName()) == null) { + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } else { + // no exception + queryBuilder.toQuery(context); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java index d5ccff8402420..ad607a561fbce 100644 --- a/server/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java @@ -119,7 +119,7 @@ public void testFromJson() throws IOException { @Override public void testMustRewrite() throws IOException { - TermQueryBuilder tqb = new TermQueryBuilder("foo", "bar"); + TermQueryBuilder tqb = new TermQueryBuilder(STRING_FIELD_NAME, "bar"); WrapperQueryBuilder qb = new WrapperQueryBuilder(tqb.toString()); UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> qb.toQuery(createShardContext())); assertEquals("this query must be rewritten first", e.getMessage()); @@ -137,7 +137,7 @@ public void testRewriteWithInnerName() throws IOException { } public void testRewriteWithInnerBoost() throws IOException { - final TermQueryBuilder query = new TermQueryBuilder("foo", "bar").boost(2); + final TermQueryBuilder query = new TermQueryBuilder(STRING_FIELD_NAME, "bar").boost(2); QueryBuilder builder = new WrapperQueryBuilder(query.toString()); QueryShardContext shardContext = createShardContext(); assertEquals(query, builder.rewrite(shardContext)); @@ -149,15 +149,15 @@ public void testRewriteInnerQueryToo() throws IOException { QueryShardContext shardContext = createShardContext(); QueryBuilder qb = new WrapperQueryBuilder( - new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()).toString() + new WrapperQueryBuilder(new TermQueryBuilder(STRING_FIELD_NAME, "bar").toString()).toString() ); - assertEquals(new TermQuery(new Term("foo", "bar")), qb.rewrite(shardContext).toQuery(shardContext)); + assertEquals(new TermQuery(new Term(STRING_FIELD_NAME, "bar")), qb.rewrite(shardContext).toQuery(shardContext)); qb = new WrapperQueryBuilder( new WrapperQueryBuilder( - new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()).toString() + new WrapperQueryBuilder(new TermQueryBuilder(STRING_FIELD_NAME, "bar").toString()).toString() ).toString() ); - assertEquals(new TermQuery(new Term("foo", "bar")), qb.rewrite(shardContext).toQuery(shardContext)); + assertEquals(new TermQuery(new Term(STRING_FIELD_NAME, "bar")), qb.rewrite(shardContext).toQuery(shardContext)); qb = new WrapperQueryBuilder(new BoolQueryBuilder().toString()); assertEquals(new MatchAllDocsQuery(), qb.rewrite(shardContext).toQuery(shardContext)); diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java index 16e5e87ba83e7..62ea7de066ae4 100644 --- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java @@ -552,11 +552,12 @@ public void testMalformedThrowsException() throws IOException { } public void testCustomWeightFactorQueryBuilderWithFunctionScore() throws IOException { - Query parsedQuery = parseQuery(functionScoreQuery(termQuery("name.last", "banon"), weightFactorFunction(1.3f))) - .toQuery(createShardContext()); + QueryShardContext context = createShardContext(); + Query parsedQuery = parseQuery(functionScoreQuery(termQuery(STRING_FIELD_NAME_2, "banon"), weightFactorFunction(1.3f))) + .rewrite(context).toQuery(context); assertThat(parsedQuery, instanceOf(FunctionScoreQuery.class)); FunctionScoreQuery functionScoreQuery = (FunctionScoreQuery) parsedQuery; - assertThat(((TermQuery) functionScoreQuery.getSubQuery()).getTerm(), equalTo(new Term("name.last", "banon"))); + assertThat(((TermQuery) functionScoreQuery.getSubQuery()).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "banon"))); assertThat((double) (functionScoreQuery.getFunctions()[0]).getWeight(), closeTo(1.3, 0.001)); } @@ -642,14 +643,14 @@ public void testFromJson() throws IOException { public void testRewrite() throws IOException { FunctionScoreQueryBuilder functionScoreQueryBuilder = - new FunctionScoreQueryBuilder(new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString())) + new FunctionScoreQueryBuilder(new WrapperQueryBuilder(new TermQueryBuilder(STRING_FIELD_NAME, "bar").toString())) .boostMode(CombineFunction.REPLACE) .scoreMode(FunctionScoreQuery.ScoreMode.SUM) .setMinScore(1) .maxBoost(100); FunctionScoreQueryBuilder rewrite = (FunctionScoreQueryBuilder) functionScoreQueryBuilder.rewrite(createShardContext()); assertNotSame(functionScoreQueryBuilder, rewrite); - assertEquals(rewrite.query(), new TermQueryBuilder("foo", "bar")); + assertEquals(rewrite.query(), new TermQueryBuilder(STRING_FIELD_NAME, "bar")); assertEquals(rewrite.boostMode(), CombineFunction.REPLACE); assertEquals(rewrite.scoreMode(), FunctionScoreQuery.ScoreMode.SUM); assertEquals(rewrite.getMinScore(), 1f, 0.0001); @@ -657,18 +658,18 @@ public void testRewrite() throws IOException { } public void testRewriteWithFunction() throws IOException { - QueryBuilder firstFunction = new WrapperQueryBuilder(new TermQueryBuilder("tq", "1").toString()); - TermQueryBuilder secondFunction = new TermQueryBuilder("tq", "2"); - QueryBuilder queryBuilder = randomBoolean() ? new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()) - : new TermQueryBuilder("foo", "bar"); + QueryBuilder firstFunction = new WrapperQueryBuilder(new TermQueryBuilder(STRING_FIELD_NAME_2, "1").toString()); + TermQueryBuilder secondFunction = new TermQueryBuilder(STRING_FIELD_NAME_2, "2"); + QueryBuilder queryBuilder = randomBoolean() ? new WrapperQueryBuilder(new TermQueryBuilder(STRING_FIELD_NAME, "bar").toString()) + : new TermQueryBuilder(STRING_FIELD_NAME, "bar"); FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder, new FunctionScoreQueryBuilder.FilterFunctionBuilder[] { new FunctionScoreQueryBuilder.FilterFunctionBuilder(firstFunction, new RandomScoreFunctionBuilder()), new FunctionScoreQueryBuilder.FilterFunctionBuilder(secondFunction, new RandomScoreFunctionBuilder()) }); FunctionScoreQueryBuilder rewrite = (FunctionScoreQueryBuilder) functionScoreQueryBuilder.rewrite(createShardContext()); assertNotSame(functionScoreQueryBuilder, rewrite); - assertEquals(rewrite.query(), new TermQueryBuilder("foo", "bar")); - assertEquals(rewrite.filterFunctionBuilders()[0].getFilter(), new TermQueryBuilder("tq", "1")); + assertEquals(rewrite.query(), new TermQueryBuilder(STRING_FIELD_NAME, "bar")); + assertEquals(rewrite.filterFunctionBuilders()[0].getFilter(), new TermQueryBuilder(STRING_FIELD_NAME_2, "1")); assertSame(rewrite.filterFunctionBuilders()[1].getFilter(), secondFunction); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 727c3ea3a87ae..e4fba342be747 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -683,13 +683,15 @@ private void termsAggregator(ValueType valueType, MappedFieldType fieldType, } if (multiValued == false) { + MappedFieldType filterFieldType = new KeywordFieldMapper.KeywordFieldType(); + filterFieldType.setName("include"); aggregationBuilder = new FilterAggregationBuilder("_name1", QueryBuilders.termQuery("include", "yes")); aggregationBuilder.subAggregation(new TermsAggregationBuilder("_name2", valueType) .executionHint(executionHint) .size(numTerms) .collectMode(randomFrom(Aggregator.SubAggCollectionMode.values())) .field("field")); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType, filterFieldType); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); aggregator.postCollection(); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 199f9b055393c..1b5f65cb44809 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -166,7 +166,9 @@ protected A createAggregator(Query query, MappedFieldType... fieldTypes) throws IOException { SearchContext searchContext = createSearchContext(indexSearcher, indexSettings, query, bucketConsumer, fieldTypes); @SuppressWarnings("unchecked") - A aggregator = (A) aggregationBuilder.build(searchContext.getQueryShardContext(), null) + A aggregator = (A) aggregationBuilder + .rewrite(searchContext.getQueryShardContext()) + .build(searchContext.getQueryShardContext(), null) .create(searchContext, null, true); return aggregator; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 2fd1bf450f51c..471cdccfc1c3f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.io.JsonStringEncoder; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.spans.SpanBoostQuery; @@ -453,7 +454,7 @@ public void testToQuery() throws IOException { rewrite(secondLuceneQuery), rewrite(firstLuceneQuery)); } - if (supportsBoost()) { + if (supportsBoost() && firstLuceneQuery instanceof MatchNoDocsQuery == false) { secondQuery.boost(firstQuery.boost() + 1f + randomFloat()); Query thirdLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context); assertNotEquals("modifying the boost doesn't affect the corresponding lucene query", rewrite(firstLuceneQuery), @@ -495,20 +496,22 @@ protected boolean supportsQueryName() { * {@link #doAssertLuceneQuery(AbstractQueryBuilder, Query, QueryShardContext)} for query specific checks. */ private void assertLuceneQuery(QB queryBuilder, Query query, QueryShardContext context) throws IOException { - if (queryBuilder.queryName() != null) { + if (queryBuilder.queryName() != null && query instanceof MatchNoDocsQuery == false) { Query namedQuery = context.copyNamedQueries().get(queryBuilder.queryName()); assertThat(namedQuery, equalTo(query)); } if (query != null) { if (queryBuilder.boost() != AbstractQueryBuilder.DEFAULT_BOOST) { - assertThat(query, either(instanceOf(BoostQuery.class)).or(instanceOf(SpanBoostQuery.class))); + assertThat(query, either(instanceOf(BoostQuery.class)).or(instanceOf(SpanBoostQuery.class)).or(instanceOf(MatchNoDocsQuery.class))); if (query instanceof SpanBoostQuery) { SpanBoostQuery spanBoostQuery = (SpanBoostQuery) query; assertThat(spanBoostQuery.getBoost(), equalTo(queryBuilder.boost())); query = spanBoostQuery.getQuery(); - } else { + } else if (query instanceof BoostQuery) { BoostQuery boostQuery = (BoostQuery) query; - assertThat(boostQuery.getBoost(), equalTo(queryBuilder.boost())); + if (boostQuery.getQuery() instanceof MatchNoDocsQuery == false) { + assertThat(boostQuery.getBoost(), equalTo(queryBuilder.boost())); + } query = boostQuery.getQuery(); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index a8bf5272e94fb..55b28ba825c1c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -241,6 +241,7 @@ private void runTestOnIndex(CheckedBiConsumer nowInMillis, null, null); + context.setMapUnmappedFieldAsString(true); body.accept(context, leaf); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index ca2b38318a06f..c113d62d1bf8a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -79,6 +79,7 @@ public void testDLS() throws Exception { QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null, null); + realQueryShardContext.setMapUnmappedFieldAsString(true); QueryShardContext queryShardContext = spy(realQueryShardContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY); XPackLicenseState licenseState = mock(XPackLicenseState.class); @@ -201,6 +202,7 @@ public void testDLSWithLimitedPermissions() throws Exception { QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null, null); + realQueryShardContext.setMapUnmappedFieldAsString(true); QueryShardContext queryShardContext = spy(realQueryShardContext); DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY); From abf27f1e4d558e6919491b77cb6505636c7499ab Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 29 Nov 2019 16:24:52 +0100 Subject: [PATCH 03/21] Rename to `constant_keyword`. --- docs/reference/mapping/types.asciidoc | 4 +- ...ord.asciidoc => constant-keyword.asciidoc} | 16 +++--- ...ava => ConstantKeywordIndexFieldData.java} | 6 +-- ...nFieldType.java => ConstantFieldType.java} | 6 +-- .../index/mapper/IndexFieldMapper.java | 6 +-- .../index/mapper/TypeFieldMapper.java | 4 +- .../index/query/PrefixQueryBuilder.java | 6 +-- .../index/query/TermQueryBuilder.java | 6 +-- .../index/query/TermsQueryBuilder.java | 6 +-- .../index/query/WildcardQueryBuilder.java | 6 +-- .../elasticsearch/xpack/core/XPackPlugin.java | 4 +- ...r.java => ConstantKeywordFieldMapper.java} | 50 +++++++++---------- ...a => ConstantKeywordFieldMapperTests.java} | 6 +-- ...ava => ConstantKeywordFieldTypeTests.java} | 16 +++--- .../10_basic.yml | 6 +-- 15 files changed, 74 insertions(+), 74 deletions(-) rename docs/reference/mapping/types/{singleton-keyword.asciidoc => constant-keyword.asciidoc} (78%) rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{SingletonKeywordIndexFieldData.java => ConstantKeywordIndexFieldData.java} (94%) rename server/src/main/java/org/elasticsearch/index/mapper/{SingletonFieldType.java => ConstantFieldType.java} (89%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/{SingletonKeywordFieldMapper.java => ConstantKeywordFieldMapper.java} (80%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/{SingletonKeywordFieldMapperTests.java => ConstantKeywordFieldMapperTests.java} (92%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/{SingletonKeywordFieldTypeTests.java => ConstantKeywordFieldTypeTests.java} (78%) rename x-pack/plugin/src/test/resources/rest-api-spec/test/{singleton_keyword => constant_keyword}/10_basic.yml (95%) diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index d20623a5fabf3..1b5afad31f0c6 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -57,7 +57,7 @@ string:: <> and <> <>:: `histogram` for pre-aggregated numerical values for percentiles aggregations. -<>:: Specialization of `keyword` for the case when all documents have the same value. +<>:: Specialization of `keyword` for the case when all documents have the same value. [float] [[types-array-handling]] @@ -130,4 +130,4 @@ include::types/token-count.asciidoc[] include::types/shape.asciidoc[] -include::types/singleton-keyword.asciidoc[] +include::types/constant-keyword.asciidoc[] diff --git a/docs/reference/mapping/types/singleton-keyword.asciidoc b/docs/reference/mapping/types/constant-keyword.asciidoc similarity index 78% rename from docs/reference/mapping/types/singleton-keyword.asciidoc rename to docs/reference/mapping/types/constant-keyword.asciidoc index 7bfbfd980c1fe..8aa4a80567d19 100644 --- a/docs/reference/mapping/types/singleton-keyword.asciidoc +++ b/docs/reference/mapping/types/constant-keyword.asciidoc @@ -1,13 +1,13 @@ [role="xpack"] [testenv="basic"] -[[singleton-keyword]] -=== Singleton keyword datatype +[[constant-keyword]] +=== Constant keyword datatype ++++ -Singleton keyword +Constant keyword ++++ -Singleton keyword is a specialization of the <> field for +Constant keyword is a specialization of the <> field for the case that all documents in the index have the same value. [source,console] @@ -23,7 +23,7 @@ PUT logs-debug "type": "text" }, "level": { - "type": "singleton_keyword", + "type": "constant_keyword", "value": "debug" } } @@ -31,7 +31,7 @@ PUT logs-debug } -------------------------------- -`singleton_keyword` supports the same queries and aggregations as `keyword` +`constant_keyword` supports the same queries and aggregations as `keyword` fields do, but takes advantage of the fact that all documents have the same value per index to be more efficient. @@ -59,8 +59,8 @@ POST logs-debug/_doc However providing a value that is different from the one configured in the mapping is disallowed. -[[singleton-keyword-params]] -==== Parameters for singleton keyword fields +[[constant-keyword-params]] +==== Parameters for constant keyword fields The following mapping parameters are accepted: diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SingletonKeywordIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java similarity index 94% rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/SingletonKeywordIndexFieldData.java rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java index 73feef26ce7c2..4296084bc15af 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SingletonKeywordIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java @@ -46,7 +46,7 @@ import java.util.Collections; import java.util.function.Function; -public class SingletonKeywordIndexFieldData extends AbstractIndexOrdinalsFieldData { +public class ConstantKeywordIndexFieldData extends AbstractIndexOrdinalsFieldData { public static class Builder implements IndexFieldData.Builder { @@ -59,7 +59,7 @@ public Builder(Function valueFunction) { @Override public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { - return new SingletonKeywordIndexFieldData(indexSettings, fieldType.name(), valueFunction.apply(mapperService)); + return new ConstantKeywordIndexFieldData(indexSettings, fieldType.name(), valueFunction.apply(mapperService)); } } @@ -128,7 +128,7 @@ public void close() { private final SingletonKeywordAtomicFieldData atomicFieldData; - private SingletonKeywordIndexFieldData(IndexSettings indexSettings, String name, String value) { + private ConstantKeywordIndexFieldData(IndexSettings indexSettings, String name, String value) { super(indexSettings, name, null, null, TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SingletonFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java similarity index 89% rename from server/src/main/java/org/elasticsearch/index/mapper/SingletonFieldType.java rename to server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java index bf12f3f938a85..f6150efbc20fc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SingletonFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java @@ -26,13 +26,13 @@ /** * A {@link MappedFieldType} that has the same value for all documents. */ -public abstract class SingletonFieldType extends MappedFieldType { +public abstract class ConstantFieldType extends MappedFieldType { - public SingletonFieldType() { + public ConstantFieldType() { super(); } - public SingletonFieldType(SingletonFieldType other) { + public ConstantFieldType(ConstantFieldType other) { super(other); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 8f531fb7bb4e5..6dd9e11ee072a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -30,7 +30,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.plain.SingletonKeywordIndexFieldData; +import org.elasticsearch.index.fielddata.plain.ConstantKeywordIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; @@ -88,7 +88,7 @@ public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext c } } - static final class IndexFieldType extends SingletonFieldType { + static final class IndexFieldType extends ConstantFieldType { IndexFieldType() {} @@ -175,7 +175,7 @@ public Query wildcardQuery(String value, @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { - return new SingletonKeywordIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName); + return new ConstantKeywordIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index cfed59d8e91d5..d2450b8b44789 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -40,7 +40,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.plain.SingletonKeywordIndexFieldData; +import org.elasticsearch.index.fielddata.plain.ConstantKeywordIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; @@ -110,7 +110,7 @@ public String typeName() { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { Function typeFunction = mapperService -> mapperService.documentMapper().type(); - return new SingletonKeywordIndexFieldData.Builder(typeFunction); + return new ConstantKeywordIndexFieldData.Builder(typeFunction); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java index 62da66dde2711..dcc60d8ddf813 100644 --- a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java @@ -32,7 +32,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.SingletonFieldType; +import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.query.support.QueryParsers; import java.io.IOException; @@ -176,8 +176,8 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws MappedFieldType fieldType = context.fieldMapper(this.fieldName); if (fieldType == null) { return new MatchNoneQueryBuilder(); - } else if (fieldType instanceof SingletonFieldType) { - // This logic is correct for all field types, but by only applying it to singleton + } else if (fieldType instanceof ConstantFieldType) { + // This logic is correct for all field types, but by only applying it to constant // fields we also have the guarantee that it doesn't perform I/O, which is important // since rewrites might happen on a network thread. Query query = fieldType.prefixQuery(value, null, context); // the rewrite method doesn't matter diff --git a/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java index e52f07b43ab44..7abb4de3a2571 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.SingletonFieldType; +import org.elasticsearch.index.mapper.ConstantFieldType; import java.io.IOException; @@ -137,8 +137,8 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws MappedFieldType fieldType = context.fieldMapper(this.fieldName); if (fieldType == null) { return new MatchNoneQueryBuilder(); - } else if (fieldType instanceof SingletonFieldType) { - // This logic is correct for all field types, but by only applying it to singleton + } else if (fieldType instanceof ConstantFieldType) { + // This logic is correct for all field types, but by only applying it to constant // fields we also have the guarantee that it doesn't perform I/O, which is important // since rewrites might happen on a network thread. Query query = fieldType.termQuery(value, context); diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java index 362aca701ad92..f83c466ca769e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java @@ -39,7 +39,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.SingletonFieldType; +import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.indices.TermsLookup; import java.io.IOException; @@ -483,8 +483,8 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) { MappedFieldType fieldType = context.fieldMapper(this.fieldName); if (fieldType == null) { return new MatchNoneQueryBuilder(); - } else if (fieldType instanceof SingletonFieldType) { - // This logic is correct for all field types, but by only applying it to singleton + } else if (fieldType instanceof ConstantFieldType) { + // This logic is correct for all field types, but by only applying it to constant // fields we also have the guarantee that it doesn't perform I/O, which is important // since rewrites might happen on a network thread. Query query = fieldType.termsQuery(values, context); diff --git a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java index fde7abd89b16e..db301ac3552d7 100644 --- a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java @@ -32,7 +32,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.SingletonFieldType; +import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.query.support.QueryParsers; import java.io.IOException; @@ -188,8 +188,8 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws MappedFieldType fieldType = context.fieldMapper(this.fieldName); if (fieldType == null) { return new MatchNoneQueryBuilder(); - } else if (fieldType instanceof SingletonFieldType) { - // This logic is correct for all field types, but by only applying it to singleton + } else if (fieldType instanceof ConstantFieldType) { + // This logic is correct for all field types, but by only applying it to constant // fields we also have the guarantee that it doesn't perform I/O, which is important // since rewrites might happen on a network thread. Query query = fieldType.wildcardQuery(value, null, context); // the rewrite method doesn't matter diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index ee9b4760bfa35..974e9c005e249 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -62,7 +62,7 @@ import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.action.XPackUsageResponse; -import org.elasticsearch.xpack.core.index.mapper.SingletonKeywordFieldMapper; +import org.elasticsearch.xpack.core.index.mapper.ConstantKeywordFieldMapper; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.rest.action.RestReloadAnalyzersAction; import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction; @@ -337,6 +337,6 @@ public List> getSettings() { @Override public Map getMappers() { - return Collections.singletonMap(SingletonKeywordFieldMapper.CONTENT_TYPE, new SingletonKeywordFieldMapper.TypeParser()); + return Collections.singletonMap(ConstantKeywordFieldMapper.CONTENT_TYPE, new ConstantKeywordFieldMapper.TypeParser()); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java similarity index 80% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapper.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java index 1ca77644bb58a..9351e769b45bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java @@ -24,28 +24,28 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.plain.SingletonKeywordIndexFieldData; +import org.elasticsearch.index.fielddata.plain.ConstantKeywordIndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.SingletonFieldType; +import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.query.QueryShardContext; -public class SingletonKeywordFieldMapper extends FieldMapper { +public class ConstantKeywordFieldMapper extends FieldMapper { - public static final String CONTENT_TYPE = "singleton_keyword"; + public static final String CONTENT_TYPE = "constant_keyword"; public static class Defaults { public static final String NULL_VALUE = null; public static final int IGNORE_ABOVE = Integer.MAX_VALUE; } - public static class Builder extends FieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private static MappedFieldType createFieldType(String value) { - SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); ft.setValue(value); ft.freeze(); return ft; @@ -61,14 +61,14 @@ public Builder(String name, String value) { } @Override - public SingletonKeywordFieldType fieldType() { - return (SingletonKeywordFieldType) super.fieldType(); + public ConstantKeywordFieldType fieldType() { + return (ConstantKeywordFieldType) super.fieldType(); } @Override - public SingletonKeywordFieldMapper build(BuilderContext context) { + public ConstantKeywordFieldMapper build(BuilderContext context) { setupFieldType(context); - return new SingletonKeywordFieldMapper( + return new ConstantKeywordFieldMapper( name, fieldType, defaultFieldType, context.indexSettings()); } @@ -85,25 +85,25 @@ public Mapper.Builder parse(String name, Map node, ParserCo throw new MapperParsingException("Property [value] of field [" + name + "] must be a number or a string, but got [" + value + "]"); } - return new SingletonKeywordFieldMapper.Builder(name, value.toString()); + return new ConstantKeywordFieldMapper.Builder(name, value.toString()); } } - public static final class SingletonKeywordFieldType extends SingletonFieldType { + public static final class ConstantKeywordFieldType extends ConstantFieldType { private String value; - public SingletonKeywordFieldType() { + public ConstantKeywordFieldType() { super(); } - protected SingletonKeywordFieldType(SingletonKeywordFieldType ref) { + protected ConstantKeywordFieldType(ConstantKeywordFieldType ref) { super(ref); this.value = ref.value; } - public SingletonKeywordFieldType clone() { - return new SingletonKeywordFieldType(this); + public ConstantKeywordFieldType clone() { + return new ConstantKeywordFieldType(this); } @Override @@ -111,14 +111,14 @@ public boolean equals(Object o) { if (super.equals(o) == false) { return false; } - SingletonKeywordFieldType other = (SingletonKeywordFieldType) o; + ConstantKeywordFieldType other = (ConstantKeywordFieldType) o; return Objects.equals(value, other.value); } @Override public void checkCompatibility(MappedFieldType otherFT, List conflicts) { super.checkCompatibility(otherFT, conflicts); - SingletonKeywordFieldType other = (SingletonKeywordFieldType) otherFT; + ConstantKeywordFieldType other = (ConstantKeywordFieldType) otherFT; if (Objects.equals(value, other.value) == false) { conflicts.add("mapper [" + name() + "] has different [value]"); } @@ -147,7 +147,7 @@ public String typeName() { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { - return new SingletonKeywordIndexFieldData.Builder(mapperService -> value); + return new ConstantKeywordIndexFieldData.Builder(mapperService -> value); } private static String valueToString(Object v) { @@ -199,19 +199,19 @@ public Query wildcardQuery(String value, } } - protected SingletonKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + protected ConstantKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings) { super(simpleName, fieldType, defaultFieldType, indexSettings, MultiFields.empty(), CopyTo.empty()); } @Override - protected SingletonKeywordFieldMapper clone() { - return (SingletonKeywordFieldMapper) super.clone(); + protected ConstantKeywordFieldMapper clone() { + return (ConstantKeywordFieldMapper) super.clone(); } @Override - public SingletonKeywordFieldType fieldType() { - return (SingletonKeywordFieldType) super.fieldType(); + public ConstantKeywordFieldType fieldType() { + return (ConstantKeywordFieldType) super.fieldType(); } @Override @@ -229,7 +229,7 @@ protected void parseCreateField(ParseContext context, List field } if (Objects.equals(fieldType().value, value) == false) { - throw new IllegalArgumentException("[singleton_keyword] field [" + name() + + throw new IllegalArgumentException("[constant_keyword] field [" + name() + "] only accepts values that are equal to the wrapped value [" + fieldType().value() + "], but got [" + value + "]"); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapperTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapperTests.java similarity index 92% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapperTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapperTests.java index 9dec3a14810a4..3c5ac10b3699d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldMapperTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapperTests.java @@ -25,7 +25,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xpack.core.XPackPlugin; -public class SingletonKeywordFieldMapperTests extends ESSingleNodeTestCase { +public class ConstantKeywordFieldMapperTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { @@ -37,7 +37,7 @@ protected Collection> getPlugins() { public void testDefaults() throws Exception { IndexService indexService = createIndex("test"); String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") - .startObject("properties").startObject("field").field("type", "singleton_keyword") + .startObject("properties").startObject("field").field("type", "constant_keyword") .field("value", "foo").endObject().endObject().endObject().endObject()); DocumentMapper mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); assertEquals(mapping, mapper.mappingSource().toString()); @@ -54,7 +54,7 @@ public void testDefaults() throws Exception { .startObject().field("field", "bar").endObject()); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(new SourceToParse("test", "1", illegalSource, XContentType.JSON))); - assertEquals("[singleton_keyword] field [field] only accepts values that are equal to the wrapped value [foo], but got [bar]", + assertEquals("[constant_keyword] field [field] only accepts values that are equal to the wrapped value [foo], but got [bar]", e.getCause().getMessage()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldTypeTests.java similarity index 78% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldTypeTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldTypeTests.java index 582ee2d6bbec6..fa27aefc8a02f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/SingletonKeywordFieldTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldTypeTests.java @@ -13,35 +13,35 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.xpack.core.index.mapper.SingletonKeywordFieldMapper.SingletonKeywordFieldType; +import org.elasticsearch.xpack.core.index.mapper.ConstantKeywordFieldMapper.ConstantKeywordFieldType; import org.junit.Before; -public class SingletonKeywordFieldTypeTests extends FieldTypeTestCase { +public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase { @Before public void setupProperties() { addModifier(new Modifier("value", false) { @Override public void modify(MappedFieldType type) { - ((SingletonKeywordFieldType) type).setValue("bar"); + ((ConstantKeywordFieldType) type).setValue("bar"); } }); } @Override protected MappedFieldType createDefaultFieldType() { - return new SingletonKeywordFieldType(); + return new ConstantKeywordFieldType(); } public void testTermQuery() { - SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.termQuery("foo", null)); assertEquals(new MatchNoDocsQuery(), ft.termQuery("bar", null)); } public void testTermsQuery() { - SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.termsQuery(Collections.singletonList("foo"), null)); assertEquals(new MatchAllDocsQuery(), ft.termsQuery(Arrays.asList("bar", "foo", "quux"), null)); @@ -51,14 +51,14 @@ public void testTermsQuery() { } public void testWildcardQuery() { - SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("f*o", null, null)); assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("b*r", null, null)); } public void testPrefixQuery() { - SingletonKeywordFieldType ft = new SingletonKeywordFieldType(); + ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.prefixQuery("fo", null, null)); assertEquals(new MatchNoDocsQuery(), ft.prefixQuery("ba", null, null)); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/singleton_keyword/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml similarity index 95% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/singleton_keyword/10_basic.yml rename to x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml index 495130cf7b263..e6a29e1681e19 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/singleton_keyword/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml @@ -2,7 +2,7 @@ setup: - skip: version: " - 7.99.99" - reason: "singleton_keyword was added in 7.6" + reason: "constant_keyword was added in 7.6" - do: indices.create: @@ -11,7 +11,7 @@ setup: mappings: properties: foo: - type: singleton_keyword + type: constant_keyword value: bar - do: @@ -21,7 +21,7 @@ setup: mappings: properties: foo: - type: singleton_keyword + type: constant_keyword value: baz - do: From 002fe968f83805672d52a749616942ec446a5519 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 29 Nov 2019 16:40:10 +0100 Subject: [PATCH 04/21] iter --- .../percolator/PercolateQueryBuilder.java | 10 +++++----- .../percolator/QueryBuilderStoreTests.java | 5 +---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 1f083aa207613..f97481254c571 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -554,9 +554,10 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { PercolatorFieldMapper.FieldType pft = (PercolatorFieldMapper.FieldType) fieldType; String name = this.name != null ? this.name : pft.name(); QueryShardContext percolateShardContext = wrap(context); + percolateShardContext.setAllowUnmappedFields(false); + percolateShardContext.setMapUnmappedFieldAsString(pft.mapUnmappedFieldsAsText); PercolateQuery.QueryStore queryStore = createStore(pft.queryBuilderField, - percolateShardContext, - pft.mapUnmappedFieldsAsText); + percolateShardContext); return pft.percolateQuery(name, queryStore, documents, docSearcher, excludeNestedDocuments, context.indexVersionCreated()); } @@ -599,8 +600,7 @@ static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, Collection

{ @@ -626,7 +626,7 @@ static PercolateQuery.QueryStore createStore(MappedFieldType queryBuilderFieldTy assert valueLength > 0; QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class); assert in.read() == -1; - return PercolatorFieldMapper.toQuery(context, mapUnmappedFieldsAsString, queryBuilder); + return queryBuilder.toQuery(context); } } } else { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index 5026d17137123..670901dbfa2e9 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -38,12 +38,9 @@ import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; -import org.elasticsearch.index.mapper.TextFieldTypeTests; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.mock.orig.Mockito; @@ -103,7 +100,7 @@ public void testStoringQueryBuilders() throws IOException { fieldType.setName((String) invocation.getArguments()[0]); return fieldType; }); - PercolateQuery.QueryStore queryStore = PercolateQueryBuilder.createStore(fieldMapper.fieldType(), queryShardContext, false); + PercolateQuery.QueryStore queryStore = PercolateQueryBuilder.createStore(fieldMapper.fieldType(), queryShardContext); try (IndexReader indexReader = DirectoryReader.open(directory)) { LeafReaderContext leafContext = indexReader.leaves().get(0); From c6738d80cd2287d80f39924b2e3f10bdd6373be5 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 29 Nov 2019 16:58:28 +0100 Subject: [PATCH 05/21] iter --- .../org/elasticsearch/test/AbstractQueryTestCase.java | 3 ++- .../searchbusinessrules/PinnedQueryBuilderTests.java | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 471cdccfc1c3f..167ce66e38965 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -502,7 +502,8 @@ private void assertLuceneQuery(QB queryBuilder, Query query, QueryShardContext c } if (query != null) { if (queryBuilder.boost() != AbstractQueryBuilder.DEFAULT_BOOST) { - assertThat(query, either(instanceOf(BoostQuery.class)).or(instanceOf(SpanBoostQuery.class)).or(instanceOf(MatchNoDocsQuery.class))); + assertThat(query, either(instanceOf(BoostQuery.class)).or(instanceOf(SpanBoostQuery.class)) + .or(instanceOf(MatchNoDocsQuery.class))); if (query instanceof SpanBoostQuery) { SpanBoostQuery spanBoostQuery = (SpanBoostQuery) query; assertThat(spanBoostQuery.getBoost(), equalTo(queryBuilder.boost())); diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java index db3d46fc1a7dd..e91ef3a5d76f9 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java @@ -170,4 +170,13 @@ public void testRewrite() throws IOException { assertThat(rewritten, instanceOf(PinnedQueryBuilder.class)); } + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + PinnedQueryBuilder queryBuilder = new PinnedQueryBuilder(new TermQueryBuilder("unmapped_field", "42")); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } } From b7b894e01025bd1cf94fe15be68d122eb663c47c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 29 Nov 2019 17:25:08 +0100 Subject: [PATCH 06/21] iter --- .../java/org/elasticsearch/index/query/QueryShardContext.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index c599fbcb4274e..44db7194561c6 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -255,11 +255,13 @@ public void setMapUnmappedFieldAsString(boolean mapUnmappedFieldAsString) { } MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMapping) { - if (fieldMapping != null || allowUnmappedFields) { + if (fieldMapping != null) { return fieldMapping; } else if (mapUnmappedFieldAsString) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); return builder.build(new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(1))).fieldType(); + } else if (allowUnmappedFields) { + return null; } else { throw new QueryShardException(this, "No field mapping can be found for the field with name [{}]", name); } From a86ad8a82c0d67e440dcf0f9b5eb0cf7c7255ef9 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 29 Nov 2019 18:08:20 +0100 Subject: [PATCH 07/21] iter --- .../index/query/WildcardQueryBuilder.java | 2 +- .../index/query/BoostingQueryBuilderTests.java | 18 ++++++++++++++++++ .../index/query/PrefixQueryBuilderTests.java | 13 ++++--------- .../index/query/TermQueryBuilderTests.java | 13 ++++--------- .../index/query/WildcardQueryBuilderTests.java | 13 ++++--------- 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java index db301ac3552d7..9b3dca36213e8 100644 --- a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java @@ -209,7 +209,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { MappedFieldType fieldType = context.fieldMapper(fieldName); if (fieldType == null) { - return new MatchNoDocsQuery("unknown field [" + fieldName + "]"); + throw new IllegalStateException("Rewrite first"); } MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod( diff --git a/server/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java index 0d859392366ce..22d4db861be82 100644 --- a/server/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java @@ -103,4 +103,22 @@ public void testRewrite() throws IOException { assertEquals(new BoostingQueryBuilder(positive.rewrite(createShardContext()), negative.rewrite(createShardContext())), rewrite); } } + + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + + BoostingQueryBuilder queryBuilder1 = new BoostingQueryBuilder( + new TermQueryBuilder("unmapped_field", "foo"), new MatchNoneQueryBuilder()); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder1.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + + BoostingQueryBuilder queryBuilder2 = new BoostingQueryBuilder( + new MatchAllQueryBuilder(), new TermQueryBuilder("unmapped_field", "foo")); + e = expectThrows(IllegalStateException.class, + () -> queryBuilder2.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java index b7f7ef132f1d0..94596ffd6c58d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java @@ -164,14 +164,9 @@ public void testRewriteIndexQueryToNotMatchNone() throws Exception { public void testMustRewrite() throws IOException { QueryShardContext context = createShardContext(); context.setAllowUnmappedFields(true); - PrefixQueryBuilder queryBuilder = createTestQueryBuilder(); - if (context.fieldMapper(queryBuilder.fieldName()) == null) { - IllegalStateException e = expectThrows(IllegalStateException.class, - () -> queryBuilder.toQuery(context)); - assertEquals("Rewrite first", e.getMessage()); - } else { - // no exception - queryBuilder.toQuery(context); - } + PrefixQueryBuilder queryBuilder = new PrefixQueryBuilder("unmapped_field", "foo"); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java index b3d782b4c2f0f..376aa54ef15b3 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java @@ -190,14 +190,9 @@ public void testRewriteIndexQueryToNotMatchNone() throws IOException { public void testMustRewrite() throws IOException { QueryShardContext context = createShardContext(); context.setAllowUnmappedFields(true); - TermQueryBuilder queryBuilder = createTestQueryBuilder(); - if (context.fieldMapper(queryBuilder.fieldName()) == null) { - IllegalStateException e = expectThrows(IllegalStateException.class, - () -> queryBuilder.toQuery(context)); - assertEquals("Rewrite first", e.getMessage()); - } else { - // no exception - queryBuilder.toQuery(context); - } + TermQueryBuilder queryBuilder = new TermQueryBuilder("unmapped_field", "foo"); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java index d0bfbfefe9341..9482e0bb87d49 100644 --- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java @@ -159,14 +159,9 @@ public void testRewriteIndexQueryNotMatchNone() throws IOException { public void testMustRewrite() throws IOException { QueryShardContext context = createShardContext(); context.setAllowUnmappedFields(true); - WildcardQueryBuilder queryBuilder = createTestQueryBuilder(); - if (context.fieldMapper(queryBuilder.fieldName()) == null) { - IllegalStateException e = expectThrows(IllegalStateException.class, - () -> queryBuilder.toQuery(context)); - assertEquals("Rewrite first", e.getMessage()); - } else { - // no exception - queryBuilder.toQuery(context); - } + WildcardQueryBuilder queryBuilder = new WildcardQueryBuilder("unmapped_field", "foo"); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); } } From a5949d6f38479875ef7594ecdaa07516a096f593 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 2 Dec 2019 10:25:40 +0100 Subject: [PATCH 08/21] iter --- .../index/query/IdsQueryBuilder.java | 26 ++++++++++++++----- .../query/SpanMultiTermQueryBuilder.java | 12 ++++++--- .../query/ConstantScoreQueryBuilderTests.java | 10 +++++++ .../index/query/IdsQueryBuilderTests.java | 10 +++++++ .../index/query/NestedQueryBuilderTests.java | 21 ++++++--------- .../query/ScriptScoreQueryBuilderTests.java | 13 ++++++++++ .../query/SpanMultiTermQueryBuilderTests.java | 3 ++- .../FunctionScoreQueryBuilderTests.java | 26 ++++++++++++++++++- 8 files changed, 95 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java index def29b91a7f81..2d1f420d9bf28 100644 --- a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.query; -import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; @@ -27,7 +26,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -137,17 +135,31 @@ public String getWriteableName() { return NAME; } + @Override + protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + if (ids.isEmpty()) { + return new MatchNoneQueryBuilder(); + } + QueryShardContext context = queryRewriteContext.convertToShardContext(); + if (context != null) { + if (context.fieldMapper(IdFieldMapper.NAME) == null) { + // no mappings yet + return new MatchNoneQueryBuilder(); + } + } + return super.doRewrite(queryRewriteContext); + } + @Override protected Query doToQuery(QueryShardContext context) throws IOException { MappedFieldType idField = context.fieldMapper(IdFieldMapper.NAME); if (idField == null) { - return new MatchNoDocsQuery("No mappings"); + throw new IllegalStateException("Rewrite first"); } - if (this.ids.isEmpty()) { - return Queries.newMatchNoDocsQuery("Missing ids in \"" + this.getName() + "\" query."); - } else { - return idField.termsQuery(new ArrayList<>(ids), context); + if (ids.isEmpty()) { + throw new IllegalStateException("Rewrite first"); } + return idField.termsQuery(new ArrayList<>(ids), context); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java index 49e5e53e1ed91..8d636141d22dc 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java @@ -126,11 +126,15 @@ public static SpanMultiTermQueryBuilder fromXContent(XContentParser parser) thro @Override protected Query doToQuery(QueryShardContext context) throws IOException { - if (multiTermQueryBuilder instanceof PrefixQueryBuilder) { + // We do the rewrite in toQuery since we need to keep a MultiTermQueryBuilder. + QueryBuilder multiTermQueryBuilder = Rewriteable.rewrite(this.multiTermQueryBuilder, context); + if (multiTermQueryBuilder instanceof MatchNoneQueryBuilder) { + return new SpanMatchNoDocsQuery(this.multiTermQueryBuilder.fieldName(), "Inner query rewrote to match_none"); + } else if (multiTermQueryBuilder instanceof PrefixQueryBuilder) { PrefixQueryBuilder prefixBuilder = (PrefixQueryBuilder) multiTermQueryBuilder; - MappedFieldType fieldType = context.fieldMapper(multiTermQueryBuilder.fieldName()); + MappedFieldType fieldType = context.fieldMapper(prefixBuilder.fieldName()); if (fieldType == null) { - return new SpanMatchNoDocsQuery(multiTermQueryBuilder.fieldName(), "unknown field"); + throw new IllegalStateException("Rewrite first"); } final SpanMultiTermQueryWrapper.SpanRewriteMethod spanRewriteMethod; if (prefixBuilder.rewrite() != null) { @@ -159,7 +163,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { } } if (subQuery instanceof MatchNoDocsQuery) { - return new SpanMatchNoDocsQuery(multiTermQueryBuilder.fieldName(), subQuery.toString()); + return new SpanMatchNoDocsQuery(this.multiTermQueryBuilder.fieldName(), subQuery.toString()); } else if (subQuery instanceof MultiTermQuery == false) { throw new UnsupportedOperationException("unsupported inner query, should be " + MultiTermQuery.class.getName() + " but was " + subQuery.getClass().getName()); diff --git a/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java index 3280aeecdda61..5281788a8307d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java @@ -110,4 +110,14 @@ public void testRewriteToMatchNone() throws IOException { QueryBuilder rewrite = constantScoreQueryBuilder.rewrite(createShardContext()); assertEquals(rewrite, new MatchNoneQueryBuilder()); } + + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + ConstantScoreQueryBuilder queryBuilder = new ConstantScoreQueryBuilder(new TermQueryBuilder("unmapped_field", "foo")); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java index 06f197f4c137e..7185210caadb8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java @@ -99,4 +99,14 @@ protected QueryBuilder parseQuery(XContentParser parser) throws IOException { assertThat(query, instanceOf(IdsQueryBuilder.class)); return (IdsQueryBuilder) query; } + + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContextWithNoType(); + context.setAllowUnmappedFields(true); + IdsQueryBuilder queryBuilder = createTestQueryBuilder(); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> queryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java index 08cf8eedb94d3..1b9b2050a911b 100644 --- a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java @@ -56,8 +56,6 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase { - boolean requiresRewrite = false; - @Override protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { mapperService.merge("_doc", new CompressedXContent(Strings.toString(PutMappingRequest.buildFromSimplifiedDef("_doc", @@ -78,10 +76,6 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws @Override protected NestedQueryBuilder doCreateTestQueryBuilder() { QueryBuilder innerQueryBuilder = RandomQueryBuilder.createQuery(random()); - if (randomBoolean()) { - requiresRewrite = true; - innerQueryBuilder = new WrapperQueryBuilder(innerQueryBuilder.toString()); - } NestedQueryBuilder nqb = new NestedQueryBuilder("nested1", innerQueryBuilder, RandomPicks.randomFrom(random(), ScoreMode.values())); nqb.ignoreUnmapped(randomBoolean()); @@ -185,13 +179,14 @@ public void testFromJson() throws IOException { @Override public void testMustRewrite() throws IOException { - try { - super.testMustRewrite(); - } catch (UnsupportedOperationException e) { - if (requiresRewrite == false) { - throw e; - } - } + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + TermQueryBuilder innerQueryBuilder = new TermQueryBuilder("nested1.unmapped_field", "foo"); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("nested1", innerQueryBuilder, + RandomPicks.randomFrom(random(), ScoreMode.values())); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> nestedQueryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); } public void testIgnoreUnmapped() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/query/ScriptScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ScriptScoreQueryBuilderTests.java index 690a7d6ae757a..8ab2c55066717 100644 --- a/server/src/test/java/org/elasticsearch/index/query/ScriptScoreQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/ScriptScoreQueryBuilderTests.java @@ -93,4 +93,17 @@ public void testCacheability() throws IOException { assertNotNull(rewriteQuery.toQuery(context)); assertFalse("query should not be cacheable: " + queryBuilder.toString(), context.isCacheable()); } + + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + TermQueryBuilder termQueryBuilder = new TermQueryBuilder("unmapped_field", "foo"); + String scriptStr = "1"; + Script script = new Script(ScriptType.INLINE, MockScriptEngine.NAME, scriptStr, Collections.emptyMap()); + ScriptScoreQueryBuilder scriptScoreQueryBuilder = new ScriptScoreQueryBuilder(termQueryBuilder, script); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> scriptScoreQueryBuilder.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java index 52d2dfaad8634..232e9b20e25b9 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java @@ -89,7 +89,8 @@ protected void doAssertLuceneQuery(SpanMultiTermQueryBuilder queryBuilder, Query if (query instanceof SpanMatchNoDocsQuery) { return; } - assertThat(query, either(instanceOf(SpanMultiTermQueryWrapper.class)).or(instanceOf(FieldMaskingSpanQuery.class))); + assertThat(query, either(instanceOf(SpanMultiTermQueryWrapper.class)) + .or(instanceOf(FieldMaskingSpanQuery.class))); if (query instanceof SpanMultiTermQueryWrapper) { SpanMultiTermQueryWrapper wrapper = (SpanMultiTermQueryWrapper) query; Query innerQuery = queryBuilder.innerQuery().toQuery(context); diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java index 62ea7de066ae4..b565778b35962 100644 --- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java @@ -686,7 +686,8 @@ public void testSingleScriptFunction() throws IOException { builder.boostMode(randomFrom(CombineFunction.values())); } - Query query = builder.toQuery(createShardContext()); + QueryShardContext shardContext = createShardContext(); + Query query = builder.rewrite(shardContext).toQuery(shardContext); assertThat(query, instanceOf(FunctionScoreQuery.class)); CombineFunction expectedBoostMode = builder.boostMode() != null @@ -841,4 +842,27 @@ private boolean isCacheable(FunctionScoreQueryBuilder queryBuilder) { } return true; } + + @Override + public void testMustRewrite() throws IOException { + QueryShardContext context = createShardContext(); + context.setAllowUnmappedFields(true); + TermQueryBuilder termQueryBuilder = new TermQueryBuilder("unmapped_field", "foo"); + + // main query needs rewriting + FunctionScoreQueryBuilder functionQueryBuilder1 = new FunctionScoreQueryBuilder(termQueryBuilder); + functionQueryBuilder1.setMinScore(1); + IllegalStateException e = expectThrows(IllegalStateException.class, + () -> functionQueryBuilder1.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + + // filter needs rewriting + FunctionScoreQueryBuilder functionQueryBuilder2 = new FunctionScoreQueryBuilder(new MatchAllQueryBuilder(), + new FilterFunctionBuilder[] { + new FilterFunctionBuilder(termQueryBuilder, new RandomScoreFunctionBuilder()) + }); + e = expectThrows(IllegalStateException.class, + () -> functionQueryBuilder2.toQuery(context)); + assertEquals("Rewrite first", e.getMessage()); + } } From c9c43aa038e76d893a25538bc6d7ea98eeae51ac Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 2 Dec 2019 10:56:38 +0100 Subject: [PATCH 09/21] iter --- .../search/fetch/subphase/highlight/HighlightBuilder.java | 3 ++- .../org/elasticsearch/search/sort/AbstractSortTestCase.java | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java index 9483e76d072a8..e3675b05b6f8f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java @@ -362,7 +362,8 @@ private static void transferOptions(AbstractHighlighterBuilder highlighterBuilde targetOptionsBuilder.options(highlighterBuilder.options); } if (highlighterBuilder.highlightQuery != null) { - targetOptionsBuilder.highlightQuery(highlighterBuilder.highlightQuery.toQuery(context)); + targetOptionsBuilder.highlightQuery( + Rewriteable.rewrite(highlighterBuilder.highlightQuery, context).toQuery(context)); } } diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 9cda381df243b..3104be06d1a0f 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.ScriptEngine; @@ -154,7 +155,8 @@ public void testBuildSortField() throws IOException { QueryShardContext mockShardContext = createMockShardContext(); for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { T sortBuilder = createTestItem(); - SortFieldAndFormat sortField = sortBuilder.build(mockShardContext); + SortFieldAndFormat sortField = Rewriteable.rewrite(sortBuilder, mockShardContext) + .build(mockShardContext); sortFieldAssertions(sortBuilder, sortField.field, sortField.format); } } From 6b907c25df8f04c8cfeb8fc0fef1f2a36bdc8585 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 2 Dec 2019 13:57:44 +0100 Subject: [PATCH 10/21] iter --- .../plain/ConstantKeywordIndexFieldData.java | 8 +++--- .../index/query/QueryShardContext.java | 4 +-- .../query/SpanMultiTermQueryBuilder.java | 3 ++- .../mapper/ConstantKeywordFieldMapper.java | 26 ++++++++----------- .../test/constant_keyword/10_basic.yml | 13 ++++++++++ 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java index 4296084bc15af..f1e71f9fa407b 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java @@ -64,11 +64,11 @@ public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fiel } - private static class SingletonKeywordAtomicFieldData extends AbstractAtomicOrdinalsFieldData { + private static class ConstantKeywordAtomicFieldData extends AbstractAtomicOrdinalsFieldData { private final String value; - SingletonKeywordAtomicFieldData(String value) { + ConstantKeywordAtomicFieldData(String value) { super(DEFAULT_SCRIPT_FUNCTION); this.value = value; } @@ -126,14 +126,14 @@ public void close() { } - private final SingletonKeywordAtomicFieldData atomicFieldData; + private final ConstantKeywordAtomicFieldData atomicFieldData; private ConstantKeywordIndexFieldData(IndexSettings indexSettings, String name, String value) { super(indexSettings, name, null, null, TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE); - atomicFieldData = new SingletonKeywordAtomicFieldData(value); + atomicFieldData = new ConstantKeywordAtomicFieldData(value); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 44db7194561c6..c599fbcb4274e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -255,13 +255,11 @@ public void setMapUnmappedFieldAsString(boolean mapUnmappedFieldAsString) { } MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMapping) { - if (fieldMapping != null) { + if (fieldMapping != null || allowUnmappedFields) { return fieldMapping; } else if (mapUnmappedFieldAsString) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); return builder.build(new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(1))).fieldType(); - } else if (allowUnmappedFields) { - return null; } else { throw new QueryShardException(this, "No field mapping can be found for the field with name [{}]", name); } diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java index 8d636141d22dc..d55f9bceaa9bd 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java @@ -126,7 +126,8 @@ public static SpanMultiTermQueryBuilder fromXContent(XContentParser parser) thro @Override protected Query doToQuery(QueryShardContext context) throws IOException { - // We do the rewrite in toQuery since we need to keep a MultiTermQueryBuilder. + // We do the rewrite in toQuery to not have to deal with the case when a multi-term builder rewrites to a non-multi-term + // builder. QueryBuilder multiTermQueryBuilder = Rewriteable.rewrite(this.multiTermQueryBuilder, context); if (multiTermQueryBuilder instanceof MatchNoneQueryBuilder) { return new SpanMatchNoDocsQuery(this.multiTermQueryBuilder.fieldName(), "Inner query rewrote to match_none"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java index 9351e769b45bb..5106cb5dfb5b1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.Objects; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -30,6 +31,8 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.KeywordFieldMapper.Defaults; +import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.query.QueryShardContext; @@ -38,26 +41,19 @@ public class ConstantKeywordFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "constant_keyword"; public static class Defaults { - public static final String NULL_VALUE = null; - public static final int IGNORE_ABOVE = Integer.MAX_VALUE; + public static final MappedFieldType FIELD_TYPE = new ConstantKeywordFieldType(); + static { + FIELD_TYPE.setIndexOptions(IndexOptions.NONE); + FIELD_TYPE.freeze(); + } } public static class Builder extends FieldMapper.Builder { - private static MappedFieldType createFieldType(String value) { - ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); - ft.setValue(value); - ft.freeze(); - return ft; - } - - private Builder(String name, MappedFieldType fieldType) { - super(name, fieldType, fieldType); - builder = this; - } - public Builder(String name, String value) { - this(name, createFieldType(value)); + super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); + builder = this; + fieldType().setValue(value); } @Override diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml index e6a29e1681e19..362a2e21a9999 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml @@ -167,3 +167,16 @@ setup: - match: { aggregations.foo_terms.buckets.1.doc_count: 1 } - length: { aggregations.foo_terms.buckets: 2 } +--- +"Sort": + + - do: + search: + index: test* + body: + sort: [ { foo: asc } ] + + - match: { "hits.total.value": 3 } + - match: {hits.hits.0._index: test1 } + - match: {hits.hits.1._index: test1 } + - match: {hits.hits.2._index: test2 } From 779d128d8198cf50b4cb9447e709b7a582ac1071 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 2 Dec 2019 14:30:25 +0100 Subject: [PATCH 11/21] unused imports --- .../xpack/core/index/mapper/ConstantKeywordFieldMapper.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java index 5106cb5dfb5b1..a6a9eebbb092b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java @@ -31,8 +31,6 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.KeywordFieldMapper.Defaults; -import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.query.QueryShardContext; From 51dc11889ea74d3b267633456e046d70354ebe68 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 2 Dec 2019 14:36:45 +0100 Subject: [PATCH 12/21] add javadocs --- .../xpack/core/index/mapper/ConstantKeywordFieldMapper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java index a6a9eebbb092b..779c82ff7e596 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java @@ -34,6 +34,9 @@ import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.query.QueryShardContext; +/** + * A {@link FieldMapper} that assigns every document the same value. + */ public class ConstantKeywordFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "constant_keyword"; From 8da5b6c4a43ca5edc100d47edbf4a05bbbaac4cb Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 2 Dec 2019 16:24:22 +0100 Subject: [PATCH 13/21] iter --- .../authz/accesscontrol/DocumentSubsetBitsetCacheTests.java | 1 + .../SecurityIndexReaderWrapperIntegrationTests.java | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index 55b28ba825c1c..934d370a82d0b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -241,6 +241,7 @@ private void runTestOnIndex(CheckedBiConsumer nowInMillis, null, null); + context.setAllowUnmappedFields(false); context.setMapUnmappedFieldAsString(true); body.accept(context, leaf); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index c113d62d1bf8a..7973ad9f555ba 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -72,7 +72,8 @@ public void testDLS() throws Exception { final Authentication authentication = mock(Authentication.class); when(authentication.getUser()).thenReturn(mock(User.class)); threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication); - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), + Settings.builder().put(IndexSettings.ALLOW_UNMAPPED.getKey(), false).build()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); @@ -195,7 +196,8 @@ public void testDLSWithLimitedPermissions() throws Exception { IndicesAccessControl.IndexAccessControl limitedIndexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), DocumentPermissions.filteredBy(queries)); - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), + Settings.builder().put(IndexSettings.ALLOW_UNMAPPED.getKey(), false).build()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); From d1b71f4e96c2f8cd533c116c483acf9a33eefee1 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 3 Dec 2019 20:31:30 +0100 Subject: [PATCH 14/21] More docs. --- docs/reference/how-to/search-speed.asciidoc | 144 ++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc index 91337b0b0a1c8..811b0878500fb 100644 --- a/docs/reference/how-to/search-speed.asciidoc +++ b/docs/reference/how-to/search-speed.asciidoc @@ -424,3 +424,147 @@ The <> field has an <> option that indexes prefixes of all terms and is automatically leveraged by query parsers to run prefix queries. If your use-case involves running lots of prefix queries, this can speed up queries significantly. + +[[faster-filtering-with-constant-keyword]] +=== Use <> to speed up filtering + +There is a general rule that the cost of a filter is mostly a function of the +number of matched documents. For instance, imagine that you have an index with +cycles. If your index has many more bicycles than tricycles, then applying a +filter on `cycle_type: tricycle` is much cheaper than applying a filter on +`cycle_type: bicycle`. There is an exception to this rule, the +<> query. The reason is that queries that +contain `match_all` filters can often remove this filter. For instance assume +the following query: + +[source,console] +-------------------------------------------------- +GET cycles/_search +{ + "query": { + "bool": { + "must": { + "match": { + "description": "dutch" + } + }, + "filter": { + "match_all": {} + } + } + } +} +-------------------------------------------------- +// TEST[s/^/PUT cycles\n/] + +Elasticsearch will internally execute it the same way as: + +[source,console] +-------------------------------------------------- +GET cycles/_search +{ + "query": { + "match": { + "description": "dutch" + } + } +} +-------------------------------------------------- +// TEST[continued] + +This makes sense: the filter had no effect in this case. This might not sound +useful though: why would you add a filter on a `match_all` in the first place? +This is where the <> field comes to play: +queries on this field rewrite to either a `match_all` query, or a query that +doesn't match any documents. See the following snippet: + +[source,console] +-------------------------------------------------- +PUT bicycles +{ + "mappings": { + "properties": { + "cycle_type": { + "type": "constant_keyword", + "value": "bicycle" + }, + "name": { + "type": "text" + } + } + } +} + +PUT other_cycles +{ + "mappings": { + "properties": { + "cycle_type": { + "type": "keyword" + }, + "name": { + "type": "text" + } + } + } +} +-------------------------------------------------- + +We are splitting our index in two: one that will contain only bicycles, and +another one that contains other cycles: unicycles, tricycles, etc. Then at +search time, we need to update the path to include both indices, but don't +need to update queries: + + +[source,console] +-------------------------------------------------- +GET bicycles,other_cycles/_search +{ + "query": { + "bool": { + "must": { + "match": { + "description": "dutch" + } + }, + "filter": { + "term": { + "cycle_type": "bicycle" + } + } + } + } +} +-------------------------------------------------- +// TEST[continued] + +The filter on `cycle_type` is interesting above: the `bicycle` value doesn't +exist in the `other_cycles`, so Elasticsearch will very easily figure out that +this query has no matches. And in the `bicycles` index, this filter will rewrite +to a `match_all`, so the overall query will then be rewritten to remove the +filter: + +[source,console] +-------------------------------------------------- +GET bicycles/_search +{ + "query": { + "match": { + "description": "dutch" + } + } +} +-------------------------------------------------- +// TEST[continued] + +This is a powerful way of making queries cheaper by putting common values in a +dedicated index. This idea can also be combined across multiple fields: for +instance if you track the color of each cycle and your `bicycles` index ends up +having a majority of black bikes, you could split it into a `bicycles-black` +and a `bicycles-other-colors` indices. + +The `constant_keyword` is not strictly required for this optimization: it is +also possible to update the client-side logic in order to route queries to the +relevant indices based on filters. However `constant_keyword` makes it +transparently and allows to decouple search requests from the index topology, +for a negligible overhead. From d34630320d23b1269ec2270c229c13450f92ead8 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 8 Jan 2020 16:19:12 +0100 Subject: [PATCH 15/21] address review comments --- docs/reference/how-to/search-speed.asciidoc | 90 ++++++------------- .../mapping/types/constant-keyword.asciidoc | 2 +- .../index/query/IdsQueryBuilder.java | 5 +- .../license/XPackLicenseState.java | 9 ++ .../xpack/core/XPackClientPlugin.java | 5 +- .../elasticsearch/xpack/core/XPackField.java | 2 + .../elasticsearch/xpack/core/XPackPlugin.java | 6 -- .../core/action/XPackInfoFeatureAction.java | 3 +- .../core/action/XPackUsageFeatureAction.java | 3 +- .../ConstantKeywordFeatureSetUsage.java | 62 +++++++++++++ .../ConstantKeywordFeatureSetUsageTests.java | 43 +++++++++ .../mapper-constant-keyword/build.gradle | 24 +++++ .../ConstantKeywordInfoTransportAction.java | 44 +++++++++ .../ConstantKeywordMapperPlugin.java | 41 +++++++++ .../ConstantKeywordUsageTransportAction.java | 86 ++++++++++++++++++ .../mapper/ConstantKeywordFieldMapper.java | 66 ++++++++++++-- ...stantKeywordUsageTransportActionTests.java | 59 ++++++++++++ .../ConstantKeywordFieldMapperTests.java | 19 ++-- .../mapper/ConstantKeywordFieldTypeTests.java | 39 ++++++-- .../test/constant_keyword/10_basic.yml | 2 +- .../20_constant_keyword_stats.yml | 45 ++++++++++ 21 files changed, 553 insertions(+), 102 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsage.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsageTests.java create mode 100644 x-pack/plugin/mapper-constant-keyword/build.gradle create mode 100644 x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordInfoTransportAction.java create mode 100644 x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordMapperPlugin.java create mode 100644 x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportAction.java rename x-pack/plugin/{core/src/main/java/org/elasticsearch/xpack/core/index => mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword}/mapper/ConstantKeywordFieldMapper.java (75%) create mode 100644 x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportActionTests.java rename x-pack/plugin/{core/src/test/java/org/elasticsearch/xpack/core/index => mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword}/mapper/ConstantKeywordFieldMapperTests.java (86%) rename x-pack/plugin/{core/src/test/java/org/elasticsearch/xpack/core/index => mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword}/mapper/ConstantKeywordFieldTypeTests.java (54%) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_constant_keyword_stats.yml diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc index 25d6f65d97af3..7fa1a2846ea25 100644 --- a/docs/reference/how-to/search-speed.asciidoc +++ b/docs/reference/how-to/search-speed.asciidoc @@ -423,54 +423,21 @@ this can speed up queries significantly. === Use <> to speed up filtering There is a general rule that the cost of a filter is mostly a function of the -number of matched documents. For instance, imagine that you have an index with -cycles. If your index has many more bicycles than tricycles, then applying a -filter on `cycle_type: tricycle` is much cheaper than applying a filter on -`cycle_type: bicycle`. There is an exception to this rule, the -<> query. The reason is that queries that -contain `match_all` filters can often remove this filter. For instance assume -the following query: - -[source,console] --------------------------------------------------- -GET cycles/_search -{ - "query": { - "bool": { - "must": { - "match": { - "description": "dutch" - } - }, - "filter": { - "match_all": {} - } - } - } -} --------------------------------------------------- -// TEST[s/^/PUT cycles\n/] - -Elasticsearch will internally execute it the same way as: - -[source,console] --------------------------------------------------- -GET cycles/_search -{ - "query": { - "match": { - "description": "dutch" - } - } -} --------------------------------------------------- -// TEST[continued] - -This makes sense: the filter had no effect in this case. This might not sound -useful though: why would you add a filter on a `match_all` in the first place? -This is where the <> field comes to play: -queries on this field rewrite to either a `match_all` query, or a query that -doesn't match any documents. See the following snippet: +number of matched documents. Imagine that you have an index containing cycles. +There are a large number of bicycles and many searches perform a filter on +`cycle_type: bicycle`. This very common filter is unfortunately also very costly +since it matches most documents. There is a simple way to avoid running this +filter: move bicycles to their own index and filter bicycles by searching this +index instead of adding a filter to the query. + +Unfortunately this can make client-side logic tricky, which is where +`constant_keyword` helps. By mapping `cycle_type` as a `constant_keyword` with +value `bicycle` on the index that contains bicycles, clients can keep running +the exact same queries as they used to run on the monolithic index and +Elasticsearch will do the right thing on the bicycles index by ignoring filters +on `cycle_type` if the value is `bicycle` and returning no hits otherwise. + +Here is what mappings could look like: [source,console] -------------------------------------------------- @@ -506,8 +473,8 @@ PUT other_cycles We are splitting our index in two: one that will contain only bicycles, and another one that contains other cycles: unicycles, tricycles, etc. Then at -search time, we need to update the path to include both indices, but don't -need to update queries: +search time, we need to search both indices, but we don't need to modify +queries. [source,console] @@ -532,25 +499,24 @@ GET bicycles,other_cycles/_search -------------------------------------------------- // TEST[continued] -The filter on `cycle_type` is interesting above: the `bicycle` value doesn't -exist in the `other_cycles`, so Elasticsearch will very easily figure out that -this query has no matches. And in the `bicycles` index, this filter will rewrite -to a `match_all`, so the overall query will then be rewritten to remove the -filter: +On the `bicycles` index, Elasticsearch will simply ignore the `cycle_type` +filter and rewrite the search request to the one below: [source,console] -------------------------------------------------- -GET bicycles/_search +GET bicycles,other_cycles/_search { - "query": { - "match": { - "description": "dutch" - } + "match": { + "description": "dutch" } } -------------------------------------------------- // TEST[continued] +On the `other_cycles` index, Elasticsearch will quickly figure out that +`bicycle` doesn't exist in the terms dictionary of the `cycle_type` field and +return a search response with no hits. + This is a powerful way of making queries cheaper by putting common values in a dedicated index. This idea can also be combined across multiple fields: for instance if you track the color of each cycle and your `bicycles` index ends up @@ -560,5 +526,5 @@ and a `bicycles-other-colors` indices. The `constant_keyword` is not strictly required for this optimization: it is also possible to update the client-side logic in order to route queries to the relevant indices based on filters. However `constant_keyword` makes it -transparently and allows to decouple search requests from the index topology, -for a negligible overhead. +transparently and allows to decouple search requests from the index topology in +exchange of very little overhead. diff --git a/docs/reference/mapping/types/constant-keyword.asciidoc b/docs/reference/mapping/types/constant-keyword.asciidoc index 8aa4a80567d19..1bb9edb4871e5 100644 --- a/docs/reference/mapping/types/constant-keyword.asciidoc +++ b/docs/reference/mapping/types/constant-keyword.asciidoc @@ -33,7 +33,7 @@ PUT logs-debug `constant_keyword` supports the same queries and aggregations as `keyword` fields do, but takes advantage of the fact that all documents have the same -value per index to be more efficient. +value per index to execute queries more efficiently. It is both allowed to submit documents that don't have a value for the field or that have a value equal to the value configured in mappings. The two below diff --git a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java index 0c5018dcc6983..6c4214a03cb6a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java @@ -152,10 +152,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws @Override protected Query doToQuery(QueryShardContext context) throws IOException { MappedFieldType idField = context.fieldMapper(IdFieldMapper.NAME); - if (idField == null) { - throw new IllegalStateException("Rewrite first"); - } - if (ids.isEmpty()) { + if (idField == null || ids.isEmpty()) { throw new IllegalStateException("Rewrite first"); } return idField.termsQuery(new ArrayList<>(ids), context); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index 3c4c6dc0d6777..e27a3ed8e7009 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -880,4 +880,13 @@ public static boolean isPlatinumOrTrialOperationMode(final OperationMode operati public synchronized XPackLicenseState copyCurrentLicenseState() { return new XPackLicenseState(this); } + + /** + * Determine if support for constant-keyword fields should be enabled. + *

+ * Constant-keyword fields are available for all license types except {@link OperationMode#MISSING}. + */ + public synchronized boolean isConstantKeywordAllowed() { + return status.active; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index efdb5e31118ae..ccea438be935b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.core.constantkeyword.ConstantKeywordFeatureSetUsage; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage; import org.elasticsearch.xpack.core.frozen.FrozenIndicesFeatureSetUsage; @@ -557,7 +558,9 @@ public List getNamedWriteables() { // Spatial new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new), // data science - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new) + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new), + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CONSTANT_KEYWORD, + ConstantKeywordFeatureSetUsage::new) ).stream(), MlEvaluationNamedXContentProvider.getNamedWriteables().stream() ).collect(toList()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 8a74272429f87..9d02fab06e55a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -53,6 +53,8 @@ public final class XPackField { public static final String ANALYTICS = "analytics"; /** Name constant for the enrich plugin. */ public static final String ENRICH = "enrich"; + /** Name constant for the constant-keyword plugin. */ + public static final String CONSTANT_KEYWORD = "constant_keyword"; private XPackField() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 6083eb4f91e34..b8154c18bacfc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -36,7 +36,6 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.LicensesMetaData; import org.elasticsearch.license.Licensing; @@ -62,7 +61,6 @@ import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.action.XPackUsageResponse; -import org.elasticsearch.xpack.core.index.mapper.ConstantKeywordFieldMapper; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.rest.action.RestReloadAnalyzersAction; import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction; @@ -344,8 +342,4 @@ public List> getSettings() { return settings; } - @Override - public Map getMappers() { - return Collections.singletonMap(ConstantKeywordFieldMapper.CONTENT_TYPE, new ConstantKeywordFieldMapper.TypeParser()); - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java index 7a98a459f02d3..be1ddbbb8eab4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java @@ -41,10 +41,11 @@ public class XPackInfoFeatureAction extends ActionType public static final XPackInfoFeatureAction SPATIAL = new XPackInfoFeatureAction(XPackField.SPATIAL); public static final XPackInfoFeatureAction ANALYTICS = new XPackInfoFeatureAction(XPackField.ANALYTICS); public static final XPackInfoFeatureAction ENRICH = new XPackInfoFeatureAction(XPackField.ENRICH); + public static final XPackInfoFeatureAction CONSTANT_KEYWORD = new XPackInfoFeatureAction(XPackField.CONSTANT_KEYWORD); public static final List ALL = Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, - TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH + TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH, CONSTANT_KEYWORD ); private XPackInfoFeatureAction(String name) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index fe43f9661488a..e02e35172cc9d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -40,10 +40,11 @@ public class XPackUsageFeatureAction extends ActionType ALL = Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, - TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS + TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, CONSTANT_KEYWORD ); private XPackUsageFeatureAction(String name) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsage.java new file mode 100644 index 0000000000000..cd8c402273a0b --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsage.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.constantkeyword; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.Objects; + +public class ConstantKeywordFeatureSetUsage extends XPackFeatureSet.Usage { + private final int fieldCount; + + public ConstantKeywordFeatureSetUsage(StreamInput input) throws IOException { + super(input); + this.fieldCount = input.getVersion().onOrAfter(Version.V_8_0_0) ? input.readInt() : 0; + } + + public ConstantKeywordFeatureSetUsage(boolean available, int fieldCount) { + super(XPackField.CONSTANT_KEYWORD, available, true); + this.fieldCount = fieldCount; + } + + public int fieldCount() { + return fieldCount; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeInt(fieldCount); + } + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.field("field_count", fieldCount); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConstantKeywordFeatureSetUsage that = (ConstantKeywordFeatureSetUsage) o; + return available == that.available && enabled == that.enabled && fieldCount == that.fieldCount; + } + + @Override + public int hashCode() { + return Objects.hash(available, enabled, fieldCount); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsageTests.java new file mode 100644 index 0000000000000..14807d8a2fa07 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsageTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.constantkeyword; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; + +public class ConstantKeywordFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + @Override + protected ConstantKeywordFeatureSetUsage createTestInstance() { + return new ConstantKeywordFeatureSetUsage(randomBoolean(), randomIntBetween(0, 1000)); + } + + @Override + protected ConstantKeywordFeatureSetUsage mutateInstance(ConstantKeywordFeatureSetUsage instance) throws IOException { + + boolean available = instance.available(); + int fieldCount = instance.fieldCount(); + + switch (between(0, 1)) { + case 0: + available = !available; + break; + case 1: + fieldCount = randomValueOtherThan(instance.fieldCount(), () -> randomIntBetween(0, 1000)); + break; + } + + return new ConstantKeywordFeatureSetUsage(available, fieldCount); + } + + @Override + protected Writeable.Reader instanceReader() { + return ConstantKeywordFeatureSetUsage::new; + } + +} diff --git a/x-pack/plugin/mapper-constant-keyword/build.gradle b/x-pack/plugin/mapper-constant-keyword/build.gradle new file mode 100644 index 0000000000000..ba4e0d1b2a757 --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'constant-keyword' + description 'Module for the constant-keyword field type, which is a specialization of keyword for the case when all documents have the same value.' + classname 'org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-constant-keyword' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +integTest.enabled = false diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordInfoTransportAction.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordInfoTransportAction.java new file mode 100644 index 0000000000000..621138d1e0230 --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordInfoTransportAction.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.constantkeyword; + +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; + +public class ConstantKeywordInfoTransportAction extends XPackInfoFeatureTransportAction { + + private final XPackLicenseState licenseState; + + @Inject + public ConstantKeywordInfoTransportAction(TransportService transportService, ActionFilters actionFilters, + Settings settings, XPackLicenseState licenseState) { + super(XPackInfoFeatureAction.CONSTANT_KEYWORD.name(), transportService, actionFilters); + this.licenseState = licenseState; + } + + @Override + public String name() { + return XPackField.CONSTANT_KEYWORD; + } + + @Override + public boolean available() { + return licenseState.isConstantKeywordAllowed(); + } + + @Override + public boolean enabled() { + return true; + } + +} diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordMapperPlugin.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordMapperPlugin.java new file mode 100644 index 0000000000000..100226475af31 --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordMapperPlugin.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.constantkeyword; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonMap; + +public class ConstantKeywordMapperPlugin extends Plugin implements MapperPlugin, ActionPlugin { + + public ConstantKeywordMapperPlugin(Settings settings) {} + + @Override + public Map getMappers() { + return singletonMap(ConstantKeywordFieldMapper.CONTENT_TYPE, new ConstantKeywordFieldMapper.TypeParser()); + } + + @Override + public List> getActions() { + return Arrays.asList( + new ActionHandler<>(XPackUsageFeatureAction.CONSTANT_KEYWORD, ConstantKeywordUsageTransportAction.class), + new ActionHandler<>(XPackInfoFeatureAction.CONSTANT_KEYWORD, ConstantKeywordInfoTransportAction.class)); + } +} diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportAction.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportAction.java new file mode 100644 index 0000000000000..13364fe5cbda9 --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportAction.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.constantkeyword; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; +import org.elasticsearch.xpack.core.constantkeyword.ConstantKeywordFeatureSetUsage; + +import java.util.Map; + +public class ConstantKeywordUsageTransportAction extends XPackUsageFeatureTransportAction { + + private final XPackLicenseState licenseState; + + @Inject + public ConstantKeywordUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + Settings settings, XPackLicenseState licenseState) { + super(XPackUsageFeatureAction.CONSTANT_KEYWORD.name(), transportService, clusterService, + threadPool, actionFilters, indexNameExpressionResolver); + this.licenseState = licenseState; + } + + @Override + protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state, + ActionListener listener) { + boolean allowed = licenseState.isConstantKeywordAllowed(); + int fieldCount = 0; + + if (allowed && state != null) { + for (IndexMetaData indexMetaData : state.metaData()) { + MappingMetaData mappingMetaData = indexMetaData.mapping(); + + if (mappingMetaData != null) { + Map mappings = mappingMetaData.getSourceAsMap(); + fieldCount += countConstantKeywordFields(mappings); + } + } + } + + ConstantKeywordFeatureSetUsage usage = new ConstantKeywordFeatureSetUsage(allowed, fieldCount); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + } + + static int countConstantKeywordFields(Map mapping) { + int count = 0; + Object properties = mapping.get("properties"); + if (properties != null && properties instanceof Map) { + @SuppressWarnings("unchecked") + Map propertiesAsMap = (Map) properties; + for (Object v : propertiesAsMap.values()) { + if (v != null && v instanceof Map) { + @SuppressWarnings("unchecked") + Map fieldMapping = (Map) v; + Object fieldType = fieldMapping.get("type"); + if (fieldType != null && fieldType.equals(ConstantKeywordFieldMapper.CONTENT_TYPE)) { + count++; + } + + count += countConstantKeywordFields(fieldMapping); + } + } + } + return count; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java similarity index 75% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java rename to x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 779c82ff7e596..1f934a8844d63 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -5,33 +5,42 @@ */ -package org.elasticsearch.xpack.core.index.mapper; +package org.elasticsearch.xpack.constantkeyword.mapper; import java.io.IOException; +import java.time.ZoneId; import java.util.List; import java.util.Map; import java.util.Objects; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.FuzzyTermsEnum; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.ConstantKeywordIndexFieldData; +import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.query.QueryShardContext; /** @@ -117,7 +126,7 @@ public void checkCompatibility(MappedFieldType otherFT, List conflicts) super.checkCompatibility(otherFT, conflicts); ConstantKeywordFieldType other = (ConstantKeywordFieldType) otherFT; if (Objects.equals(value, other.value) == false) { - conflicts.add("mapper [" + name() + "] has different [value]"); + conflicts.add("mapper [" + name() + "] has different [value]: [" + value + "] vs. [" + other.value + "]"); } } @@ -194,6 +203,48 @@ public Query wildcardQuery(String value, return new MatchNoDocsQuery(); } } + + @Override + public Query rangeQuery( + Object lowerTerm, Object upperTerm, + boolean includeLower, boolean includeUpper, + ShapeRelation relation, ZoneId timeZone, DateMathParser parser, + QueryShardContext context) { + final BytesRef valueAsBytesRef = new BytesRef(value); + if (lowerTerm != null && BytesRefs.toBytesRef(lowerTerm).compareTo(valueAsBytesRef) >= (includeLower ? 1 : 0)) { + return new MatchNoDocsQuery(); + } + if (upperTerm != null && valueAsBytesRef.compareTo(BytesRefs.toBytesRef(upperTerm)) >= (includeUpper ? 1 : 0)) { + return new MatchNoDocsQuery(); + } + return new MatchAllDocsQuery(); + } + + @Override + public Query fuzzyQuery(Object term, Fuzziness fuzziness, int prefixLength, int maxExpansions, + boolean transpositions) { + final String termAsString = BytesRefs.toString(term); + final int maxEdits = fuzziness.asDistance(termAsString); + final Automaton automaton = FuzzyTermsEnum.buildAutomaton(termAsString, prefixLength, transpositions, maxEdits); + final CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton); + if (runAutomaton.run(this.value)) { + return new MatchAllDocsQuery(); + } else { + return new MatchNoDocsQuery(); + } + } + + @Override + public Query regexpQuery(String value, int flags, int maxDeterminizedStates, + MultiTermQuery.RewriteMethod method, QueryShardContext context) { + final Automaton automaton = new RegExp(value, flags).toAutomaton(maxDeterminizedStates); + final CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton); + if (runAutomaton.run(this.value)) { + return new MatchAllDocsQuery(); + } else { + return new MatchNoDocsQuery(); + } + } } protected ConstantKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, @@ -218,16 +269,13 @@ protected void parseCreateField(ParseContext context, List field value = context.externalValue().toString(); } else { XContentParser parser = context.parser(); - if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { - value = fieldType().nullValueAsString(); - } else { - value = parser.textOrNull(); - } + value = parser.textOrNull(); } if (Objects.equals(fieldType().value, value) == false) { throw new IllegalArgumentException("[constant_keyword] field [" + name() + - "] only accepts values that are equal to the wrapped value [" + fieldType().value() + "], but got [" + value + "]"); + "] only accepts values that are equal to the value defined in the mappings [" + fieldType().value() + + "], but got [" + value + "]"); } } @Override diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportActionTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportActionTests.java new file mode 100644 index 0000000000000..2a830f50f52f0 --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportActionTests.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.constantkeyword; + +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class ConstantKeywordUsageTransportActionTests extends ESTestCase { + + public void testCountTopLevelFields() { + Map mapping = new HashMap<>(); + assertEquals(0, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); + + Map properties = new HashMap<>(); + mapping.put("properties", properties); + + Map keywordField = new HashMap<>(); + keywordField.put("type", "keyword"); + properties.put("foo", keywordField); + assertEquals(0, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); + + Map constantKeywordField = new HashMap<>(); + constantKeywordField.put("type", "constant_keyword"); + properties.put("bar", constantKeywordField); + assertEquals(1, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); + + properties.put("baz", constantKeywordField); + assertEquals(2, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); + } + + public void testCountInnerFields() { + Map constantKeywordField = new HashMap<>(); + constantKeywordField.put("type", "constant_keyword"); + + Map properties = new HashMap<>(); + properties.put("foo", constantKeywordField); + + Map objectMapping = new HashMap<>(); + objectMapping.put("properties", properties); + + Map mapping = new HashMap<>(); + assertEquals(0, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); + + properties = new HashMap<>(); + properties.put("obj", objectMapping); + mapping.put("properties", properties); + assertEquals(1, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); + + properties.put("bar", constantKeywordField); + assertEquals(2, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java similarity index 86% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapperTests.java rename to x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 3c5ac10b3699d..e822b36bc5a3d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -4,11 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.core.index.mapper; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +package org.elasticsearch.xpack.constantkeyword.mapper; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -23,15 +19,16 @@ import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.util.Collection; public class ConstantKeywordFieldMapperTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - List> plugins = new ArrayList<>(super.getPlugins()); - plugins.add(XPackPlugin.class); - return plugins; + return pluginList(ConstantKeywordMapperPlugin.class, LocalStateCompositeXPackPlugin.class); } public void testDefaults() throws Exception { @@ -54,8 +51,8 @@ public void testDefaults() throws Exception { .startObject().field("field", "bar").endObject()); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(new SourceToParse("test", "1", illegalSource, XContentType.JSON))); - assertEquals("[constant_keyword] field [field] only accepts values that are equal to the wrapped value [foo], but got [bar]", - e.getCause().getMessage()); + assertEquals("[constant_keyword] field [field] only accepts values that are equal to the value defined in the mappings [foo], " + + "but got [bar]", e.getCause().getMessage()); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java similarity index 54% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldTypeTests.java rename to x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java index fa27aefc8a02f..897f92c68c4a8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/index/mapper/ConstantKeywordFieldTypeTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -4,18 +4,20 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.core.index.mapper; - -import java.util.Arrays; -import java.util.Collections; +package org.elasticsearch.xpack.constantkeyword.mapper; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.xpack.core.index.mapper.ConstantKeywordFieldMapper.ConstantKeywordFieldType; +import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper.ConstantKeywordFieldType; import org.junit.Before; +import java.util.Arrays; +import java.util.Collections; + public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase { @Before @@ -63,4 +65,31 @@ public void testPrefixQuery() { assertEquals(new MatchAllDocsQuery(), ft.prefixQuery("fo", null, null)); assertEquals(new MatchNoDocsQuery(), ft.prefixQuery("ba", null, null)); } + + public void testRangeQuery() { + ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + ft.setValue("foo"); + assertEquals(new MatchAllDocsQuery(), ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null, null, null, null)); + assertEquals(new MatchAllDocsQuery(), ft.rangeQuery("foo", null, true, randomBoolean(), null, null, null, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("foo", null, false, randomBoolean(), null, null, null, null)); + assertEquals(new MatchAllDocsQuery(), ft.rangeQuery(null, "foo", randomBoolean(), true, null, null, null, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(null, "foo", randomBoolean(), false, null, null, null, null)); + assertEquals(new MatchAllDocsQuery(), ft.rangeQuery("abc", "xyz", randomBoolean(), randomBoolean(), null, null, null, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("abc", "def", randomBoolean(), randomBoolean(), null, null, null, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("mno", "xyz", randomBoolean(), randomBoolean(), null, null, null, null)); + } + + public void testFuzzyQuery() { + ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + ft.setValue("foobar"); + assertEquals(new MatchAllDocsQuery(), ft.fuzzyQuery("foobaz", Fuzziness.AUTO, 3, 50, randomBoolean())); + assertEquals(new MatchNoDocsQuery(), ft.fuzzyQuery("fooquux", Fuzziness.AUTO, 3, 50, randomBoolean())); + } + + public void testRegexpQuery() { + ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + ft.setValue("foo"); + assertEquals(new MatchAllDocsQuery(), ft.regexpQuery("f.o", RegExp.ALL, 10, null, null)); + assertEquals(new MatchNoDocsQuery(), ft.regexpQuery("f..o", RegExp.ALL, 10, null, null)); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml index 362a2e21a9999..53dd747897925 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml @@ -1,7 +1,7 @@ setup: - skip: - version: " - 7.99.99" + version: " - 7.99.99" # TODO: make it 7.5.99 after backport reason: "constant_keyword was added in 7.6" - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_constant_keyword_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_constant_keyword_stats.yml new file mode 100644 index 0000000000000..bc7a6bb955be9 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_constant_keyword_stats.yml @@ -0,0 +1,45 @@ +setup: + - skip: + version: " - 7.99.99" # TODO: make this 7.5.99 after backport + reason: "telemetry for constant-keyword fields was added in 7.6" + +--- +"Usage stats for constant-keyword fields": + - do: + xpack.usage: {} + + - match: { constant_keyword.available: true } + - match: { constant_keyword.enabled: true } + - match: { constant_keyword.field_count: 0 } + + - do: + indices.create: + index: test-index1 + body: + mappings: + properties: + quux: + type: constant_keyword + value: foo + + - do: + indices.create: + index: test-index2 + body: + mappings: + properties: + foo: + type: constant_keyword + value: bar + obj: + properties: + bar: + type: constant_keyword + value: quux + + - do: + xpack.usage: {} + + - match: { constant_keyword.available: true } + - match: { constant_keyword.enabled: true } + - match: { constant_keyword.field_count: 3 } From c378cdf35c386d03da63875e301728386cd734fd Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 10 Jan 2020 18:46:32 +0100 Subject: [PATCH 16/21] Fix failures. --- docs/reference/how-to/search-speed.asciidoc | 6 ++++-- docs/reference/rest-api/info.asciidoc | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc index 7fa1a2846ea25..7a73727f401ab 100644 --- a/docs/reference/how-to/search-speed.asciidoc +++ b/docs/reference/how-to/search-speed.asciidoc @@ -506,8 +506,10 @@ filter and rewrite the search request to the one below: -------------------------------------------------- GET bicycles,other_cycles/_search { - "match": { - "description": "dutch" + "query": { + "match": { + "description": "dutch" + } } } -------------------------------------------------- diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index 7c73777dd61f7..18daa14c673c2 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -71,6 +71,10 @@ Example response: "available" : true, "enabled" : true }, + "constant_keyword" : { + "available" : true, + "enabled" : true + }, "enrich" : { "available" : true, "enabled" : true From 11e643ddca955d5c402512a4d25285562af16702 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 27 Feb 2020 11:14:40 +0100 Subject: [PATCH 17/21] iter --- ...dData.java => ConstantIndexFieldData.java} | 6 +- .../index/mapper/TypeFieldMapper.java | 4 +- .../license/XPackLicenseState.java | 4 + .../xpack/core/XPackClientPlugin.java | 5 +- .../ConstantKeywordFeatureSetUsage.java | 62 ------------- .../ConstantKeywordFeatureSetUsageTests.java | 43 ---------- .../ConstantKeywordMapperPlugin.java | 2 - .../ConstantKeywordUsageTransportAction.java | 86 ------------------- .../mapper/ConstantKeywordFieldMapper.java | 80 ++++++----------- ...stantKeywordUsageTransportActionTests.java | 59 ------------- .../mapper/ConstantKeywordFieldTypeTests.java | 4 +- 11 files changed, 37 insertions(+), 318 deletions(-) rename server/src/main/java/org/elasticsearch/index/fielddata/plain/{ConstantKeywordIndexFieldData.java => ConstantIndexFieldData.java} (95%) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsage.java delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsageTests.java delete mode 100644 x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportAction.java delete mode 100644 x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportActionTests.java diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java similarity index 95% rename from server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java rename to server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java index 14598bf2ac551..229cf4541d201 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantKeywordIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java @@ -51,7 +51,7 @@ import java.util.Collections; import java.util.function.Function; -public class ConstantKeywordIndexFieldData extends AbstractIndexOrdinalsFieldData { +public class ConstantIndexFieldData extends AbstractIndexOrdinalsFieldData { public static class Builder implements IndexFieldData.Builder { @@ -64,7 +64,7 @@ public Builder(Function valueFunction) { @Override public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { - return new ConstantKeywordIndexFieldData(indexSettings, fieldType.name(), valueFunction.apply(mapperService)); + return new ConstantIndexFieldData(indexSettings, fieldType.name(), valueFunction.apply(mapperService)); } } @@ -133,7 +133,7 @@ public void close() { private final ConstantKeywordAtomicFieldData atomicFieldData; - private ConstantKeywordIndexFieldData(IndexSettings indexSettings, String name, String value) { + private ConstantIndexFieldData(IndexSettings indexSettings, String name, String value) { super(indexSettings, name, null, null, TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 5e9ed8067b1bd..e5c33b83f24e5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -40,7 +40,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.plain.ConstantKeywordIndexFieldData; +import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -112,7 +112,7 @@ public String typeName() { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { Function typeFunction = mapperService -> mapperService.documentMapper().type(); - return new ConstantKeywordIndexFieldData.Builder(typeFunction); + return new ConstantIndexFieldData.Builder(typeFunction); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index 74677f27b039c..1d90ef9488789 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -626,6 +626,10 @@ public boolean isDataScienceAllowed() { return allowForAllLicenses(); } + public boolean isConstantKeywordAllowed() { + return allowForAllLicenses(); + } + /** * @return true if security is available to be used with the current license type */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index b6418ac79762d..abad8feba7d42 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -36,7 +36,6 @@ import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; -import org.elasticsearch.xpack.core.constantkeyword.ConstantKeywordFeatureSetUsage; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage; import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage; @@ -567,9 +566,7 @@ public List getNamedWriteables() { // Spatial new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new), // data science - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new), - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CONSTANT_KEYWORD, - ConstantKeywordFeatureSetUsage::new) + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new) ).stream(), MlEvaluationNamedXContentProvider.getNamedWriteables().stream() ).collect(toList()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsage.java deleted file mode 100644 index cd8c402273a0b..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsage.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.core.constantkeyword; - -import org.elasticsearch.Version; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.XPackFeatureSet; -import org.elasticsearch.xpack.core.XPackField; - -import java.io.IOException; -import java.util.Objects; - -public class ConstantKeywordFeatureSetUsage extends XPackFeatureSet.Usage { - private final int fieldCount; - - public ConstantKeywordFeatureSetUsage(StreamInput input) throws IOException { - super(input); - this.fieldCount = input.getVersion().onOrAfter(Version.V_8_0_0) ? input.readInt() : 0; - } - - public ConstantKeywordFeatureSetUsage(boolean available, int fieldCount) { - super(XPackField.CONSTANT_KEYWORD, available, true); - this.fieldCount = fieldCount; - } - - public int fieldCount() { - return fieldCount; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { - out.writeInt(fieldCount); - } - } - - @Override - protected void innerXContent(XContentBuilder builder, Params params) throws IOException { - super.innerXContent(builder, params); - builder.field("field_count", fieldCount); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ConstantKeywordFeatureSetUsage that = (ConstantKeywordFeatureSetUsage) o; - return available == that.available && enabled == that.enabled && fieldCount == that.fieldCount; - } - - @Override - public int hashCode() { - return Objects.hash(available, enabled, fieldCount); - } -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsageTests.java deleted file mode 100644 index 14807d8a2fa07..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/constantkeyword/ConstantKeywordFeatureSetUsageTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.constantkeyword; - -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -import java.io.IOException; - -public class ConstantKeywordFeatureSetUsageTests extends AbstractWireSerializingTestCase { - - @Override - protected ConstantKeywordFeatureSetUsage createTestInstance() { - return new ConstantKeywordFeatureSetUsage(randomBoolean(), randomIntBetween(0, 1000)); - } - - @Override - protected ConstantKeywordFeatureSetUsage mutateInstance(ConstantKeywordFeatureSetUsage instance) throws IOException { - - boolean available = instance.available(); - int fieldCount = instance.fieldCount(); - - switch (between(0, 1)) { - case 0: - available = !available; - break; - case 1: - fieldCount = randomValueOtherThan(instance.fieldCount(), () -> randomIntBetween(0, 1000)); - break; - } - - return new ConstantKeywordFeatureSetUsage(available, fieldCount); - } - - @Override - protected Writeable.Reader instanceReader() { - return ConstantKeywordFeatureSetUsage::new; - } - -} diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordMapperPlugin.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordMapperPlugin.java index 100226475af31..b129d3074dc31 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordMapperPlugin.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordMapperPlugin.java @@ -15,7 +15,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import java.util.Arrays; import java.util.List; @@ -35,7 +34,6 @@ public Map getMappers() { @Override public List> getActions() { return Arrays.asList( - new ActionHandler<>(XPackUsageFeatureAction.CONSTANT_KEYWORD, ConstantKeywordUsageTransportAction.class), new ActionHandler<>(XPackInfoFeatureAction.CONSTANT_KEYWORD, ConstantKeywordInfoTransportAction.class)); } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportAction.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportAction.java deleted file mode 100644 index 13364fe5cbda9..0000000000000 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportAction.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.constantkeyword; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.protocol.xpack.XPackUsageRequest; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; -import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; -import org.elasticsearch.xpack.core.constantkeyword.ConstantKeywordFeatureSetUsage; - -import java.util.Map; - -public class ConstantKeywordUsageTransportAction extends XPackUsageFeatureTransportAction { - - private final XPackLicenseState licenseState; - - @Inject - public ConstantKeywordUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - Settings settings, XPackLicenseState licenseState) { - super(XPackUsageFeatureAction.CONSTANT_KEYWORD.name(), transportService, clusterService, - threadPool, actionFilters, indexNameExpressionResolver); - this.licenseState = licenseState; - } - - @Override - protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state, - ActionListener listener) { - boolean allowed = licenseState.isConstantKeywordAllowed(); - int fieldCount = 0; - - if (allowed && state != null) { - for (IndexMetaData indexMetaData : state.metaData()) { - MappingMetaData mappingMetaData = indexMetaData.mapping(); - - if (mappingMetaData != null) { - Map mappings = mappingMetaData.getSourceAsMap(); - fieldCount += countConstantKeywordFields(mappings); - } - } - } - - ConstantKeywordFeatureSetUsage usage = new ConstantKeywordFeatureSetUsage(allowed, fieldCount); - listener.onResponse(new XPackUsageFeatureResponse(usage)); - } - - static int countConstantKeywordFields(Map mapping) { - int count = 0; - Object properties = mapping.get("properties"); - if (properties != null && properties instanceof Map) { - @SuppressWarnings("unchecked") - Map propertiesAsMap = (Map) properties; - for (Object v : propertiesAsMap.values()) { - if (v != null && v instanceof Map) { - @SuppressWarnings("unchecked") - Map fieldMapping = (Map) v; - Object fieldType = fieldMapping.get("type"); - if (fieldType != null && fieldType.equals(ConstantKeywordFieldMapper.CONTENT_TYPE)) { - count++; - } - - count += countConstantKeywordFields(fieldMapping); - } - } - } - return count; - } -} diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 1f934a8844d63..ea11ca4aaf08e 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -15,16 +15,16 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.search.FuzzyTermsEnum; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.LevenshteinAutomata; import org.apache.lucene.util.automaton.RegExp; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.regex.Regex; @@ -34,7 +34,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.plain.ConstantKeywordIndexFieldData; +import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -153,56 +153,13 @@ public String typeName() { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { - return new ConstantKeywordIndexFieldData.Builder(mapperService -> value); + return new ConstantIndexFieldData.Builder(mapperService -> value); } - private static String valueToString(Object v) { - if (v instanceof BytesRef) { - return ((BytesRef) v).utf8ToString(); - } else { - return v.toString(); - } - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - if (Objects.equals(valueToString(value), this.value)) { - return new MatchAllDocsQuery(); - } else { - return new MatchNoDocsQuery(); - } - } - - @Override - public Query termsQuery(List values, QueryShardContext context) { - for (Object v : values) { - if (Objects.equals(valueToString(v), value)) { - return new MatchAllDocsQuery(); - } - } - return new MatchNoDocsQuery(); - } - - @Override - public Query prefixQuery(String value, - @Nullable MultiTermQuery.RewriteMethod method, - QueryShardContext context) { - if (this.value.startsWith(value)) { - return new MatchAllDocsQuery(); - } else { - return new MatchNoDocsQuery(); - } - } - - public Query wildcardQuery(String value, - @Nullable MultiTermQuery.RewriteMethod method, - QueryShardContext context) { - if (Regex.simpleMatch(value, this.value)) { - return new MatchAllDocsQuery(); - } else { - return new MatchNoDocsQuery(); - } - } + @Override + protected boolean matches(String pattern, QueryShardContext context) { + return Regex.simpleMatch(pattern, value); + } @Override public Query rangeQuery( @@ -221,11 +178,23 @@ public Query rangeQuery( } @Override - public Query fuzzyQuery(Object term, Fuzziness fuzziness, int prefixLength, int maxExpansions, - boolean transpositions) { - final String termAsString = BytesRefs.toString(term); + public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, + boolean transpositions, QueryShardContext context) { + final String termAsString = BytesRefs.toString(value); final int maxEdits = fuzziness.asDistance(termAsString); - final Automaton automaton = FuzzyTermsEnum.buildAutomaton(termAsString, prefixLength, transpositions, maxEdits); + + final int[] termText = new int[termAsString.codePointCount(0, termAsString.length())]; + for (int cp, i = 0, j = 0; i < termAsString.length(); i += Character.charCount(cp)) { + termText[j++] = cp = termAsString.codePointAt(i); + } + final int termLength = termText.length; + + prefixLength = Math.min(prefixLength, termLength); + final String suffix = UnicodeUtil.newString(termText, prefixLength, termText.length - prefixLength); + final LevenshteinAutomata builder = new LevenshteinAutomata(suffix, transpositions); + final String prefix = UnicodeUtil.newString(termText, 0, prefixLength); + final Automaton automaton = builder.toAutomaton(maxEdits, prefix); + final CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton); if (runAutomaton.run(this.value)) { return new MatchAllDocsQuery(); @@ -245,6 +214,7 @@ public Query regexpQuery(String value, int flags, int maxDeterminizedStates, return new MatchNoDocsQuery(); } } + } protected ConstantKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportActionTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportActionTests.java deleted file mode 100644 index 2a830f50f52f0..0000000000000 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordUsageTransportActionTests.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.constantkeyword; - -import org.elasticsearch.test.ESTestCase; - -import java.util.HashMap; -import java.util.Map; - -public class ConstantKeywordUsageTransportActionTests extends ESTestCase { - - public void testCountTopLevelFields() { - Map mapping = new HashMap<>(); - assertEquals(0, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); - - Map properties = new HashMap<>(); - mapping.put("properties", properties); - - Map keywordField = new HashMap<>(); - keywordField.put("type", "keyword"); - properties.put("foo", keywordField); - assertEquals(0, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); - - Map constantKeywordField = new HashMap<>(); - constantKeywordField.put("type", "constant_keyword"); - properties.put("bar", constantKeywordField); - assertEquals(1, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); - - properties.put("baz", constantKeywordField); - assertEquals(2, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); - } - - public void testCountInnerFields() { - Map constantKeywordField = new HashMap<>(); - constantKeywordField.put("type", "constant_keyword"); - - Map properties = new HashMap<>(); - properties.put("foo", constantKeywordField); - - Map objectMapping = new HashMap<>(); - objectMapping.put("properties", properties); - - Map mapping = new HashMap<>(); - assertEquals(0, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); - - properties = new HashMap<>(); - properties.put("obj", objectMapping); - mapping.put("properties", properties); - assertEquals(1, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); - - properties.put("bar", constantKeywordField); - assertEquals(2, ConstantKeywordUsageTransportAction.countConstantKeywordFields(mapping)); - } - -} diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java index 897f92c68c4a8..7e023f4921602 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -82,8 +82,8 @@ public void testRangeQuery() { public void testFuzzyQuery() { ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); ft.setValue("foobar"); - assertEquals(new MatchAllDocsQuery(), ft.fuzzyQuery("foobaz", Fuzziness.AUTO, 3, 50, randomBoolean())); - assertEquals(new MatchNoDocsQuery(), ft.fuzzyQuery("fooquux", Fuzziness.AUTO, 3, 50, randomBoolean())); + assertEquals(new MatchAllDocsQuery(), ft.fuzzyQuery("foobaz", Fuzziness.AUTO, 3, 50, randomBoolean(), null)); + assertEquals(new MatchNoDocsQuery(), ft.fuzzyQuery("fooquux", Fuzziness.AUTO, 3, 50, randomBoolean(), null)); } public void testRegexpQuery() { From 97df16911ab6615e5fe46ac957e30935c331b1d8 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 27 Feb 2020 14:41:04 +0100 Subject: [PATCH 18/21] iter --- .../mapping/types/constant-keyword.asciidoc | 4 +- .../plain/ConstantIndexFieldData.java | 8 +- .../SignificantTermsAggregatorFactory.java | 3 +- .../subphase/highlight/HighlightBuilder.java | 3 +- .../elasticsearch/xpack/core/XPackPlugin.java | 4 +- .../mapper/ConstantKeywordFieldMapper.java | 89 ++++++++++++++----- .../ConstantKeywordFieldMapperTests.java | 25 ++++++ .../mapper/ConstantKeywordFieldTypeTests.java | 37 +++++++- .../test/constant_keyword/10_basic.yml | 4 +- .../20_constant_keyword_stats.yml | 45 ---------- .../constant_keyword/20_dynamic_mapping.yml | 38 ++++++++ 11 files changed, 177 insertions(+), 83 deletions(-) delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_constant_keyword_stats.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_dynamic_mapping.yml diff --git a/docs/reference/mapping/types/constant-keyword.asciidoc b/docs/reference/mapping/types/constant-keyword.asciidoc index 1bb9edb4871e5..2d5f083da5634 100644 --- a/docs/reference/mapping/types/constant-keyword.asciidoc +++ b/docs/reference/mapping/types/constant-keyword.asciidoc @@ -68,6 +68,6 @@ The following mapping parameters are accepted: `value`:: - The value to associate with all documents in the index. This parameter is - required and can't be updated on an existing index. + The value to associate with all documents in the index. If this parameter + is not provided, it is set based on the first document that gets indexed. diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java index 229cf4541d201..9fb7e20e28382 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java @@ -69,11 +69,11 @@ public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fiel } - private static class ConstantKeywordAtomicFieldData extends AbstractAtomicOrdinalsFieldData { + private static class ConstantAtomicFieldData extends AbstractAtomicOrdinalsFieldData { private final String value; - ConstantKeywordAtomicFieldData(String value) { + ConstantAtomicFieldData(String value) { super(DEFAULT_SCRIPT_FUNCTION); this.value = value; } @@ -131,14 +131,14 @@ public void close() { } - private final ConstantKeywordAtomicFieldData atomicFieldData; + private final ConstantAtomicFieldData atomicFieldData; private ConstantIndexFieldData(IndexSettings indexSettings, String name, String value) { super(indexSettings, name, null, null, TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE); - atomicFieldData = new ConstantKeywordAtomicFieldData(value); + atomicFieldData = new ConstantAtomicFieldData(value); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java index bcf94f01d4d72..0687c81a64802 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java @@ -37,7 +37,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; @@ -98,7 +97,7 @@ public SignificantTermsAggregatorFactory(String name, this.executionHint = executionHint; this.filter = filterBuilder == null ? null - : Rewriteable.rewrite(filterBuilder, queryShardContext).toQuery(queryShardContext); + : filterBuilder.toQuery(queryShardContext); IndexSearcher searcher = queryShardContext.searcher(); this.supersetNumDocs = filter == null // Important - need to use the doc count that includes deleted docs diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java index e3675b05b6f8f..9483e76d072a8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java @@ -362,8 +362,7 @@ private static void transferOptions(AbstractHighlighterBuilder highlighterBuilde targetOptionsBuilder.options(highlighterBuilder.options); } if (highlighterBuilder.highlightQuery != null) { - targetOptionsBuilder.highlightQuery( - Rewriteable.rewrite(highlighterBuilder.highlightQuery, context).toQuery(context)); + targetOptionsBuilder.highlightQuery(highlighterBuilder.highlightQuery.toQuery(context)); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index fd94b18c8c528..bfa8fd1ba17f8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -42,7 +42,6 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.ExtensiblePlugin; -import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.protocol.xpack.XPackInfoResponse; @@ -85,7 +84,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; -public class XPackPlugin extends XPackClientPlugin implements ExtensiblePlugin, RepositoryPlugin, EnginePlugin, MapperPlugin { +public class XPackPlugin extends XPackClientPlugin implements ExtensiblePlugin, RepositoryPlugin, EnginePlugin { private static Logger logger = LogManager.getLogger(XPackPlugin.class); private static DeprecationLogger deprecationLogger = new DeprecationLogger(logger); @@ -342,5 +341,4 @@ public List> getSettings() { settings.add(SourceOnlySnapshotRepository.SOURCE_ONLY); return settings; } - } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index ea11ca4aaf08e..44c7c3162d9d6 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -60,10 +60,14 @@ public static class Defaults { public static class Builder extends FieldMapper.Builder { - public Builder(String name, String value) { + public Builder(String name) { super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); builder = this; + } + + public Builder setValue(String value) { fieldType().setValue(value); + return this; } @Override @@ -83,15 +87,22 @@ public ConstantKeywordFieldMapper build(BuilderContext context) { public static class TypeParser implements Mapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - final Object value = node.remove("value"); - if (value == null) { - throw new MapperParsingException("Property [value] of field [" + name + "] is required and can't be null."); + Object value = null; + if (node.containsKey("value")) { + value = node.remove("value"); + if (value == null) { + throw new MapperParsingException("Property [value] of field [" + name + "] can't be [null]."); + } + if (value instanceof Number == false && value instanceof CharSequence == false) { + throw new MapperParsingException("Property [value] of field [" + name + + "] must be a number or a string, but got [" + value + "]"); + } } - if (value instanceof Number == false && value instanceof CharSequence == false) { - throw new MapperParsingException("Property [value] of field [" + name + - "] must be a number or a string, but got [" + value + "]"); + ConstantKeywordFieldMapper.Builder builder = new ConstantKeywordFieldMapper.Builder(name); + if (value != null) { + builder.setValue(value.toString()); } - return new ConstantKeywordFieldMapper.Builder(name, value.toString()); + return builder; } } @@ -122,11 +133,16 @@ public boolean equals(Object o) { } @Override - public void checkCompatibility(MappedFieldType otherFT, List conflicts) { - super.checkCompatibility(otherFT, conflicts); - ConstantKeywordFieldType other = (ConstantKeywordFieldType) otherFT; - if (Objects.equals(value, other.value) == false) { - conflicts.add("mapper [" + name() + "] has different [value]: [" + value + "] vs. [" + other.value + "]"); + public void checkCompatibility(MappedFieldType newFT, List conflicts) { + super.checkCompatibility(newFT, conflicts); + ConstantKeywordFieldType newConstantKeywordFT = (ConstantKeywordFieldType) newFT; + if (this.value != null) { + if (newConstantKeywordFT.value == null) { + conflicts.add("mapper [" + name() + "] cannot unset [value]"); + } else if (Objects.equals(value, newConstantKeywordFT.value) == false) { + conflicts.add("mapper [" + name() + "] has different [value] from the value that is configured in mappings: [" + value + + "] vs. [" + newConstantKeywordFT.value + "]"); + } } } @@ -156,10 +172,13 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { return new ConstantIndexFieldData.Builder(mapperService -> value); } - @Override - protected boolean matches(String pattern, QueryShardContext context) { - return Regex.simpleMatch(pattern, value); - } + @Override + protected boolean matches(String pattern, QueryShardContext context) { + if (value == null) { + return false; + } + return Regex.simpleMatch(pattern, value); + } @Override public Query rangeQuery( @@ -167,6 +186,10 @@ public Query rangeQuery( boolean includeLower, boolean includeUpper, ShapeRelation relation, ZoneId timeZone, DateMathParser parser, QueryShardContext context) { + if (this.value == null) { + return new MatchNoDocsQuery(); + } + final BytesRef valueAsBytesRef = new BytesRef(value); if (lowerTerm != null && BytesRefs.toBytesRef(lowerTerm).compareTo(valueAsBytesRef) >= (includeLower ? 1 : 0)) { return new MatchNoDocsQuery(); @@ -179,10 +202,14 @@ public Query rangeQuery( @Override public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, - boolean transpositions, QueryShardContext context) { + boolean transpositions, QueryShardContext context) { + if (this.value == null) { + return new MatchNoDocsQuery(); + } + final String termAsString = BytesRefs.toString(value); final int maxEdits = fuzziness.asDistance(termAsString); - + final int[] termText = new int[termAsString.codePointCount(0, termAsString.length())]; for (int cp, i = 0, j = 0; i < termAsString.length(); i += Character.charCount(cp)) { termText[j++] = cp = termAsString.codePointAt(i); @@ -206,6 +233,10 @@ public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int @Override public Query regexpQuery(String value, int flags, int maxDeterminizedStates, MultiTermQuery.RewriteMethod method, QueryShardContext context) { + if (this.value == null) { + return new MatchNoDocsQuery(); + } + final Automaton automaton = new RegExp(value, flags).toAutomaton(maxDeterminizedStates); final CharacterRunAutomaton runAutomaton = new CharacterRunAutomaton(automaton); if (runAutomaton.run(this.value)) { @@ -217,7 +248,7 @@ public Query regexpQuery(String value, int flags, int maxDeterminizedStates, } - protected ConstantKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + ConstantKeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings) { super(simpleName, fieldType, defaultFieldType, indexSettings, MultiFields.empty(), CopyTo.empty()); } @@ -242,12 +273,24 @@ protected void parseCreateField(ParseContext context, List field value = parser.textOrNull(); } - if (Objects.equals(fieldType().value, value) == false) { + if (value == null) { + throw new IllegalArgumentException("[constant_keyword] field [" + name() + "] doesn't accept [null] values"); + } + + if (fieldType().value == null) { + ConstantKeywordFieldType newFieldType = new ConstantKeywordFieldType(fieldType()); + newFieldType.setValue(value); + newFieldType.freeze(); + Mapper update = new ConstantKeywordFieldMapper( + simpleName(), newFieldType, defaultFieldType, context.indexSettings().getSettings()); + context.addDynamicMapper(update); + } else if (Objects.equals(fieldType().value, value) == false) { throw new IllegalArgumentException("[constant_keyword] field [" + name() + "] only accepts values that are equal to the value defined in the mappings [" + fieldType().value() + "], but got [" + value + "]"); } } + @Override protected String contentType() { return CONTENT_TYPE; @@ -256,6 +299,8 @@ protected String contentType() { @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); - builder.field("value", fieldType().value()); + if (fieldType().value() != null) { + builder.field("value", fieldType().value()); + } } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index e822b36bc5a3d..cdab3340129c2 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -55,4 +55,29 @@ public void testDefaults() throws Exception { "but got [bar]", e.getCause().getMessage()); } + public void testDynamicValue() throws Exception { + IndexService indexService = createIndex("test"); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "constant_keyword") + .endObject().endObject().endObject().endObject()); + DocumentMapper mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "foo").endObject()); + ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", source, XContentType.JSON)); + assertNull(doc.rootDoc().getField("field")); + assertNotNull(doc.dynamicMappingsUpdate()); + + CompressedXContent mappingUpdate = new CompressedXContent(Strings.toString(doc.dynamicMappingsUpdate())); + DocumentMapper updatedMapper = indexService.mapperService().merge("_doc", mappingUpdate, MergeReason.MAPPING_UPDATE); + String expectedMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "constant_keyword") + .field("value", "foo").endObject().endObject().endObject().endObject()); + assertEquals(expectedMapping, updatedMapper.mappingSource().toString()); + + doc = updatedMapper.parse(new SourceToParse("test", "1", source, XContentType.JSON)); + assertNull(doc.rootDoc().getField("field")); + assertNull(doc.dynamicMappingsUpdate()); + } + } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java index 7e023f4921602..45843d3b3102f 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -15,8 +15,10 @@ import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper.ConstantKeywordFieldType; import org.junit.Before; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase { @@ -30,13 +32,38 @@ public void modify(MappedFieldType type) { }); } + public void testSetValue() { + ConstantKeywordFieldType ft1 = new ConstantKeywordFieldType(); + ft1.setName("field"); + ConstantKeywordFieldType ft2 = new ConstantKeywordFieldType(); + ft2.setName("field"); + ft2.setValue("bar"); + List conflicts = new ArrayList<>(); + ft1.checkCompatibility(ft2, conflicts); + assertEquals(Collections.emptyList(), conflicts); + } + + public void testUnsetValue() { + ConstantKeywordFieldType ft1 = new ConstantKeywordFieldType(); + ft1.setName("field"); + ft1.setValue("foo"); + ConstantKeywordFieldType ft2 = new ConstantKeywordFieldType(); + ft2.setName("field"); + List conflicts = new ArrayList<>(); + ft1.checkCompatibility(ft2, conflicts); + assertEquals(Collections.singletonList("mapper [field] cannot unset [value]"), conflicts); + } + @Override protected MappedFieldType createDefaultFieldType() { - return new ConstantKeywordFieldType(); + ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + ft.setValue("foo"); + return ft; } public void testTermQuery() { ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + assertEquals(new MatchNoDocsQuery(), ft.termQuery("foo", null)); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.termQuery("foo", null)); assertEquals(new MatchNoDocsQuery(), ft.termQuery("bar", null)); @@ -44,6 +71,7 @@ public void testTermQuery() { public void testTermsQuery() { ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + assertEquals(new MatchNoDocsQuery(), ft.termsQuery(Collections.singletonList("foo"), null)); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.termsQuery(Collections.singletonList("foo"), null)); assertEquals(new MatchAllDocsQuery(), ft.termsQuery(Arrays.asList("bar", "foo", "quux"), null)); @@ -54,6 +82,7 @@ public void testTermsQuery() { public void testWildcardQuery() { ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("f*o", null, null)); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("f*o", null, null)); assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("b*r", null, null)); @@ -61,6 +90,7 @@ public void testWildcardQuery() { public void testPrefixQuery() { ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + assertEquals(new MatchNoDocsQuery(), ft.prefixQuery("fo", null, null)); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.prefixQuery("fo", null, null)); assertEquals(new MatchNoDocsQuery(), ft.prefixQuery("ba", null, null)); @@ -68,6 +98,9 @@ public void testPrefixQuery() { public void testRangeQuery() { ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null, null, null, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(null, "foo", randomBoolean(), randomBoolean(), null, null, null, null)); + assertEquals(new MatchNoDocsQuery(), ft.rangeQuery("foo", null, randomBoolean(), randomBoolean(), null, null, null, null)); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null, null, null, null)); assertEquals(new MatchAllDocsQuery(), ft.rangeQuery("foo", null, true, randomBoolean(), null, null, null, null)); @@ -81,6 +114,7 @@ public void testRangeQuery() { public void testFuzzyQuery() { ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + assertEquals(new MatchNoDocsQuery(), ft.fuzzyQuery("fooquux", Fuzziness.AUTO, 3, 50, randomBoolean(), null)); ft.setValue("foobar"); assertEquals(new MatchAllDocsQuery(), ft.fuzzyQuery("foobaz", Fuzziness.AUTO, 3, 50, randomBoolean(), null)); assertEquals(new MatchNoDocsQuery(), ft.fuzzyQuery("fooquux", Fuzziness.AUTO, 3, 50, randomBoolean(), null)); @@ -88,6 +122,7 @@ public void testFuzzyQuery() { public void testRegexpQuery() { ConstantKeywordFieldType ft = new ConstantKeywordFieldType(); + assertEquals(new MatchNoDocsQuery(), ft.regexpQuery("f..o", RegExp.ALL, 10, null, null)); ft.setValue("foo"); assertEquals(new MatchAllDocsQuery(), ft.regexpQuery("f.o", RegExp.ALL, 10, null, null)); assertEquals(new MatchNoDocsQuery(), ft.regexpQuery("f..o", RegExp.ALL, 10, null, null)); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml index 53dd747897925..0635de2add546 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml @@ -1,8 +1,8 @@ setup: - skip: - version: " - 7.99.99" # TODO: make it 7.5.99 after backport - reason: "constant_keyword was added in 7.6" + version: " - 7.99.99" # TODO: make it 7.6.99 after backport + reason: "constant_keyword was added in 7.7" - do: indices.create: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_constant_keyword_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_constant_keyword_stats.yml deleted file mode 100644 index bc7a6bb955be9..0000000000000 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_constant_keyword_stats.yml +++ /dev/null @@ -1,45 +0,0 @@ -setup: - - skip: - version: " - 7.99.99" # TODO: make this 7.5.99 after backport - reason: "telemetry for constant-keyword fields was added in 7.6" - ---- -"Usage stats for constant-keyword fields": - - do: - xpack.usage: {} - - - match: { constant_keyword.available: true } - - match: { constant_keyword.enabled: true } - - match: { constant_keyword.field_count: 0 } - - - do: - indices.create: - index: test-index1 - body: - mappings: - properties: - quux: - type: constant_keyword - value: foo - - - do: - indices.create: - index: test-index2 - body: - mappings: - properties: - foo: - type: constant_keyword - value: bar - obj: - properties: - bar: - type: constant_keyword - value: quux - - - do: - xpack.usage: {} - - - match: { constant_keyword.available: true } - - match: { constant_keyword.enabled: true } - - match: { constant_keyword.field_count: 3 } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_dynamic_mapping.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_dynamic_mapping.yml new file mode 100644 index 0000000000000..61a78edb0b15e --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_dynamic_mapping.yml @@ -0,0 +1,38 @@ +--- +"Dynamic mappings": + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + foo: + type: constant_keyword + + - do: + index: + index: test1 + id: 1 + body: {} + + - do: + indices.get_mapping: + index: test1 + + - match: { test1.mappings.properties.foo.type: constant_keyword } + - is_false: test1.mappings.properties.foo.value + + - do: + index: + index: test1 + id: 1 + body: + foo: bar + + - do: + indices.get_mapping: + index: test1 + + - match: { test1.mappings.properties.foo.type: constant_keyword } + - match: { test1.mappings.properties.foo.value: bar } From c384b07219c3b935ed729ab56f8a46534d7def19 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 27 Feb 2020 15:25:38 +0100 Subject: [PATCH 19/21] Remove constant_keyword from usage API. --- .../xpack/core/action/XPackUsageFeatureAction.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index 7deed49923cf5..ffc1651ab74ef 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -41,11 +41,10 @@ public class XPackUsageFeatureAction extends ActionType ALL = Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, - TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, CONSTANT_KEYWORD + TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS ); private XPackUsageFeatureAction(String name) { From 460f3958fd4335dd1e4eee2e18e2ff4f0113e560 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 28 Feb 2020 08:46:37 +0100 Subject: [PATCH 20/21] iter --- .../mapping/types/constant-keyword.asciidoc | 8 ++++ .../plain/ConstantIndexFieldData.java | 3 ++ .../mapper/ConstantKeywordFieldMapper.java | 2 +- .../constant_keyword/20_dynamic_mapping.yml | 44 +++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/constant-keyword.asciidoc b/docs/reference/mapping/types/constant-keyword.asciidoc index 2d5f083da5634..4b565d7427b7f 100644 --- a/docs/reference/mapping/types/constant-keyword.asciidoc +++ b/docs/reference/mapping/types/constant-keyword.asciidoc @@ -59,6 +59,14 @@ POST logs-debug/_doc However providing a value that is different from the one configured in the mapping is disallowed. +In case no `value` is provided in the mappings, the field will automatically +configure itself based on the value contained in the first indexed document. +While this behavior can be convenient, note that it means that a single +poisonous document can cause all other documents to be rejected if it had a +wrong value. + +The `value` of the field cannot be changed after it has been set. + [[constant-keyword-params]] ==== Parameters for constant keyword fields diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java index 9fb7e20e28382..d0dad640ff979 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java @@ -91,6 +91,9 @@ public Collection getChildResources() { @Override public SortedSetDocValues getOrdinalsValues() { + if (value == null) { + return DocValues.emptySortedSet(); + } final BytesRef term = new BytesRef(value); final SortedDocValues sortedValues = new AbstractSortedDocValues() { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 44c7c3162d9d6..798878578c130 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -151,7 +151,7 @@ public int hashCode() { return 31 * super.hashCode() + Objects.hashCode(value); } - /** Return the value that this field wraps. */ + /** Return the value that this field wraps. This may be {@code null} if the field is not configured yet. */ public String value() { return value; } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_dynamic_mapping.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_dynamic_mapping.yml index 61a78edb0b15e..4357b4ecf171a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_dynamic_mapping.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/20_dynamic_mapping.yml @@ -23,6 +23,47 @@ - match: { test1.mappings.properties.foo.type: constant_keyword } - is_false: test1.mappings.properties.foo.value + - do: + index: + index: test1 + id: 1 + body: {} + + - do: + indices.refresh: {} + + - do: + indices.get_mapping: + index: test1 + + - match: { test1.mappings.properties.foo.type: constant_keyword } + - is_false: test1.mappings.properties.foo.value + + - do: + search: + index: test1 + body: + size: 0 + query: + term: + foo: + value: bar + + - match: { hits.total.value: 0 } + + - do: + search: + index: test1 + body: + size: 0 + aggs: + foo_terms: + terms: + field: foo + + - match: { hits.total.value: 1 } + - length: { aggregations.foo_terms.buckets: 0 } + - do: index: index: test1 @@ -30,6 +71,9 @@ body: foo: bar + - do: + indices.refresh: {} + - do: indices.get_mapping: index: test1 From 4f833f2873fbf4afccbc4c0766cad31dcb17b7a2 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 2 Mar 2020 10:01:30 +0100 Subject: [PATCH 21/21] Add support for metadata. --- .../mapping/types/constant-keyword.asciidoc | 4 +++ .../mapper/ConstantKeywordFieldMapper.java | 2 ++ .../ConstantKeywordFieldMapperTests.java | 27 +++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/docs/reference/mapping/types/constant-keyword.asciidoc b/docs/reference/mapping/types/constant-keyword.asciidoc index 4b565d7427b7f..3f9008d265d25 100644 --- a/docs/reference/mapping/types/constant-keyword.asciidoc +++ b/docs/reference/mapping/types/constant-keyword.asciidoc @@ -74,6 +74,10 @@ The following mapping parameters are accepted: [horizontal] +<>:: + + Metadata about the field. + `value`:: The value to associate with all documents in the index. If this parameter diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 798878578c130..115968b69ee65 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.query.QueryShardContext; /** @@ -102,6 +103,7 @@ public Mapper.Builder parse(String name, Map node, ParserCo if (value != null) { builder.setValue(value.toString()); } + TypeParsers.parseMeta(builder, name, node); return builder; } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index cdab3340129c2..587f032563b2d 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import java.util.Collection; +import java.util.Collections; public class ConstantKeywordFieldMapperTests extends ESSingleNodeTestCase { @@ -80,4 +81,30 @@ public void testDynamicValue() throws Exception { assertNull(doc.dynamicMappingsUpdate()); } + public void testMeta() throws Exception { + IndexService indexService = createIndex("test"); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "constant_keyword") + .field("meta", Collections.singletonMap("foo", "bar")) + .endObject().endObject().endObject().endObject()); + + DocumentMapper mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "constant_keyword") + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + assertEquals(mapping2, mapper.mappingSource().toString()); + + String mapping3 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "constant_keyword") + .field("meta", Collections.singletonMap("baz", "quux")) + .endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", + new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); + assertEquals(mapping3, mapper.mappingSource().toString()); + } }