Add an option to split keyword field on whitespace at query time (#30691)

This change adds an option named `split_queries_on_whitespace` to the `keyword`
field type. When set to true full text queries (`match`, `multi_match`, `query_string`, ...) that target the field will split the input on whitespace to build the query terms. Defaults to `false`.
Closes #30393

Author: jimczi

docs/reference/mapping/types/keyword.asciidoc

@@ -103,6 +103,12 @@ The following parameters are accepted by `keyword` fields:
     How to pre-process the keyword prior to indexing. Defaults to `null`,
     meaning the keyword is kept as-is.
 
+`split_queries_on_whitespace`::
+
+    Whether <<full-text-queries,full text queries>> should split the input on whitespace
+    when building a query for this field.
+    Accepts `true` or `false` (default).
+
 NOTE: Indexes imported from 2.x do not support `keyword`. Instead they will
 attempt to downgrade `keyword` into `string`. This allows you to merge modern
 mappings with legacy mappings. Long lived indexes will have to be recreated

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java

@@ -186,7 +186,7 @@ public Set<Entry<String, NamedAnalyzer>> entrySet() {
                     return Collections.emptySet();
                 }
             };
-            try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(indexSettings, fakeDefault, fakeDefault, fakeDefault, analyzerMap, analyzerMap)) {
+            try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(indexSettings, fakeDefault, fakeDefault, fakeDefault, analyzerMap, analyzerMap, analyzerMap)) {
                 MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService,
                         mapperRegistry, () -> null);
                 mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, false);

server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java

@@ -20,6 +20,7 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -461,13 +462,16 @@ public IndexAnalyzers build(IndexSettings indexSettings,
         Map<String, NamedAnalyzer> analyzerAliases = new HashMap<>();
         Map<String, NamedAnalyzer> analyzers = new HashMap<>();
         Map<String, NamedAnalyzer> normalizers = new HashMap<>();
+        Map<String, NamedAnalyzer> whitespaceNormalizers = new HashMap<>();
         for (Map.Entry<String, AnalyzerProvider<?>> entry : analyzerProviders.entrySet()) {
             processAnalyzerFactory(deprecationLogger, indexSettings, entry.getKey(), entry.getValue(), analyzerAliases, analyzers,
                 tokenFilterFactoryFactories, charFilterFactoryFactories, tokenizerFactoryFactories);
         }
         for (Map.Entry<String, AnalyzerProvider<?>> entry : normalizerProviders.entrySet()) {
-            processNormalizerFactory(deprecationLogger, indexSettings, entry.getKey(), entry.getValue(), normalizers,
-                    tokenizerFactoryFactories.get("keyword"), tokenFilterFactoryFactories, charFilterFactoryFactories);
+            processNormalizerFactory(entry.getKey(), entry.getValue(), normalizers,
+                    "keyword", tokenizerFactoryFactories.get("keyword"), tokenFilterFactoryFactories, charFilterFactoryFactories);
+            processNormalizerFactory(entry.getKey(), entry.getValue(), whitespaceNormalizers,
+                    "whitespace", () -> new WhitespaceTokenizer(), tokenFilterFactoryFactories, charFilterFactoryFactories);
         }
         for (Map.Entry<String, NamedAnalyzer> entry : analyzerAliases.entrySet()) {
             String key = entry.getKey();
@@ -514,7 +518,7 @@ public IndexAnalyzers build(IndexSettings indexSettings,
             }
         }
         return new IndexAnalyzers(indexSettings, defaultIndexAnalyzer, defaultSearchAnalyzer, defaultSearchQuoteAnalyzer,
-            unmodifiableMap(analyzers), unmodifiableMap(normalizers));
+            unmodifiableMap(analyzers), unmodifiableMap(normalizers), unmodifiableMap(whitespaceNormalizers));
     }
 
     private void processAnalyzerFactory(DeprecationLogger deprecationLogger,
@@ -581,20 +585,20 @@ private void processAnalyzerFactory(DeprecationLogger deprecationLogger,
         }
     }
 
-    private void processNormalizerFactory(DeprecationLogger deprecationLogger,
-            IndexSettings indexSettings,
-            String name,
+    private void processNormalizerFactory(String name,
             AnalyzerProvider<?> normalizerFactory,
             Map<String, NamedAnalyzer> normalizers,
-            TokenizerFactory keywordTokenizerFactory,
+            String tokenizerName,
+            TokenizerFactory tokenizerFactory,
             Map<String, TokenFilterFactory> tokenFilters,
             Map<String, CharFilterFactory> charFilters) {
-        if (keywordTokenizerFactory == null) {
+
+        if (tokenizerFactory == null) {
             throw new IllegalStateException("keyword tokenizer factory is null, normalizers require analysis-common module");
         }
 
         if (normalizerFactory instanceof CustomNormalizerProvider) {
-            ((CustomNormalizerProvider) normalizerFactory).build(keywordTokenizerFactory, charFilters, tokenFilters);
+            ((CustomNormalizerProvider) normalizerFactory).build(tokenizerName, tokenizerFactory, charFilters, tokenFilters);
         }
         Analyzer normalizerF = normalizerFactory.get();
         if (normalizerF == null) {

server/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java

@@ -38,15 +38,14 @@ public final class CustomNormalizerProvider extends AbstractIndexAnalyzerProvide
     private CustomAnalyzer customAnalyzer;
 
     public CustomNormalizerProvider(IndexSettings indexSettings,
-                                  String name, Settings settings) {
+                                    String name, Settings settings) {
         super(indexSettings, name, settings);
         this.analyzerSettings = settings;
     }
 
-    public void build(final TokenizerFactory keywordTokenizerFactory, final Map<String, CharFilterFactory> charFilters,
+    public void build(final String tokenizerName, final TokenizerFactory tokenizerFactory, final Map<String, CharFilterFactory> charFilters,
             final Map<String, TokenFilterFactory> tokenFilters) {
-        String tokenizerName = analyzerSettings.get("tokenizer");
-        if (tokenizerName != null) {
+        if (analyzerSettings.get("tokenizer") != null) {
             throw new IllegalArgumentException("Custom normalizer [" + name() + "] cannot configure a tokenizer");
         }
 
@@ -82,8 +81,8 @@ public void build(final TokenizerFactory keywordTokenizerFactory, final Map<Stri
         }
 
         this.customAnalyzer = new CustomAnalyzer(
-                "keyword",
-                keywordTokenizerFactory,
+                tokenizerName,
+                tokenizerFactory,
                 charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
                 tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()])
         );

server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java

@@ -40,17 +40,19 @@ public final class IndexAnalyzers extends AbstractIndexComponent implements Clos
     private final NamedAnalyzer defaultSearchQuoteAnalyzer;
     private final Map<String, NamedAnalyzer> analyzers;
     private final Map<String, NamedAnalyzer> normalizers;
+    private final Map<String, NamedAnalyzer> whitespaceNormalizers;
     private final IndexSettings indexSettings;
 
     public IndexAnalyzers(IndexSettings indexSettings, NamedAnalyzer defaultIndexAnalyzer, NamedAnalyzer defaultSearchAnalyzer,
                           NamedAnalyzer defaultSearchQuoteAnalyzer, Map<String, NamedAnalyzer> analyzers,
-                          Map<String, NamedAnalyzer> normalizers) {
+                          Map<String, NamedAnalyzer> normalizers, Map<String, NamedAnalyzer> whitespaceNormalizers) {
         super(indexSettings);
         this.defaultIndexAnalyzer = defaultIndexAnalyzer;
         this.defaultSearchAnalyzer = defaultSearchAnalyzer;
         this.defaultSearchQuoteAnalyzer = defaultSearchQuoteAnalyzer;
         this.analyzers = analyzers;
         this.normalizers = normalizers;
+        this.whitespaceNormalizers = whitespaceNormalizers;
         this.indexSettings = indexSettings;
     }
 
@@ -68,6 +70,13 @@ public NamedAnalyzer getNormalizer(String name) {
         return normalizers.get(name);
     }
 
+    /**
+     * Returns a normalizer that splits on whitespace mapped to the given name or <code>null</code> if not present
+     */
+    public NamedAnalyzer getWhitespaceNormalizer(String name) {
+        return whitespaceNormalizers.get(name);
+    }
+
     /**
      * Returns the default index analyzer for this index
      */

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;
@@ -35,6 +36,8 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
@@ -73,6 +76,8 @@ public static class Builder extends FieldMapper.Builder<Builder, KeywordFieldMap
 
         protected String nullValue = Defaults.NULL_VALUE;
         protected int ignoreAbove = Defaults.IGNORE_ABOVE;
+        private IndexAnalyzers indexAnalyzers;
+        private String normalizerName;
 
         public Builder(String name) {
             super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@@ -106,15 +111,36 @@ public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
             return builder;
         }
 
-        public Builder normalizer(NamedAnalyzer normalizer) {
-            fieldType().setNormalizer(normalizer);
-            fieldType().setSearchAnalyzer(normalizer);
+        public Builder splitQueriesOnWhitespace(boolean splitQueriesOnWhitespace) {
+            fieldType().setSplitQueriesOnWhitespace(splitQueriesOnWhitespace);
+            return builder;
+        }
+
+        public Builder normalizer(IndexAnalyzers indexAnalyzers, String name) {
+            this.indexAnalyzers = indexAnalyzers;
+            this.normalizerName = name;
             return builder;
         }
 
         @Override
         public KeywordFieldMapper build(BuilderContext context) {
             setupFieldType(context);
+            if (normalizerName != null) {
+                NamedAnalyzer normalizer = indexAnalyzers.getNormalizer(normalizerName);
+                if (normalizer == null) {
+                    throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
+                }
+                fieldType().setNormalizer(normalizer);
+                final NamedAnalyzer searchAnalyzer;
+                if (fieldType().splitQueriesOnWhitespace) {
+                    searchAnalyzer = indexAnalyzers.getWhitespaceNormalizer(normalizerName);
+                } else {
+                    searchAnalyzer = normalizer;
+                }
+                fieldType().setSearchAnalyzer(searchAnalyzer);
+            } else if (fieldType().splitQueriesOnWhitespace) {
+                fieldType().setSearchAnalyzer(new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()));
+            }
             return new KeywordFieldMapper(
                     name, fieldType, defaultFieldType, ignoreAbove, includeInAll,
                     context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
@@ -147,13 +173,12 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserCo
                     iterator.remove();
                 } else if (propName.equals("normalizer")) {
                     if (propNode != null) {
-                        NamedAnalyzer normalizer = parserContext.getIndexAnalyzers().getNormalizer(propNode.toString());
-                        if (normalizer == null) {
-                            throw new MapperParsingException("normalizer [" + propNode.toString() + "] not found for field [" + name + "]");
-                        }
-                        builder.normalizer(normalizer);
+                        builder.normalizer(parserContext.getIndexAnalyzers(), propNode.toString());
                     }
                     iterator.remove();
+                } else if (propName.equals("split_queries_on_whitespace")) {
+                    builder.splitQueriesOnWhitespace(XContentMapValues.nodeBooleanValue(propNode, "split_queries_on_whitespace"));
+                    iterator.remove();
                 }
             }
             return builder;
@@ -163,6 +188,7 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserCo
     public static final class KeywordFieldType extends StringFieldType {
 
         private NamedAnalyzer normalizer = null;
+        private boolean splitQueriesOnWhitespace;
 
         public KeywordFieldType() {
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
@@ -172,6 +198,7 @@ public KeywordFieldType() {
         protected KeywordFieldType(KeywordFieldType ref) {
             super(ref);
             this.normalizer = ref.normalizer;
+            this.splitQueriesOnWhitespace = splitQueriesOnWhitespace;
         }
 
         public KeywordFieldType clone() {
@@ -183,7 +210,9 @@ public boolean equals(Object o) {
             if (super.equals(o) == false) {
                 return false;
             }
-            return Objects.equals(normalizer, ((KeywordFieldType) o).normalizer);
+            KeywordFieldType other = (KeywordFieldType) o;
+            return Objects.equals(normalizer, other.normalizer) &&
+                splitQueriesOnWhitespace == other.splitQueriesOnWhitespace;
         }
 
         @Override
@@ -197,7 +226,7 @@ public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts,
 
         @Override
         public int hashCode() {
-            return 31 * super.hashCode() + Objects.hashCode(normalizer);
+            return 31 * super.hashCode() + Objects.hash(normalizer, splitQueriesOnWhitespace);
         }
 
         @Override
@@ -214,6 +243,15 @@ public void setNormalizer(NamedAnalyzer normalizer) {
             this.normalizer = normalizer;
         }
 
+        public boolean splitQueriesOnWhitespace() {
+            return splitQueriesOnWhitespace;
+        }
+
+        public void setSplitQueriesOnWhitespace(boolean splitQueriesOnWhitespace) {
+            checkIfFrozen();
+            this.splitQueriesOnWhitespace = splitQueriesOnWhitespace;
+        }
+
         @Override
         public Query existsQuery(QueryShardContext context) {
             if (hasDocValues()) {
@@ -393,5 +431,9 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         } else if (includeDefaults) {
             builder.nullField("normalizer");
         }
+
+        if (includeDefaults || fieldType().splitQueriesOnWhitespace) {
+            builder.field("split_queries_on_whitespace", fieldType().splitQueriesOnWhitespace);
+        }
     }
 }

server/src/main/java/org/elasticsearch/index/search/MatchQuery.java

@@ -54,6 +54,7 @@
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.support.QueryParsers;
@@ -264,7 +265,8 @@ public Query parse(Type type, String fieldName, Object value) throws IOException
          * passing through QueryBuilder.
          */
         boolean noForcedAnalyzer = this.analyzer == null;
-        if (fieldType.tokenized() == false && noForcedAnalyzer) {
+        if (fieldType.tokenized() == false && noForcedAnalyzer &&
+                fieldType instanceof KeywordFieldMapper.KeywordFieldType == false) {
             return blendTermQuery(new Term(fieldName, value.toString()), fieldType);
         }
 

server/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java

@@ -54,6 +54,12 @@ public void testBasics() throws IOException {
         assertEquals("my_normalizer", normalizer.name());
         assertTokenStreamContents(normalizer.tokenStream("foo", "Cet été-là"), new String[] {"cet été-là"});
         assertEquals(new BytesRef("cet été-là"), normalizer.normalize("foo", "Cet été-là"));
+
+        normalizer = analysis.indexAnalyzers.getWhitespaceNormalizer("my_normalizer");
+        assertNotNull(normalizer);
+        assertEquals("my_normalizer", normalizer.name());
+        assertTokenStreamContents(normalizer.tokenStream("foo", "Cet été-là"), new String[] {"cet", "été-là"});
+        assertEquals(new BytesRef("cet été-là"), normalizer.normalize("foo", "Cet été-là"));
     }
 
     public void testUnknownType() {
@@ -88,7 +94,13 @@ public void testCharFilters() throws IOException {
         NamedAnalyzer normalizer = analysis.indexAnalyzers.getNormalizer("my_normalizer");
         assertNotNull(normalizer);
         assertEquals("my_normalizer", normalizer.name());
-        assertTokenStreamContents(normalizer.tokenStream("foo", "abc"), new String[] {"zbc"});
+        assertTokenStreamContents(normalizer.tokenStream("foo", "abc acd"), new String[] {"zbc zcd"});
+        assertEquals(new BytesRef("zbc"), normalizer.normalize("foo", "abc"));
+
+        normalizer = analysis.indexAnalyzers.getWhitespaceNormalizer("my_normalizer");
+        assertNotNull(normalizer);
+        assertEquals("my_normalizer", normalizer.name());
+        assertTokenStreamContents(normalizer.tokenStream("foo", "abc acd"), new String[] {"zbc", "zcd"});
         assertEquals(new BytesRef("zbc"), normalizer.normalize("foo", "abc"));
     }