Make simple_query_string leniency more fine-grained

src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java

@@ -19,7 +19,8 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.search.Query;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.*;
 
 import java.util.Locale;
 import java.util.Map;
@@ -50,11 +51,19 @@ private Query rethrowUnlessLenient(RuntimeException e) {
 
     @Override
     public Query newDefaultQuery(String text) {
-        try {
-            return super.newDefaultQuery(text);
-        } catch (RuntimeException e) {
-            return rethrowUnlessLenient(e);
+        BooleanQuery bq = new BooleanQuery(true);
+        for (Map.Entry<String,Float> entry : weights.entrySet()) {
+            try {
+                Query q = createBooleanQuery(entry.getKey(), text, super.getDefaultOperator());
+                if (q != null) {
+                    q.setBoost(entry.getValue());
+                    bq.add(q, BooleanClause.Occur.SHOULD);
+                }
+            } catch (RuntimeException e) {
+                rethrowUnlessLenient(e);
+            }
         }
+        return super.simplify(bq);
     }
 
     /**
@@ -66,20 +75,36 @@ public Query newFuzzyQuery(String text, int fuzziness) {
         if (settings.lowercaseExpandedTerms()) {
             text = text.toLowerCase(settings.locale());
         }
-        try {
-            return super.newFuzzyQuery(text, fuzziness);
-        } catch (RuntimeException e) {
-            return rethrowUnlessLenient(e);
+        BooleanQuery bq = new BooleanQuery(true);
+        for (Map.Entry<String,Float> entry : weights.entrySet()) {
+            try {
+                Query q = new FuzzyQuery(new Term(entry.getKey(), text), fuzziness);
+                if (q != null) {
+                    q.setBoost(entry.getValue());
+                    bq.add(q, BooleanClause.Occur.SHOULD);
+                }
+            } catch (RuntimeException e) {
+                rethrowUnlessLenient(e);
+            }
         }
+        return super.simplify(bq);
     }
 
     @Override
     public Query newPhraseQuery(String text, int slop) {
-        try {
-            return super.newPhraseQuery(text, slop);
-        } catch (RuntimeException e) {
-            return rethrowUnlessLenient(e);
+        BooleanQuery bq = new BooleanQuery(true);
+        for (Map.Entry<String,Float> entry : weights.entrySet()) {
+            try {
+                Query q = createPhraseQuery(entry.getKey(), text, slop);
+                if (q != null) {
+                    q.setBoost(entry.getValue());
+                    bq.add(q, BooleanClause.Occur.SHOULD);
+                }
+            } catch (RuntimeException e) {
+                rethrowUnlessLenient(e);
+            }
         }
+        return super.simplify(bq);
     }
 
     /**
@@ -91,11 +116,17 @@ public Query newPrefixQuery(String text) {
         if (settings.lowercaseExpandedTerms()) {
             text = text.toLowerCase(settings.locale());
         }
-        try {
-            return super.newPrefixQuery(text);
-        } catch (RuntimeException e) {
-            return rethrowUnlessLenient(e);
+        BooleanQuery bq = new BooleanQuery(true);
+        for (Map.Entry<String,Float> entry : weights.entrySet()) {
+            try {
+                PrefixQuery prefix = new PrefixQuery(new Term(entry.getKey(), text));
+                prefix.setBoost(entry.getValue());
+                bq.add(prefix, BooleanClause.Occur.SHOULD);
+            } catch (RuntimeException e) {
+                return rethrowUnlessLenient(e);
+            }
         }
+        return super.simplify(bq);
     }
 
     /**

src/test/java/org/elasticsearch/search/query/SimpleQueryStringTests.java

@@ -0,0 +1,269 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.query;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.SimpleQueryStringBuilder;
+import org.elasticsearch.index.query.SimpleQueryStringFlag;
+import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.queryString;
+import static org.elasticsearch.index.query.QueryBuilders.simpleQueryString;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Tests for the {@code simple_query_string} query
+ */
+public class SimpleQueryStringTests extends ElasticsearchIntegrationTest {
+
+    @Test
+    public void testSimpleQueryString() throws ExecutionException, InterruptedException {
+        createIndex("test");
+        indexRandom(true, false,
+                client().prepareIndex("test", "type1", "1").setSource("body", "foo"),
+                client().prepareIndex("test", "type1", "2").setSource("body", "bar"),
+                client().prepareIndex("test", "type1", "3").setSource("body", "foo bar"),
+                client().prepareIndex("test", "type1", "4").setSource("body", "quux baz eggplant"),
+                client().prepareIndex("test", "type1", "5").setSource("body", "quux baz spaghetti"),
+                client().prepareIndex("test", "type1", "6").setSource("otherbody", "spaghetti"));
+
+        SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryString("foo bar")).get();
+        assertHitCount(searchResponse, 3l);
+        assertSearchHits(searchResponse, "1", "2", "3");
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("foo bar").defaultOperator(SimpleQueryStringBuilder.Operator.AND)).get();
+        assertHitCount(searchResponse, 1l);
+        assertFirstHit(searchResponse, hasId("3"));
+
+        searchResponse = client().prepareSearch().setQuery(simpleQueryString("\"quux baz\" +(eggplant | spaghetti)")).get();
+        assertHitCount(searchResponse, 2l);
+        assertSearchHits(searchResponse, "4", "5");
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("eggplants").analyzer("snowball")).get();
+        assertHitCount(searchResponse, 1l);
+        assertFirstHit(searchResponse, hasId("4"));
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("spaghetti").field("body", 10.0f).field("otherbody", 2.0f).queryName("myquery")).get();
+        assertHitCount(searchResponse, 2l);
+        assertFirstHit(searchResponse, hasId("5"));
+        assertSearchHits(searchResponse, "5", "6");
+        assertThat(searchResponse.getHits().getAt(0).getMatchedQueries()[0], equalTo("myquery"));
+
+        searchResponse = client().prepareSearch().setQuery(simpleQueryString("spaghetti").field("*body")).get();
+        assertHitCount(searchResponse, 2l);
+        assertSearchHits(searchResponse, "5", "6");
+
+        // Have to bypass the builder here because the builder always uses "fields" instead of "field"
+        searchResponse = client().prepareSearch().setQuery("{\"simple_query_string\": {\"query\": \"spaghetti\", \"field\": \"_all\"}}").get();
+        assertHitCount(searchResponse, 2l);
+        assertSearchHits(searchResponse, "5", "6");
+    }
+
+    @Test
+    public void testSimpleQueryStringLowercasing() {
+        createIndex("test");
+        client().prepareIndex("test", "type1", "1").setSource("body", "Professional").get();
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryString("Professio*")).get();
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("Professio*").lowercaseExpandedTerms(false)).get();
+        assertHitCount(searchResponse, 0l);
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("Professionan~1")).get();
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("Professionan~1").lowercaseExpandedTerms(false)).get();
+        assertHitCount(searchResponse, 0l);
+    }
+
+    @Test
+    public void testQueryStringLocale() {
+        createIndex("test");
+        client().prepareIndex("test", "type1", "1").setSource("body", "bılly").get();
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryString("BILL*")).get();
+        assertHitCount(searchResponse, 0l);
+        searchResponse = client().prepareSearch().setQuery(queryString("body:BILL*")).get();
+        assertHitCount(searchResponse, 0l);
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("BILL*").locale(new Locale("tr", "TR"))).get();
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+        searchResponse = client().prepareSearch().setQuery(
+                queryString("body:BILL*").locale(new Locale("tr", "TR"))).get();
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+    }
+
+    @Test
+    public void testNestedFieldSimpleQueryString() throws IOException {
+        assertAcked(prepareCreate("test")
+                .addMapping("type1", jsonBuilder()
+                        .startObject()
+                        .startObject("type1")
+                        .startObject("properties")
+                        .startObject("body").field("type", "string")
+                        .startObject("fields")
+                        .startObject("sub").field("type", "string")
+                        .endObject() // sub
+                        .endObject() // fields
+                        .endObject() // body
+                        .endObject() // properties
+                        .endObject() // type1
+                        .endObject()));
+        client().prepareIndex("test", "type1", "1").setSource("body", "foo bar baz").get();
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("foo bar baz").field("body")).get();
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("foo bar baz").field("type1.body")).get();
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("foo bar baz").field("body.sub")).get();
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("foo bar baz").field("type1.body.sub")).get();
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+    }
+
+    @Test
+    public void testSimpleQueryStringFlags() throws ExecutionException, InterruptedException {
+        createIndex("test");
+        indexRandom(true,
+                client().prepareIndex("test", "type1", "1").setSource("body", "foo"),
+                client().prepareIndex("test", "type1", "2").setSource("body", "bar"),
+                client().prepareIndex("test", "type1", "3").setSource("body", "foo bar"),
+                client().prepareIndex("test", "type1", "4").setSource("body", "quux baz eggplant"),
+                client().prepareIndex("test", "type1", "5").setSource("body", "quux baz spaghetti"),
+                client().prepareIndex("test", "type1", "6").setSource("otherbody", "spaghetti"));
+
+        SearchResponse searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("foo bar").flags(SimpleQueryStringFlag.ALL)).get();
+        assertHitCount(searchResponse, 3l);
+        assertSearchHits(searchResponse, "1", "2", "3");
+
+        // Sending a negative 'flags' value is the same as SimpleQueryStringFlag.ALL
+        searchResponse = client().prepareSearch().setQuery("{\"simple_query_string\": {\"query\": \"foo bar\", \"flags\": -1}}").get();
+        assertHitCount(searchResponse, 3l);
+        assertSearchHits(searchResponse, "1", "2", "3");
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("foo | bar")
+                        .defaultOperator(SimpleQueryStringBuilder.Operator.AND)
+                        .flags(SimpleQueryStringFlag.OR)).get();
+        assertHitCount(searchResponse, 3l);
+        assertSearchHits(searchResponse, "1", "2", "3");
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("foo | bar")
+                        .defaultOperator(SimpleQueryStringBuilder.Operator.AND)
+                        .flags(SimpleQueryStringFlag.NONE)).get();
+        assertHitCount(searchResponse, 1l);
+        assertFirstHit(searchResponse, hasId("3"));
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("baz | egg*")
+                        .defaultOperator(SimpleQueryStringBuilder.Operator.AND)
+                        .flags(SimpleQueryStringFlag.NONE)).get();
+        assertHitCount(searchResponse, 0l);
+
+        searchResponse = client().prepareSearch().setSource("{\n" +
+                "  \"query\": {\n" +
+                "    \"simple_query_string\": {\n" +
+                "      \"query\": \"foo|bar\",\n" +
+                "      \"default_operator\": \"AND\"," +
+                "      \"flags\": \"NONE\"\n" +
+                "    }\n" +
+                "  }\n" +
+                "}").get();
+        assertHitCount(searchResponse, 1l);
+
+        searchResponse = client().prepareSearch().setQuery(
+                simpleQueryString("baz | egg*")
+                        .defaultOperator(SimpleQueryStringBuilder.Operator.AND)
+                        .flags(SimpleQueryStringFlag.WHITESPACE, SimpleQueryStringFlag.PREFIX)).get();
+        assertHitCount(searchResponse, 1l);
+        assertFirstHit(searchResponse, hasId("4"));
+    }
+
+    @Test
+    public void testSimpleQueryStringLenient() throws ExecutionException, InterruptedException {
+        createIndex("test1", "test2");
+        indexRandom(true, client().prepareIndex("test1", "type1", "1").setSource("field", "foo"),
+                client().prepareIndex("test2", "type1", "10").setSource("field", 5));
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryString("foo").field("field")).get();
+        assertFailures(searchResponse);
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+
+        searchResponse = client().prepareSearch().setQuery(simpleQueryString("foo").field("field").lenient(true)).get();
+        assertNoFailures(searchResponse);
+        assertHitCount(searchResponse, 1l);
+        assertSearchHits(searchResponse, "1");
+    }
+
+    @Test // see: https://github.com/elasticsearch/elasticsearch/issues/7967
+    public void testLenientFlagBeingTooLenient() throws Exception {
+        indexRandom(true,
+                client().prepareIndex("test", "doc", "1").setSource("num", 1, "body", "foo bar baz"),
+                client().prepareIndex("test", "doc", "2").setSource("num", 2, "body", "eggplant spaghetti lasagna"));
+
+        BoolQueryBuilder q = boolQuery().should(simpleQueryString("bar").field("num").field("body").lenient(true));
+        SearchResponse resp = client().prepareSearch("test").setQuery(q).get();
+        assertNoFailures(resp);
+        // the bug is that this would be parsed into basically a match_all
+        // query and this would match both documents
+        assertHitCount(resp, 1);
+        assertSearchHits(resp, "1");
+    }
+}