Deprecate CommonTermsQuery and cutoff_frequency

Since the max_score optimization landed in Elasticsearch 7,
the CommonTermsQuery is redundant and slower. Moreover the
cutoff_frequency parameter for MatchQuery and MultiMatchQuery
is also redundant.

Relates to elastic#27096

Author: matriv

docs/reference/query-dsl/common-terms-query.asciidoc

@@ -1,6 +1,8 @@
 [[query-dsl-common-terms-query]]
 === Common Terms Query
 
+deprecated[7.3.0,Replaced by the more performant `max_score` optimization which is applied automatically without any configuration on a <<query-dsl-match-query>>]
+
 The `common` terms query is a modern alternative to stopwords which
 improves the precision and recall of search results (by taking stopwords
 into account), without sacrificing performance.

docs/reference/query-dsl/match-query.asciidoc

@@ -122,6 +122,8 @@ GET /_search
 [[query-dsl-match-query-cutoff]]
 ===== Cutoff frequency
 
+deprecated[7.3.0,Replaced by the more performant `max_score` optimization which is applied automatically without any configuration]
+
 The match query supports a `cutoff_frequency` that allows
 specifying an absolute or relative document frequency where high
 frequency terms are moved into an optional subquery and are only scored

server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java

@@ -278,6 +278,11 @@ public int hashCode() {
         return Objects.hash(classHash(), Arrays.hashCode(equalsTerms()));
     }
 
+    /**
+     * @deprecated Since max_score optimization landed in 7.0, normal MultiMatchQuery
+     *             will achieve the same result without any configuration.
+     */
+    @Deprecated
     public static BlendedTermQuery commonTermsBlendedQuery(Term[] terms, final float[] boosts, final float maxTermFrequency) {
         return new BlendedTermQuery(terms, boosts) {
             @Override

server/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java

@@ -26,7 +26,11 @@
  * Extended version of {@link CommonTermsQuery} that allows to pass in a
  * {@code minimumNumberShouldMatch} specification that uses the actual num of high frequent terms
  * to calculate the minimum matching terms.
+ *
+ * @deprecated Since max_optimization optimization landed in 7.0, normal MatchQuery
+ *             will achieve the same result without any configuration.
  */
+@Deprecated
 public class ExtendedCommonTermsQuery extends CommonTermsQuery {
 
     public ExtendedCommonTermsQuery(Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency) {

server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java

@@ -47,7 +47,11 @@
  * and high-frequency terms are added to an optional boolean clause. The
  * optional clause is only executed if the required "low-frequency' clause
  * matches.
+ *
+ * @deprecated Since max_optimization optimization landed in 7.0, normal MatchQuery
+ *             will achieve the same result without any configuration.
  */
+@Deprecated
 public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQueryBuilder> {
 
     public static final String NAME = "common";

server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java

@@ -19,12 +19,14 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@@ -42,8 +44,20 @@
  * result of the analysis.
  */
 public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
+
+    private static final DeprecationLogger deprecationLogger =
+        new DeprecationLogger(LogManager.getLogger(MatchQueryBuilder.class));
+
+    static final String CUTOFF_FREQUENCY_DEPRECATION_MSG = "[cutoff_frequency] has been deprecated in favor of the " +
+        "[max_score] optimization which is applied automatically without any configuration";
+
     public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query");
-    public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency");
+    /**
+     * @deprecated Since max_optimization optimization landed in 7.0, normal MatchQuery
+     *             will achieve the same result without any configuration.
+     */
+    @Deprecated
+    public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency", "cutoff_frequency");
     public static final ParseField LENIENT_FIELD = new ParseField("lenient");
     public static final ParseField FUZZY_TRANSPOSITIONS_FIELD = new ParseField("fuzzy_transpositions");
     public static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite");
@@ -85,6 +99,10 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
 
     private MatchQuery.ZeroTermsQuery zeroTermsQuery = MatchQuery.DEFAULT_ZERO_TERMS_QUERY;
 
+    /**
+     * @deprecated See {@link MatchQueryBuilder#CUTOFF_FREQUENCY_FIELD} for more details
+     */
+    @Deprecated
     private Float cutoffFrequency = null;
 
     private boolean autoGenerateSynonymsPhraseQuery = true;
@@ -235,8 +253,12 @@ public int maxExpansions() {
      * Set a cutoff value in [0..1] (or absolute number &gt;=1) representing the
      * maximum threshold of a terms document frequency to be considered a low
      * frequency term.
+     *
+     * @deprecated see {@link MatchQueryBuilder#CUTOFF_FREQUENCY_FIELD} for more details
      */
+    @Deprecated
     public MatchQueryBuilder cutoffFrequency(float cutoff) {
+        deprecationLogger.deprecated(CUTOFF_FREQUENCY_DEPRECATION_MSG);
         this.cutoffFrequency = cutoff;
         return this;
     }

server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
@@ -28,6 +29,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@@ -50,6 +52,13 @@
  * Same as {@link MatchQueryBuilder} but supports multiple fields.
  */
 public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQueryBuilder> {
+
+    private static final DeprecationLogger deprecationLogger =
+        new DeprecationLogger(LogManager.getLogger(MultiMatchQueryBuilder.class));
+
+    static final String CUTOFF_FREQUENCY_DEPRECATION_MSG = "[cutoff_frequency] has been deprecated in favor of the " +
+        "[max_score] optimization which is applied automatically without any configuration";
+
     public static final String NAME = "multi_match";
 
     public static final MultiMatchQueryBuilder.Type DEFAULT_TYPE = MultiMatchQueryBuilder.Type.BEST_FIELDS;
@@ -63,7 +72,12 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
     private static final ParseField SLOP_FIELD = new ParseField("slop");
     private static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query");
     private static final ParseField LENIENT_FIELD = new ParseField("lenient");
-    private static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency");
+    /**
+     * @deprecated Since max_score optimization landed in 7.0, normal MultiMatchQuery
+     *             will achieve the same result without any configuration.
+     */
+    @Deprecated
+    private static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency", "cutoff_frequency");
     private static final ParseField TIE_BREAKER_FIELD = new ParseField("tie_breaker");
     private static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite");
     private static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match");
@@ -91,6 +105,10 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
     private String fuzzyRewrite = null;
     private Float tieBreaker;
     private Boolean lenient;
+    /**
+     * @deprecated See {@link MultiMatchQueryBuilder#CUTOFF_FREQUENCY_FIELD} for more details
+     */
+    @Deprecated
     private Float cutoffFrequency = null;
     private MatchQuery.ZeroTermsQuery zeroTermsQuery = DEFAULT_ZERO_TERMS_QUERY;
     private boolean autoGenerateSynonymsPhraseQuery = true;
@@ -484,8 +502,11 @@ public boolean lenient() {
      * Set a cutoff value in [0..1] (or absolute number &gt;=1) representing the
      * maximum threshold of a terms document frequency to be considered a low
      * frequency term.
+     * @deprecated See {@link MultiMatchQueryBuilder#CUTOFF_FREQUENCY_FIELD} for more details
      */
+    @Deprecated
     public MultiMatchQueryBuilder cutoffFrequency(float cutoff) {
+        deprecationLogger.deprecated(CUTOFF_FREQUENCY_DEPRECATION_MSG);
         this.cutoffFrequency = cutoff;
         return this;
     }
@@ -494,8 +515,14 @@ public MultiMatchQueryBuilder cutoffFrequency(float cutoff) {
      * Set a cutoff value in [0..1] (or absolute number &gt;=1) representing the
      * maximum threshold of a terms document frequency to be considered a low
      * frequency term.
+     *
+     * @deprecated See {@link MultiMatchQueryBuilder#CUTOFF_FREQUENCY_FIELD} for more details
      */
+    @Deprecated
     public MultiMatchQueryBuilder cutoffFrequency(Float cutoff) {
+        if (cutoff != null) {
+            deprecationLogger.deprecated(CUTOFF_FREQUENCY_DEPRECATION_MSG);
+        }
         this.cutoffFrequency = cutoff;
         return this;
     }

server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java

@@ -66,7 +66,10 @@ public static MatchQueryBuilder matchQuery(String name, Object text) {
      *
      * @param fieldName The field name.
      * @param text The query text (to be analyzed).
+     *
+     * @deprecated See {@link CommonTermsQueryBuilder}
      */
+    @Deprecated
     public static CommonTermsQueryBuilder commonTermsQuery(String fieldName, Object text) {
         return new CommonTermsQueryBuilder(fieldName, text);
     }

server/src/main/java/org/elasticsearch/index/search/MatchQuery.java

@@ -55,6 +55,7 @@
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.TextFieldMapper;
+import org.elasticsearch.index.query.CommonTermsQueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.support.QueryParsers;
 
@@ -171,6 +172,10 @@ public void writeTo(StreamOutput out) throws IOException {
 
     protected ZeroTermsQuery zeroTermsQuery = DEFAULT_ZERO_TERMS_QUERY;
 
+    /**
+     * @deprecated See {@link MatchQueryBuilder#setCommonTermsCutoff(Float)} for more details
+     */
+    @Deprecated
     protected Float commonTermsCutoff = null;
 
     protected boolean autoGenerateSynonymsPhraseQuery = true;
@@ -194,6 +199,10 @@ public void setOccur(BooleanClause.Occur occur) {
         this.occur = occur;
     }
 
+    /**
+     * @deprecated See {@link MatchQueryBuilder#setCommonTermsCutoff(Float)} for more details
+     */
+    @Deprecated
     public void setCommonTermsCutoff(Float cutoff) {
         this.commonTermsCutoff = cutoff;
     }
@@ -298,6 +307,10 @@ protected final Query parseInternal(Type type, String fieldName, MatchQueryBuild
         return query == null ? zeroTermsQuery() : query;
     }
 
+    /**
+     * @deprecated See {@link CommonTermsQueryBuilder}
+     */
+    @Deprecated
     private Query createCommonTermsQuery(MatchQueryBuilder builder, String field, String queryText,
                                          Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency) {
         Query booleanQuery = builder.createBooleanQuery(field, queryText, lowFreqOccur);
@@ -308,6 +321,10 @@ private Query createCommonTermsQuery(MatchQueryBuilder builder, String field, St
         return booleanQuery;
     }
 
+    /**
+     * @deprecated See {@link CommonTermsQueryBuilder}
+     */
+    @Deprecated
     private Query boolToExtendedCommonTermsQuery(BooleanQuery bq,
                                                  Occur highFreqOccur,
                                                  Occur lowFreqOccur,

server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java

@@ -37,6 +37,10 @@
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
 
+/**
+ * @deprecated See {@link CommonTermsQueryBuilder}
+ */
+@Deprecated
 public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTermsQueryBuilder> {
 
     @Override

server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryParserTests.java

@@ -22,10 +22,12 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
-import java.io.IOException;
-
+/**
+ * @deprecated See {@link CommonTermsQueryBuilder}
+ */
+@Deprecated
 public class CommonTermsQueryParserTests extends ESSingleNodeTestCase {
-    public void testWhenParsedQueryIsNullNoNullPointerExceptionIsThrown() throws IOException {
+    public void testWhenParsedQueryIsNullNoNullPointerExceptionIsThrown() {
         final String index = "test-index";
         final String type = "test-type";
         client()

server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java

@@ -124,10 +124,6 @@ protected MatchQueryBuilder doCreateTestQueryBuilder() {
             matchQuery.zeroTermsQuery(randomFrom(ZeroTermsQuery.ALL, ZeroTermsQuery.NONE));
         }
 
-        if (randomBoolean()) {
-            matchQuery.cutoffFrequency((float) 10 / randomIntBetween(1, 100));
-        }
-
         if (randomBoolean()) {
             matchQuery.autoGenerateSynonymsPhraseQuery(randomBoolean());
         }
@@ -478,6 +474,11 @@ public void testMaxBooleanClause() {
         query.setAnalyzer(new MockGraphAnalyzer(createGiantGraphMultiTerms()));
         expectThrows(BooleanQuery.TooManyClauses.class, () -> query.parse(Type.PHRASE, STRING_FIELD_NAME, ""));
     }
+
+    public void testCutoffFrequency() {
+        new MatchQueryBuilder("field", "value").cutoffFrequency((float) 10 / randomIntBetween(1, 100));
+        assertWarnings(MatchQueryBuilder.CUTOFF_FREQUENCY_DEPRECATION_MSG);
+    }
 
     private static class MockGraphAnalyzer extends Analyzer {
 

server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java

@@ -134,9 +134,6 @@ protected MultiMatchQueryBuilder doCreateTestQueryBuilder() {
         if (randomBoolean()) {
             query.tieBreaker(randomFloat());
         }
-        if (randomBoolean() && query.type() != Type.BOOL_PREFIX) {
-            query.cutoffFrequency((float) 10 / randomIntBetween(1, 100));
-        }
         if (randomBoolean()) {
             query.zeroTermsQuery(randomFrom(MatchQuery.ZeroTermsQuery.NONE, MatchQuery.ZeroTermsQuery.ALL));
         }
@@ -558,6 +555,14 @@ public void testNegativeFieldBoost() {
         assertThat(exc.getMessage(), containsString("negative [boost]"));
     }
 
+    public void testCutoffFrequency() {
+        new MultiMatchQueryBuilder("value", "field1", "field2").cutoffFrequency(10f / randomIntBetween(1, 100));
+        assertWarnings(MultiMatchQueryBuilder.CUTOFF_FREQUENCY_DEPRECATION_MSG);
+        new MultiMatchQueryBuilder("value", "field1", "field2").cutoffFrequency(Float.valueOf(10f / randomIntBetween(1, 100)));
+        assertWarnings(MultiMatchQueryBuilder.CUTOFF_FREQUENCY_DEPRECATION_MSG);
+
+    }
+
     private static IndexMetaData newIndexMeta(String name, Settings oldIndexSettings, Settings indexSettings) {
         Settings build = Settings.builder().put(oldIndexSettings)
             .put(indexSettings)

server/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java

@@ -169,6 +169,10 @@ private static QueryBuilder randomConstantScoreQuery(List<String> stringFields,
         return QueryBuilders.constantScoreQuery(randomQueryBuilder(stringFields, numericFields, numDocs, depth - 1));
     }
 
+    /**
+     * @deprecated See {@link CommonTermsQueryBuilder}
+     */
+    @Deprecated
     private static QueryBuilder randomCommonTermsQuery(List<String> fields, int numDocs) {
         int numTerms = randomInt(numDocs);