QL: Optimize Like/Rlike all (#62682)

Replace common Like and RLike queries that match all characters with
IsNotNull (exists) queries

Fix #62585

Author: costin

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java

@@ -39,6 +39,7 @@
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneLiteralsInOrderBy;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceSurrogateFunction;
+import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceMatchAll;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SetAsOptimized;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.TransformDirection;
 import org.elasticsearch.xpack.ql.plan.logical.Filter;
@@ -66,7 +67,11 @@ public LogicalPlan optimize(LogicalPlan verified) {
     @Override
     protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
         Batch substitutions = new Batch("Substitution", Limiter.ONCE,
-                new ReplaceSurrogateFunction());
+            // needed for replace wildcards
+            new BooleanLiteralsOnTheRight(),
+            new ReplaceWildcards(),
+            new ReplaceSurrogateFunction(),
+            new ReplaceMatchAll());
 
         Batch operators = new Batch("Operator Optimization",
                 new ConstantFolding(),
@@ -75,7 +80,6 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
                 new BooleanLiteralsOnTheRight(),
                 new BooleanEqualsSimplification(),
                 // needs to occur before BinaryComparison combinations
-                new ReplaceWildcards(),
                 new ReplaceNullChecks(),
                 new PropagateEquals(),
                 new CombineBinaryComparisons(),
@@ -106,7 +110,7 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
     private static class ReplaceWildcards extends OptimizerRule<Filter> {
 
         private static boolean isWildcard(Expression expr) {
-            if (expr.foldable()) {
+            if (expr instanceof Literal) {
                 Object value = expr.fold();
                 return value instanceof String && value.toString().contains("*");
             }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java

@@ -39,6 +39,8 @@ final class QueryTranslator {
             new ExpressionTranslators.BinaryComparisons(),
             new ExpressionTranslators.Ranges(),
             new BinaryLogic(),
+            new ExpressionTranslators.IsNotNulls(),
+            new ExpressionTranslators.IsNulls(),
             new ExpressionTranslators.Nots(),
             new ExpressionTranslators.Likes(),
             new ExpressionTranslators.InComparisons(),

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.planner;
+
+import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class QueryTranslationTests extends AbstractQueryFolderTestCase {
+
+    public void testLikeOptimization() throws Exception {
+        PhysicalPlan plan = plan("process where process_name == \"*\" ");
+        assertThat(asQuery(plan), containsString("\"exists\":{\"field\":\"process_name\""));
+    }
+
+    public void testMatchOptimization() throws Exception {
+        PhysicalPlan plan = plan("process where match(process_name, \".*\") ");
+        assertThat(asQuery(plan), containsString("\"exists\":{\"field\":\"process_name\""));
+    }
+
+    private static String asQuery(PhysicalPlan plan) {
+        return plan.toString().replaceAll("\\s+", "");
+    }
+}

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java

@@ -5,6 +5,11 @@
  */
 package org.elasticsearch.xpack.ql.expression.predicate.regex;
 
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.MinimizationOperations;
+import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.xpack.ql.util.StringUtils;
 
 import java.util.Objects;
@@ -48,6 +53,12 @@ public String asJavaRegex() {
         return regex;
     }
 
+    @Override
+    public boolean matchesAll() {
+        Automaton automaton = WildcardQuery.toAutomaton(new Term(null, wildcard));
+        return Operations.isTotal(MinimizationOperations.minimize(automaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES));
+    }
+
     /**
      * Returns the pattern in (Lucene) wildcard format.
      */

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java

@@ -5,6 +5,9 @@
  */
 package org.elasticsearch.xpack.ql.expression.predicate.regex;
 
+import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.automaton.RegExp;
+
 public class RLikePattern implements StringPattern {
 
     private final String regexpPattern;
@@ -17,4 +20,9 @@ public RLikePattern(String regexpPattern) {
     public String asJavaRegex() {
         return regexpPattern;
     }
+
+    @Override
+    public boolean matchesAll() {
+        return Operations.isTotal(new RegExp(regexpPattern).toAutomaton());
+    }
 }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java

@@ -25,12 +25,12 @@
 public abstract class RegexMatch<T extends StringPattern> extends UnaryScalarFunction {
 
     private final T pattern;
-    
+
     protected RegexMatch(Source source, Expression value, T pattern) {
         super(source, value);
         this.pattern = pattern;
     }
-    
+
     public T pattern() {
         return pattern;
     }
@@ -65,6 +65,10 @@ public Boolean fold() {
         return RegexProcessor.RegexOperation.match(val, pattern().asJavaRegex());
     }
 
+    public boolean matchesAll() {
+        return pattern.matchesAll();
+    }
+
     @Override
     protected Processor makeProcessor() {
         return new RegexProcessor(pattern().asJavaRegex());

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java

@@ -10,4 +10,11 @@ interface StringPattern {
      * Returns the pattern in (Java) regex format.
      */
     String asJavaRegex();
+
+    /**
+     * Hint method on whether this pattern matches everything or not.
+     */
+    default boolean matchesAll() {
+        return false;
+    }
 }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
@@ -26,6 +27,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch;
 import org.elasticsearch.xpack.ql.plan.logical.Filter;
 import org.elasticsearch.xpack.ql.plan.logical.Limit;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
@@ -50,7 +52,7 @@
 
 
 public final class OptimizerRules {
-    
+
     public static final class ConstantFolding extends OptimizerExpressionRule {
 
         public ConstantFolding() {
@@ -90,7 +92,7 @@ protected Expression rule(Expression e) {
             return e;
         }
     }
-    
+
     public static final class BooleanSimplification extends OptimizerExpressionRule {
 
         public BooleanSimplification() {
@@ -230,7 +232,7 @@ private Expression literalToTheRight(BinaryOperator<?, ?, ?, ?> be) {
             return be.left() instanceof Literal && !(be.right() instanceof Literal) ? be.swapLeftAndRight() : be;
         }
     }
-    
+
     /**
      * Propagate Equals to eliminate conjuncted Ranges or BinaryComparisons.
      * When encountering a different Equals, non-containing {@link Range} or {@link BinaryComparison}, the conjunction becomes false.
@@ -537,7 +539,7 @@ private Expression propagate(Or or) {
             return updated ? Predicates.combineOr(CollectionUtils.combine(exps, equals, notEquals, inequalities, ranges)) : or;
         }
     }
-    
+
     public static final class CombineBinaryComparisons extends OptimizerExpressionRule {
 
         public CombineBinaryComparisons() {
@@ -1046,7 +1048,7 @@ protected Expression rule(Expression e) {
             return e;
         }
     }
-    
+
     public abstract static class PruneFilters extends OptimizerRule<Filter> {
 
         @Override
@@ -1094,7 +1096,7 @@ private static Expression foldBinaryLogic(Expression expression) {
             return expression;
         }
     }
-    
+
     public static final class PruneLiteralsInOrderBy extends OptimizerRule<OrderBy> {
 
         @Override
@@ -1120,7 +1122,7 @@ protected LogicalPlan rule(OrderBy ob) {
             return ob;
         }
     }
-    
+
 
     public abstract static class SkipQueryOnLimitZero extends OptimizerRule<Limit> {
         @Override
@@ -1136,6 +1138,23 @@ protected LogicalPlan rule(Limit limit) {
         protected abstract LogicalPlan skipPlan(Limit limit);
     }
 
+    public static class ReplaceMatchAll extends OptimizerExpressionRule {
+
+        public ReplaceMatchAll() {
+            super(TransformDirection.DOWN);
+        }
+
+        protected Expression rule(Expression e) {
+            if (e instanceof RegexMatch) {
+                RegexMatch<?> regexMatch = (RegexMatch<?>) e;
+                if (regexMatch.matchesAll()) {
+                    return new IsNotNull(e.source(), regexMatch.field());
+                }
+            }
+            return e;
+        }
+    }
+
     public static final class SetAsOptimized extends Rule<LogicalPlan, LogicalPlan> {
 
         @Override

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java

@@ -20,6 +20,8 @@
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
+import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
@@ -34,6 +36,7 @@
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
 import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch;
 import org.elasticsearch.xpack.ql.querydsl.query.BoolQuery;
+import org.elasticsearch.xpack.ql.querydsl.query.ExistsQuery;
 import org.elasticsearch.xpack.ql.querydsl.query.MatchQuery;
 import org.elasticsearch.xpack.ql.querydsl.query.MultiMatchQuery;
 import org.elasticsearch.xpack.ql.querydsl.query.NotQuery;
@@ -73,6 +76,8 @@ public final class ExpressionTranslators {
             new BinaryComparisons(),
             new Ranges(),
             new BinaryLogic(),
+            new IsNulls(),
+            new IsNotNulls(),
             new Nots(),
             new Likes(),
             new InComparisons(),
@@ -206,6 +211,46 @@ public static Query doTranslate(Not not, TranslatorHandler handler) {
         }
     }
 
+    public static class IsNotNulls extends ExpressionTranslator<IsNotNull> {
+
+        @Override
+        protected Query asQuery(IsNotNull isNotNull, TranslatorHandler handler) {
+            return doTranslate(isNotNull, handler);
+        }
+
+        public static Query doTranslate(IsNotNull isNotNull, TranslatorHandler handler) {
+            Query query = null;
+
+            if (isNotNull.field() instanceof FieldAttribute) {
+                query = new ExistsQuery(isNotNull.source(), handler.nameOf(isNotNull.field()));
+            } else {
+                query = new ScriptQuery(isNotNull.source(), isNotNull.asScript());
+            }
+
+            return handler.wrapFunctionQuery(isNotNull, isNotNull.field(), query);
+        }
+    }
+
+    public static class IsNulls extends ExpressionTranslator<IsNull> {
+
+        @Override
+        protected Query asQuery(IsNull isNull, TranslatorHandler handler) {
+            return doTranslate(isNull, handler);
+        }
+
+        public static Query doTranslate(IsNull isNull, TranslatorHandler handler) {
+            Query query = null;
+
+            if (isNull.field() instanceof FieldAttribute) {
+                query = new NotQuery(isNull.source(), new ExistsQuery(isNull.source(), handler.nameOf(isNull.field())));
+            } else {
+                query = new ScriptQuery(isNull.source(), isNull.asScript());
+            }
+
+            return handler.wrapFunctionQuery(isNull, isNull.field(), query);
+        }
+    }
+
     // assume the Optimizer properly orders the predicates to ease the translation
     public static class BinaryComparisons extends ExpressionTranslator<BinaryComparison> {
 

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java

@@ -62,7 +62,7 @@ public static String camelCaseToUnderscore(String string) {
         }
         return sb.toString().toUpperCase(Locale.ROOT);
     }
-    
+
     //CAMEL_CASE to camelCase
     public static String underscoreToLowerCamelCase(String string) {
         if (!Strings.hasText(string)) {
@@ -337,4 +337,4 @@ public static String ordinal(int i) {
 
         }
     }
-}
+}

x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPatternTests.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ql.expression.predicate.regex;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class StringPatternTests extends ESTestCase {
+
+    private boolean isTotalWildcard(String pattern, char escape) {
+        return new LikePattern(pattern, escape).matchesAll();
+    }
+
+    private boolean isTotalRegex(String pattern) {
+        return new RLikePattern(pattern).matchesAll();
+    }
+
+    public void testWildcardMatchAll() throws Exception {
+        assertTrue(isTotalWildcard("%", '0'));
+        assertTrue(isTotalWildcard("%%", '0'));
+
+        assertFalse(isTotalWildcard("a%", '0'));
+        assertFalse(isTotalWildcard("%_", '0'));
+        assertFalse(isTotalWildcard("%_%_%", '0'));
+        assertFalse(isTotalWildcard("_%", '0'));
+        assertFalse(isTotalWildcard("0%", '0'));
+    }
+
+    public void testRegexMatchAll() throws Exception {
+        assertTrue(isTotalRegex(".*"));
+        assertTrue(isTotalRegex(".*.*"));
+        assertTrue(isTotalRegex(".*.?"));
+        assertTrue(isTotalRegex(".?.*"));
+        assertTrue(isTotalRegex(".*.?.*"));
+
+        assertFalse(isTotalRegex("..*"));
+        assertFalse(isTotalRegex("ab."));
+        assertFalse(isTotalRegex("..?"));
+    }
+}