EQL: Add wildcard functionality to : operator (#65188)

Additionally, restrict the grammar to allow only a constant on the right hand side of :.

Fix #65154

(cherry picked from commit db43a1c)

Author: costin

x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml

@@ -342,19 +342,19 @@ query = '''
 registry where length(bytes_written_string_list) == 2 and bytes_written_string_list[1] : "EN"
 '''
 
-# [[queries]]
-# name = "keyPathWildcard"
-# query = '''
-# registry where key_path : "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"
-# '''
-# expected_event_ids  = [79]
-#
-# [[queries]]
-# name = "processPathWildcardAndIN"
-# query = '''
-# process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4)
-# '''
-# expected_event_ids  = [84, 85]
+[[queries]]
+name = "keyPathWildcard"
+query = '''
+registry where key_path : "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"
+'''
+expected_event_ids  = [79]
+
+[[queries]]
+name = "processPathWildcardAndIN"
+query = '''
+process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4)
+'''
+expected_event_ids  = [84, 85]
 
 [[queries]]
 name = "descendant1"
@@ -385,13 +385,13 @@ process where opcode==1 and process_name : "smss.exe"
 '''
 expected_event_ids  = [78]
 
-# [[queries]]
-# name = "wildcardAndMultipleConditions1"
-# query = '''
-# file where file_path:"*\\red_ttp\\winin*.*"
-#   and opcode in (0,1,2) and user_name:"vagrant"
-# '''
-# expected_event_ids  = [83, 86]
+[[queries]]
+name = "wildcardAndMultipleConditions1"
+query = '''
+file where file_path:"*\\red_ttp\\winin*.*"
+  and opcode in (0,1,2) and user_name:"vagrant"
+'''
+expected_event_ids  = [83, 86]
 
 [[queries]]
 name = "wildcardAndMultipleConditions2"
@@ -401,13 +401,13 @@ file where file_path:"*\\red_ttp\\winin*.*"
 '''
 expected_event_ids  = []
 
-# [[queries]]
-# name = "wildcardAndMultipleConditions3"
-# query = '''
-# file where file_path:"*\\red_ttp\\winin*.*"
-#   and opcode not in (3, 4, 5, 6 ,7) and user_name:"vagrant"
-# '''
-# expected_event_ids  = [83, 86]
+[[queries]]
+name = "wildcardAndMultipleConditions3"
+query = '''
+file where file_path:"*\\red_ttp\\winin*.*"
+  and opcode not in (3, 4, 5, 6 ,7) and user_name:"vagrant"
+'''
+expected_event_ids  = [83, 86]
 
 
 [[queries]]
@@ -878,16 +878,16 @@ sequence
 '''
 expected_event_ids  = [87, 92]
 
-# [[queries]]
-# name = "doubleSameSequenceWithByUntilAndHead1"
-# query = '''
-# sequence
-#   [file where opcode==0 and file_name:"*.exe"] by unique_pid
-#   [file where opcode==0 and file_name:"*.exe"] by unique_pid
-# until [process where opcode==5000] by unique_ppid
-# | head 1
-# '''
-# expected_event_ids  = [55, 61]
+[[queries]]
+name = "doubleSameSequenceWithByUntilAndHead1"
+query = '''
+sequence
+  [file where opcode==0 and file_name:"*.exe"] by unique_pid
+  [file where opcode==0 and file_name:"*.exe"] by unique_pid
+until [process where opcode==5000] by unique_ppid
+| head 1
+'''
+expected_event_ids  = [55, 61]
 
 [[queries]]
 name = "doubleSameSequenceWithByUntilAndHead2"
@@ -1040,16 +1040,16 @@ query = '''
 registry where length(bad_field) > 0
 '''
 
-# [[queries]]
-# name = "multipleConditions2"
-# query = '''
-# process where opcode == 1
-#   and process_name in ("net.exe", "net1.exe")
-#   and not (parent_process_name : "net.exe"
-#   and process_name : "net1.exe")
-#   and command_line : "*group *admin*" and command_line != "* /add*"
-#   '''
-# expected_event_ids  = [97]
+[[queries]]
+name = "multipleConditions2"
+query = '''
+process where opcode == 1
+  and process_name in ("net.exe", "net1.exe")
+  and not (parent_process_name : "net.exe"
+  and process_name : "net1.exe")
+  and command_line : "*group *admin*" and command_line != "* /add*"
+  '''
+expected_event_ids  = [97]
 
 [[queries]]
 name = "anyWithUnique"
@@ -1246,26 +1246,26 @@ sequence
 '''
 expected_event_ids  = [54, 55, 56, 54, 61, 62, 54, 67, 68, 54, 72, 73]
 
-# [[queries]]
-# name = "wildcard1"
-# query = '''
-# process where command_line : "*%*"
-# '''
-# expected_event_ids  = [4, 6, 28]
-#
-# [[queries]]
-# name = "wildcard2"
-# query = '''
-# process where command_line : "*%*%*"
-# '''
-# expected_event_ids  = [4, 6, 28]
-#
-# [[queries]]
-# name = "wildcard3"
-# query = '''
-# process where command_line : "%*%*"
-# '''
-# expected_event_ids  = [4, 6, 28]
+[[queries]]
+name = "wildcard1"
+query = '''
+process where command_line : "*%*"
+'''
+expected_event_ids  = [4, 6, 28]
+
+[[queries]]
+name = "wildcard2"
+query = '''
+process where command_line : "*%*%*"
+'''
+expected_event_ids  = [4, 6, 28]
+
+[[queries]]
+name = "wildcard3"
+query = '''
+process where command_line : "%*%*"
+'''
+expected_event_ids  = [4, 6, 28]
 
 [[queries]]
 name = "uniqueCount1"

x-pack/plugin/eql/qa/correctness/src/javaRestTest/resources/queries.toml

@@ -487,26 +487,26 @@ sequence by hostname with maxspan=5m
 time = 7.179757833480835
 type = "sequence"
 
-#[[queries]]
-#queryNo = 28
-#case_insensitive = true
-#count = 1
-#expected_event_ids = [3299718, 3364047]
-#filter_counts = [24, 3, 37]
-#filters = [
-#  'process where process_name == "powershell.exe" and opcode == 1',
-#  'powershell where message == "*Get-NetShare*"',
-#  'process where process_name == "powershell.exe" and opcode == 2'
-#]
-#query = """
-#sequence by hostname, pid
-#   [process where process_name == "powershell.exe" and opcode == 1]
-#   [powershell where message == "*Get-NetShare*"]
-#until
-#   [process where process_name == "powershell.exe" and opcode == 2]
-#"""
-#time = 2.123962879180908
-#type = "sequence"
+[[queries]]
+queryNo = 28
+case_insensitive = true
+count = 1
+expected_event_ids = [3299718, 3364047]
+filter_counts = [24, 3, 37]
+filters = [
+  'process where process_name : "powershell.exe" and opcode == 1',
+  'powershell where message : "*Get-NetShare*"',
+  'process where process_name : "powershell.exe" and opcode == 2'
+]
+query = """
+sequence by hostname, pid
+   [process where process_name : "powershell.exe" and opcode == 1]
+   [powershell where message : "*Get-NetShare*"]
+until
+   [process where process_name : "powershell.exe" and opcode == 2]
+"""
+time = 2.123962879180908
+type = "sequence"
 
 [[queries]]
 queryNo = 29

x-pack/plugin/eql/src/main/antlr/EqlBase.g4

@@ -100,6 +100,7 @@ operatorExpression
 //   https://github.com/antlr/antlr4/issues/781
 predicate
     : NOT? kind=IN LP expression (COMMA expression)* RP
+    | seqPredicate
     ;
 
 primaryExpression
@@ -124,8 +125,12 @@ constant
     | string                                                                            #stringLiteral
     ;
 
+seqPredicate
+    : SEQ constant                                                                      #seqValue
+    ;
+
 comparisonOperator
-    : SEQ | EQ | NEQ | LT | LTE | GT | GTE
+    : EQ | NEQ | LT | LTE | GT | GTE
     ;
 
 booleanValue

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java

@@ -7,12 +7,15 @@
 package org.elasticsearch.xpack.eql.optimizer;
 
 import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
+import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveBinaryComparison;
+import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveNotEquals;
 import org.elasticsearch.xpack.eql.plan.logical.Join;
 import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter;
 import org.elasticsearch.xpack.eql.plan.logical.LimitWithOffset;
 import org.elasticsearch.xpack.eql.plan.physical.LocalRelation;
 import org.elasticsearch.xpack.eql.session.Payload.Type;
 import org.elasticsearch.xpack.eql.util.MathUtils;
+import org.elasticsearch.xpack.eql.util.StringUtils;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.Literal;
@@ -23,12 +26,14 @@
 import org.elasticsearch.xpack.ql.expression.predicate.Predicates;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic;
+import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
+import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanFunctionEqualsElimination;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
@@ -66,6 +71,7 @@ public LogicalPlan optimize(LogicalPlan verified) {
     @Override
     protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
         Batch substitutions = new Batch("Substitution", Limiter.ONCE,
+                new ReplaceWildcards(),
                 new ReplaceSurrogateFunction(),
                 new ReplaceRegexMatch());
 
@@ -106,6 +112,47 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
         return asList(substitutions, syntactic, operators, constraints, operators, ordering, local, label);
     }
 
+    private static class ReplaceWildcards extends OptimizerRule<Filter> {
+
+        @Override
+        protected LogicalPlan rule(Filter filter) {
+            return filter.transformExpressionsUp(e -> {
+                // expr : "wildcard*phrase" || expr !: "wildcard*phrase"
+                if (e instanceof InsensitiveBinaryComparison) {
+                    InsensitiveBinaryComparison cmp = (InsensitiveBinaryComparison) e;
+
+                    Expression target = null;
+                    String wildString = null;
+
+                    // check only the right side
+                    if (isWildcard(cmp.right())) {
+                        wildString = (String) cmp.right().fold();
+                        target = cmp.left();
+                    }
+
+                    if (target != null) {
+                        Expression like = new Like(e.source(), target, StringUtils.toLikePattern(wildString), true);
+                        if (e instanceof InsensitiveNotEquals) {
+                            like = new Not(e.source(), like);
+                        }
+
+                        e = like;
+                    }
+                }
+
+                return e;
+            });
+        }
+
+        private static boolean isWildcard(Expression expr) {
+            if (expr instanceof Literal) {
+                Object value = expr.fold();
+                return value instanceof String && ((String) value).contains("*");
+            }
+            return false;
+        }
+    }
+
     private static class AddMissingEquals extends OptimizerRule<Filter> {
 
         @Override

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseListener.java

@@ -431,6 +431,18 @@ class EqlBaseBaseListener implements EqlBaseListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitStringLiteral(EqlBaseParser.StringLiteralContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterSeqValue(EqlBaseParser.SeqValueContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitSeqValue(EqlBaseParser.SeqValueContext ctx) { }
   /**
    * {@inheritDoc}
    *

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseVisitor.java

@@ -256,6 +256,13 @@ class EqlBaseBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements EqlBa
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitStringLiteral(EqlBaseParser.StringLiteralContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitSeqValue(EqlBaseParser.SeqValueContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseListener.java

@@ -391,6 +391,18 @@ interface EqlBaseListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitStringLiteral(EqlBaseParser.StringLiteralContext ctx);
+  /**
+   * Enter a parse tree produced by the {@code seqValue}
+   * labeled alternative in {@link EqlBaseParser#seqPredicate}.
+   * @param ctx the parse tree
+   */
+  void enterSeqValue(EqlBaseParser.SeqValueContext ctx);
+  /**
+   * Exit a parse tree produced by the {@code seqValue}
+   * labeled alternative in {@link EqlBaseParser#seqPredicate}.
+   * @param ctx the parse tree
+   */
+  void exitSeqValue(EqlBaseParser.SeqValueContext ctx);
   /**
    * Enter a parse tree produced by {@link EqlBaseParser#comparisonOperator}.
    * @param ctx the parse tree