EQL: Introduce ~ grammar for case-insensitive functions (#67869)

By default, operators and string comparisons are case sensitive. This PR
enables support for case insensitive comparisons through the ~ (tilde)
symbol which works on functions.

For example:
process where endsWith~(file.path, "ExE")

will match all file.path that end with "exe", regardless of their case.

In the process, refactored the code by introducing a common class and
improving code consistency.

Fix #67868

(cherry picked from commit afa4ce0)

Author: costin

x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml

@@ -1445,7 +1445,7 @@ description = "check built-in string functions"
 [[queries]]
 name = "startsWithCaseInsensitive1"
 query = '''
-file where opcode==0 and startsWith(file_name, "explorer.")
+file where opcode==0 and startsWith~(file_name, "explorer.")
 '''
 expected_event_ids  = [88, 92]
 description = "check built-in string functions"
@@ -1454,15 +1454,15 @@ description = "check built-in string functions"
 [[queries]]
 name = "startsWithCaseInsensitive2"
 query = '''
-file where opcode==0 and startsWith(file_name, "exploRER.")
+file where opcode==0 and startsWith~(file_name, "exploRER.")
 '''
 expected_event_ids  = [88, 92]
 description = "check built-in string functions"
 
 [[queries]]
 name = "startsWithCaseInsensitive3"
 query = '''
-file where opcode==0 and startsWith(file_name, "expLORER.exe")
+file where opcode==0 and startsWith~(file_name, "expLORER.ExE")
 '''
 expected_event_ids  = [88, 92]
 description = "check built-in string functions"
@@ -1479,7 +1479,7 @@ description = "check built-in string functions"
 [[queries]]
 name = "endsWithCaseInsensitive"
 query = '''
-file where opcode==0 and endsWith(file_name, "loREr.exe")
+file where opcode==0 and endsWith~(file_name, "loREr.exe")
 '''
 expected_event_ids  = [88]
 description = "check built-in string functions"
@@ -1495,7 +1495,7 @@ description = "check built-in string functions"
 [[queries]]
 name = "endsWithAndCondition"
 query = '''
-file where opcode==0 and serial_event_id == 88 and startsWith("explorer.exeaAAAA", "EXPLORER.exe")
+file where opcode==0 and serial_event_id == 88 and startsWith~("explorer.exeaAAAA", "EXPLORER.exe")
 '''
 expected_event_ids  = [88]
 description = "check built-in string functions"
@@ -1511,7 +1511,7 @@ description = "check built-in string functions"
 [[queries]]
 name = "indexOfCaseInsensitive"
 query = '''
-file where opcode==0 and indexOf(file_name, "plore") == 2 and indexOf(file_name, ".pf") == null
+file where opcode==0 and indexOf~(file_name, "plore") == 2 and indexOf~(file_name, ".pf") == null
 '''
 expected_event_ids  = [88]
 description = "check built-in string functions"
@@ -1535,7 +1535,7 @@ description = "check built-in string functions"
 [[queries]]
 name = "indexOf3"
 query = '''
-file where opcode==0 and indexOf(file_name, "plorer.", 0) == 2
+file where opcode==0 and indexOf~(file_name, "plorer.", 0) == 2
 '''
 expected_event_ids  = [88, 92]
 description = "check built-in string functions"
@@ -1551,7 +1551,7 @@ description = "check built-in string functions"
 [[queries]]
 name = "indexOf5"
 query = '''
-file where opcode==0 and indexOf(file_name, "plorer.", 2) != null
+file where opcode==0 and indexOf~(file_name, "plorer.", 2) != null
 '''
 expected_event_ids  = [88, 92]
 description = "check built-in string functions"
@@ -1575,7 +1575,7 @@ description = "check built-in string functions"
 [[queries]]
 name = "indexOf8"
 query = '''
-file where opcode==0 and indexOf(file_name, "plorer.", 2) == 2
+file where opcode==0 and indexOf~(file_name, "pLOrer.", 2) == 2
 '''
 expected_event_ids  = [88, 92]
 description = "check substring ranges"
@@ -1599,7 +1599,7 @@ description = "check substring ranges"
 [[queries]]
 name = "indexOf11"
 query = '''
-file where opcode==0 and indexOf(file_name, "explorer.", 0) == 0
+file where opcode==0 and indexOf~(file_name, "explorer.", 0) == 0
 '''
 expected_event_ids  = [88, 92]
 description = "check substring ranges"
@@ -1901,14 +1901,14 @@ query = "file where serial_event_id % 40 == 2"
 name = "betweenCaseInsensitive1"
 expected_event_ids = [1, 2]
 query = '''
-process where between(process_name, "s", "e") : "yst"
+process where between~(process_name, "s", "E") : "yst"
 '''
 
 [[queries]]
 name = "betweenCaseInsensitive2"
 expected_event_ids = [1, 2]
 query = '''
-process where between(process_name, "s", "e", false) : "yst"
+process where between~(process_name, "s", "E", false) : "yst"
 '''
 
 [[queries]]
@@ -1929,7 +1929,7 @@ process where between(process_name, "S", "e", true) : "ystem Idle Proc"
 name = "betweenCaseInsensitive3"
 expected_event_ids = [1]
 query = '''
-process where between(process_name, "s", "e", true) : "ystem Idle Proc"
+process where between~(process_name, "s", "e", true) : "ystem Idle Proc"
 '''
 
 [[queries]]

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml

@@ -8,99 +8,6 @@
 # in order to allow at least one round-trip test with the test harness.
 # This will be removed once the EQL implementation is wired and actually supports this query.
 
-[[queries]]
-name = "startsWithCaseInsensitive1"
-query = '''
-file where opcode==0 and startsWith(file_name, "explorer.")
-'''
-expected_event_ids  = [88, 92]
-description = "check built-in string functions"
-
-[[queries]]
-name = "startsWithCaseInsensitive2"
-query = '''
-file where opcode==0 and startsWith(file_name, "exploRER.")
-'''
-expected_event_ids  = [88, 92]
-description = "check built-in string functions"
-
-[[queries]]
-name = "startsWithCaseInsensitive3"
-query = '''
-file where opcode==0 and startsWith(file_name, "expLORER.exe")
-'''
-expected_event_ids  = [88, 92]
-description = "check built-in string functions"
-
-[[queries]]
-name = "endsWithCaseInsensitive"
-query = '''
-file where opcode==0 and endsWith(file_name, "loREr.exe")
-'''
-expected_event_ids  = [88]
-description = "check built-in string functions"
-
-[[queries]]
-name = "endsWithAndCondition"
-query = '''
-file where opcode==0 and serial_event_id == 88 and startsWith("explorer.exeaAAAA", "EXPLORER.exe")
-'''
-expected_event_ids  = [88]
-description = "check built-in string functions"
-
-[[queries]]
-name = "indexOf3"
-query = '''
-file where opcode==0 and indexOf(file_name, "plorer.", 0) == 2
-'''
-expected_event_ids  = [88, 92]
-description = "check built-in string functions"
-
-[[queries]]
-name = "indexOf5"
-query = '''
-file where opcode==0 and indexOf(file_name, "plorer.", 2) != null
-'''
-expected_event_ids  = [88, 92]
-description = "check built-in string functions"
-
-[[queries]]
-name = "indexOf8"
-query = '''
-file where opcode==0 and indexOf(file_name, "plorer.", 2) == 2
-'''
-expected_event_ids  = [88, 92]
-description = "check substring ranges"
-
-[[queries]]
-name = "indexOf11"
-query = '''
-file where opcode==0 and indexOf(file_name, "explorer.", 0) == 0
-'''
-expected_event_ids  = [88, 92]
-description = "check substring ranges"
-
-[[queries]]
-name = "betweenCaseInsensitive1"
-expected_event_ids = [1, 2]
-query = '''
-process where between(process_name, "s", "e") : "yst"
-'''
-
-[[queries]]
-name = "betweenCaseInsensitive2"
-expected_event_ids = [1, 2]
-query = '''
-process where between(process_name, "s", "e", false) : "yst"
-'''
-
-[[queries]]
-name = "betweenCaseInsensitive3"
-expected_event_ids = [1]
-query = '''
-process where between(process_name, "s", "e", true) : "ystem Idle Proc"
-'''
-
 [[queries]]
 name = "twoSequencesWithKeys"
 query = '''

x-pack/plugin/eql/src/main/antlr/EqlBase.g4

@@ -117,6 +117,7 @@ functionExpression
 
 functionName
     : IDENTIFIER
+    | TILDE_IDENTIFIER
     ;
 
 constant
@@ -226,6 +227,10 @@ QUOTED_IDENTIFIER
     : '`' ( ~'`' | '``' )* '`'
     ;
 
+TILDE_IDENTIFIER
+    : LETTER (LETTER | DIGIT | '_')* '~'
+    ;
+
 eventValue
     : STRING
     | IDENTIFIER

x-pack/plugin/eql/src/main/antlr/EqlBase.tokens

@@ -39,9 +39,10 @@ INTEGER_VALUE=38
 DECIMAL_VALUE=39
 IDENTIFIER=40
 QUOTED_IDENTIFIER=41
-LINE_COMMENT=42
-BRACKETED_COMMENT=43
-WS=44
+TILDE_IDENTIFIER=42
+LINE_COMMENT=43
+BRACKETED_COMMENT=44
+WS=45
 'and'=1
 'any'=2
 'by'=3

x-pack/plugin/eql/src/main/antlr/EqlBaseLexer.tokens

@@ -39,9 +39,10 @@ INTEGER_VALUE=38
 DECIMAL_VALUE=39
 IDENTIFIER=40
 QUOTED_IDENTIFIER=41
-LINE_COMMENT=42
-BRACKETED_COMMENT=43
-WS=44
+TILDE_IDENTIFIER=42
+LINE_COMMENT=43
+BRACKETED_COMMENT=44
+WS=45
 'and'=1
 'any'=2
 'by'=3

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionDefinition.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function;
+
+import org.elasticsearch.xpack.ql.expression.function.Function;
+import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
+
+import java.util.List;
+
+/**
+ * Dedicated wrapper for EQL specific function definitions.
+ * Used mainly for validating case sensitivity of wrapping functions.
+ */
+class EqlFunctionDefinition extends FunctionDefinition {
+
+    private final boolean caseAware;
+
+    protected EqlFunctionDefinition(String name,
+                                    List<String> aliases,
+                                    Class<? extends Function> clazz,
+                                    boolean caseAware,
+                                    Builder builder) {
+        super(name, aliases, clazz, builder);
+        this.caseAware = caseAware;
+    }
+
+    public boolean isCaseAware() {
+        return caseAware;
+    }
+
+    @Override
+    protected Builder builder() {
+        return super.builder();
+    }
+}